movefasta/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2025-06-10 03:23:29 +03:00
Code Issues Projects Releases Packages Wiki Activity

freebsd.source: Use hashes from git archive

Browse source 
An in-tree dependency of FreeBSD uses an `export-ignore` directive
to hide certain files from the git export.
Unfortunately, that means that `fetchFromGitHub` has different outputs
when `forceFetchGit = true` and `forceFetchGit = false`.

We cannot download source with git during native FreeBSD bootstrap due
to a circular dependency,
so change the update script to output archives matching GitHub.
This commit is contained in:
Artemis Tosini 2025-04-09 22:47:18 +00:00
parent 1b62e27af6
commit e6b11fe5ef
No known key found for this signature in database
GPG key ID: EE5227935FE3FF18
3 changed files with 42 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

12
pkgs/os-specific/bsd/freebsd/pkgs/source.nix
View file

@ -1,16 +1,12 @@
{ fetchFromGitHub, sourceData }: { fetchFromGitHub, sourceData }:
# Using fetchFromGitHub from their mirror because it's a lot faster than their git server # Using fetchFromGitHub from their mirror because we cannot use git during bootstrap
# If you want you could fetchgit from "https://git.FreeBSD.org/src.git" instead. # If you want you could fetchurl from "https://cgit.FreeBSD.org/src" instead.
# The update script still pulls directly from git.freebsd.org # The update script still pulls directly from git.freebsd.org
# Note that hashes with `forceFetchGit = true` may be different due to `export-ignore`
# directives in gitattributes files.
fetchFromGitHub { fetchFromGitHub {
owner = "freebsd"; owner = "freebsd";
repo = "freebsd-src"; repo = "freebsd-src";
inherit (sourceData) rev hash; inherit (sourceData) rev hash;
# The GitHub export excludes some files in the git source
# that were marked `export-ignore`.
# A normal git checkout will keep those files,
# matching the update script
forceFetchGit = true;
} }

49
pkgs/os-specific/bsd/freebsd/update.py
View file

@ -11,12 +11,13 @@ import pandas
import re import re
import subprocess import subprocess
import sys import sys
import tarfile
import tempfile import tempfile
import typing import typing
import urllib.request import urllib.request
_QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$') _QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$')
_RELEASE_PATCH_PATTERN = re.compile('^RELEASE-p([0-9]+)$') _RELEASE_PATCH_PATTERN = re.compile("^RELEASE-p([0-9]+)$")
BASE_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_DIR = os.path.dirname(os.path.abspath(__file__))
MIN_VERSION = packaging.version.Version("13.0.0") MIN_VERSION = packaging.version.Version("13.0.0")
MAIN_BRANCH = "main" MAIN_BRANCH = "main"
@ -32,7 +33,7 @@ BRANCH_PATTERN = re.compile(
def request_supported_refs() -> list[str]: def request_supported_refs() -> list[str]:
# Looks pretty shady but I think this should work with every version of the page in the last 20 years # Looks pretty shady but I think this should work with every version of the page in the last 20 years
r = re.compile("^h\d$", re.IGNORECASE) r = re.compile(r"^h\d$", re.IGNORECASE)
soup = bs4.BeautifulSoup( soup = bs4.BeautifulSoup(
urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml" urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml"
) )
@ -45,11 +46,11 @@ def request_supported_refs() -> list[str]:
return list(df["Branch"]) return list(df["Branch"])
def query_version(repo: git.Repo) -> dict[str, typing.Any]: def query_version(work_dir: str) -> dict[str, typing.Any]:
# This only works on FreeBSD 13 and later # This only works on FreeBSD 13 and later
text = ( text = (
subprocess.check_output( subprocess.check_output(
["bash", os.path.join(repo.working_dir, "sys", "conf", "newvers.sh"), "-v"] ["bash", os.path.join(work_dir, "sys", "conf", "newvers.sh"), "-v"]
) )
.decode("utf-8") .decode("utf-8")
.strip() .strip()
@ -86,17 +87,22 @@ def handle_commit(
print(f"{ref_name}: revision still {rev.hexsha}, skipping") print(f"{ref_name}: revision still {rev.hexsha}, skipping")
return old_versions[ref_name] return old_versions[ref_name]
repo.git.checkout(rev) tar_content = io.BytesIO()
print(f"{ref_name}: checked out {rev.hexsha}") repo.archive(tar_content, rev, format="tar")
tar_content.seek(0)
with tempfile.TemporaryDirectory(dir="/tmp") as work_dir:
file = tarfile.TarFile(fileobj=tar_content)
file.extractall(path=work_dir, filter="data")
full_hash = ( full_hash = (
subprocess.check_output(["nix", "hash", "path", "--sri", repo.working_dir]) subprocess.check_output(["nix", "hash", "path", "--sri", work_dir])
.decode("utf-8") .decode("utf-8")
.strip() .strip()
) )
print(f"{ref_name}: hash is {full_hash}") print(f"{ref_name}: hash is {full_hash}")
version = query_version(repo) version = query_version(work_dir)
print(f"{ref_name}: version is {version['version']}") print(f"{ref_name}: version is {version['version']}")
return { return {
@ -115,30 +121,19 @@ def main() -> None:
print(f"Selected temporary directory {temp_dir.name}") print(f"Selected temporary directory {temp_dir.name}")
if len(sys.argv) >= 2: if len(sys.argv) >= 2:
orig_repo = git.Repo(sys.argv[1]) repo = git.Repo(sys.argv[1])
print(f"Fetching updates on {orig_repo.git_dir}") print(f"Fetching updates on {repo.git_dir}")
orig_repo.remote("origin").fetch() repo.remote("origin").fetch()
else: else:
print("Cloning source repo") print("Cloning source repo")
orig_repo = git.Repo.clone_from( repo = git.Repo.clone_from(
"https://git.FreeBSD.org/src.git", to_path=os.path.join(temp_dir.name, "orig") "https://git.FreeBSD.org/src.git", to_path=temp_dir.name
) )
supported_refs = request_supported_refs() supported_refs = request_supported_refs()
print(f"Supported refs are: {' '.join(supported_refs)}") print(f"Supported refs are: {' '.join(supported_refs)}")
print("Doing git crimes, do not run `git worktree prune` until after script finishes!") print(f"git directory {repo.git_dir}")
workdir = os.path.join(temp_dir.name, "work")
git.cmd.Git(orig_repo.git_dir).worktree("add", "--orphan", workdir)
# Have to create object before removing .git otherwise it will complain
repo = git.Repo(workdir)
repo.git.set_persistent_git_options(git_dir=repo.git_dir)
# Remove so that nix hash doesn't see the file
os.remove(os.path.join(workdir, ".git"))
print(f"Working in directory {repo.working_dir} with git directory {repo.git_dir}")
try: try:
with open(os.path.join(BASE_DIR, "versions.json"), "r") as f: with open(os.path.join(BASE_DIR, "versions.json"), "r") as f:
@ -191,10 +186,10 @@ def main() -> None:
) )
versions[fullname] = result versions[fullname] = result
with open(os.path.join(BASE_DIR, "versions.json"), "w") as out: with open(os.path.join(BASE_DIR, "versions.json"), "w") as out:
json.dump(versions, out, sort_keys=True, indent=2) json.dump(versions, out, sort_keys=True, indent=2)
out.write("\n") out.write("\n")
if __name__ == '__main__':
if __name__ == "__main__":
main() main()

20
pkgs/os-specific/bsd/freebsd/versions.json
View file

@ -1,15 +1,15 @@
{ {
"main": { "main": {
"hash": "sha256-xsgY5Ex/B5ngOTa5OZRauSaSYvET5lWI7veJRrSq1oY=", "hash": "sha256-f69U2FX+3GQimdUJunWApFseeKJhzjgNw4h26ZBPiT0=",
"ref": "main", "ref": "main",
"refType": "branch", "refType": "branch",
"rev": "c5773d366ecc5271b9bd6e5506c00fb3520f19ae", "rev": "9357c694e8dca627c25b15529e8435b2ab3dd48b",
"supported": false, "supported": false,
"version": { "version": {
"branch": "CURRENT", "branch": "CURRENT",
"major": 15, "major": 15,
"minor": 0, "minor": 0,
"reldate": "1500035", "reldate": "1500036",
"release": "15.0-CURRENT", "release": "15.0-CURRENT",
"revision": "15.0", "revision": "15.0",
"type": "FreeBSD", "type": "FreeBSD",
@ -161,7 +161,7 @@
} }
}, },
"release/14.2.0": { "release/14.2.0": {
"hash": "sha256-qZkeuUZbuPOvXZBgP5x6Hii1YN7XdDJzwZeYacIR5BI=", "hash": "sha256-q5/0pKRlmzZUN+VMwUImkve+NTT1+O68YlwgXp2/aQM=",
"ref": "release/14.2.0", "ref": "release/14.2.0",
"refType": "tag", "refType": "tag",
"rev": "c8918d6c7412fce87922e9bd7e4f5c7d7ca96eb7", "rev": "c8918d6c7412fce87922e9bd7e4f5c7d7ca96eb7",
@ -308,7 +308,7 @@
"ref": "releng/14.1", "ref": "releng/14.1",
"refType": "branch", "refType": "branch",
"rev": "f389e68ca980b7e053a34d9eddde89b4c2a1ee6c", "rev": "f389e68ca980b7e053a34d9eddde89b4c2a1ee6c",
"supported": true, "supported": false,
"version": { "version": {
"branch": "RELEASE-p8", "branch": "RELEASE-p8",
"major": 14, "major": 14,
@ -322,7 +322,7 @@
} }
}, },
"releng/14.2": { "releng/14.2": {
"hash": "sha256-XP8BFnXvziaC9wOJj8q31UZXFqCUE7WQ5FdJHEZWGbg=", "hash": "sha256-iSeTif/cEkqhIQ5hSNF+Rx7dORU13Bc0Dk8Zv4TYWtA=",
"ref": "releng/14.2", "ref": "releng/14.2",
"refType": "branch", "refType": "branch",
"rev": "ac2cbb46b5f1efa7f7b5d4eb15631337329ec5b2", "rev": "ac2cbb46b5f1efa7f7b5d4eb15631337329ec5b2",
@ -340,10 +340,10 @@
} }
}, },
"stable/13": { "stable/13": {
"hash": "sha256-J9SJKeR6Den3Sep2o4r0cqIDd2V5gbY0Ow9eP69Ny0o=", "hash": "sha256-rex3CUXyyNDz/TouNqlzZFo6DAwSOKXTdC5HaRQoKKc=",
"ref": "stable/13", "ref": "stable/13",
"refType": "branch", "refType": "branch",
"rev": "a8431b47adae8f8b731206dc38d82b2245ad245e", "rev": "64f5a71c1cb79c09e50a37ddbe958224bb64add4",
"supported": true, "supported": true,
"version": { "version": {
"branch": "STABLE", "branch": "STABLE",
@ -357,10 +357,10 @@
} }
}, },
"stable/14": { "stable/14": {
"hash": "sha256-tleB6J5Cg1SIN2LCfvV3Cfp4Lxx65UHmiILpin6UYGY=", "hash": "sha256-s+pj8LttAnKNNohUsJNOJRFmLg6x06tlkJn3xQnFQiY=",
"ref": "stable/14", "ref": "stable/14",
"refType": "branch", "refType": "branch",
"rev": "6e510d8fbaf8d91da235fe28250cd48124edda9f", "rev": "e6a470ffcbd708cf404472bb871c2cb76eaa7b39",
"supported": true, "supported": true,
"version": { "version": {
"branch": "STABLE", "branch": "STABLE",