mirror of
https://github.com/NixOS/nixpkgs.git
synced 2025-06-09 19:13:26 +03:00
freebsd.source: Use hashes from git archive
An in-tree dependency of FreeBSD uses an `export-ignore` directive to hide certain files from the git export. Unfortunately, that means that `fetchFromGitHub` has different outputs when `forceFetchGit = true` and `forceFetchGit = false`. We cannot download source with git during native FreeBSD bootstrap due to a circular dependency, so change the update script to output archives matching GitHub.
This commit is contained in:
Artemis Tosini
parent
1b62e27af6
commit
e6b11fe5ef
3 changed files with 42 additions and 51 deletions
|
|
@ -1,16 +1,12 @@
|
|||
{ fetchFromGitHub, sourceData }:
|
||||
|
||||
# Using fetchFromGitHub from their mirror because it's a lot faster than their git server
|
||||
# If you want you could fetchgit from "https://git.FreeBSD.org/src.git" instead.
|
||||
# Using fetchFromGitHub from their mirror because we cannot use git during bootstrap
|
||||
# If you want you could fetchurl from "https://cgit.FreeBSD.org/src" instead.
|
||||
# The update script still pulls directly from git.freebsd.org
|
||||
# Note that hashes with `forceFetchGit = true` may be different due to `export-ignore`
|
||||
# directives in gitattributes files.
|
||||
fetchFromGitHub {
|
||||
owner = "freebsd";
|
||||
repo = "freebsd-src";
|
||||
inherit (sourceData) rev hash;
|
||||
|
||||
# The GitHub export excludes some files in the git source
|
||||
# that were marked `export-ignore`.
|
||||
# A normal git checkout will keep those files,
|
||||
# matching the update script
|
||||
forceFetchGit = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,12 +11,13 @@ import pandas
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import typing
|
||||
import urllib.request
|
||||
|
||||
_QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$')
|
||||
_RELEASE_PATCH_PATTERN = re.compile('^RELEASE-p([0-9]+)$')
|
||||
_RELEASE_PATCH_PATTERN = re.compile("^RELEASE-p([0-9]+)$")
|
||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
MIN_VERSION = packaging.version.Version("13.0.0")
|
||||
MAIN_BRANCH = "main"
|
||||
|
|
@ -32,7 +33,7 @@ BRANCH_PATTERN = re.compile(
|
|||
|
||||
def request_supported_refs() -> list[str]:
|
||||
# Looks pretty shady but I think this should work with every version of the page in the last 20 years
|
||||
r = re.compile("^h\d$", re.IGNORECASE)
|
||||
r = re.compile(r"^h\d$", re.IGNORECASE)
|
||||
soup = bs4.BeautifulSoup(
|
||||
urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml"
|
||||
)
|
||||
|
|
@ -45,11 +46,11 @@ def request_supported_refs() -> list[str]:
|
|||
return list(df["Branch"])
|
||||
|
||||
|
||||
def query_version(repo: git.Repo) -> dict[str, typing.Any]:
|
||||
def query_version(work_dir: str) -> dict[str, typing.Any]:
|
||||
# This only works on FreeBSD 13 and later
|
||||
text = (
|
||||
subprocess.check_output(
|
||||
["bash", os.path.join(repo.working_dir, "sys", "conf", "newvers.sh"), "-v"]
|
||||
["bash", os.path.join(work_dir, "sys", "conf", "newvers.sh"), "-v"]
|
||||
)
|
||||
.decode("utf-8")
|
||||
.strip()
|
||||
|
|
@ -86,17 +87,22 @@ def handle_commit(
|
|||
print(f"{ref_name}: revision still {rev.hexsha}, skipping")
|
||||
return old_versions[ref_name]
|
||||
|
||||
repo.git.checkout(rev)
|
||||
print(f"{ref_name}: checked out {rev.hexsha}")
|
||||
tar_content = io.BytesIO()
|
||||
repo.archive(tar_content, rev, format="tar")
|
||||
tar_content.seek(0)
|
||||
|
||||
with tempfile.TemporaryDirectory(dir="/tmp") as work_dir:
|
||||
file = tarfile.TarFile(fileobj=tar_content)
|
||||
file.extractall(path=work_dir, filter="data")
|
||||
|
||||
full_hash = (
|
||||
subprocess.check_output(["nix", "hash", "path", "--sri", repo.working_dir])
|
||||
subprocess.check_output(["nix", "hash", "path", "--sri", work_dir])
|
||||
.decode("utf-8")
|
||||
.strip()
|
||||
)
|
||||
print(f"{ref_name}: hash is {full_hash}")
|
||||
|
||||
version = query_version(repo)
|
||||
version = query_version(work_dir)
|
||||
print(f"{ref_name}: version is {version['version']}")
|
||||
|
||||
return {
|
||||
|
|
@ -115,30 +121,19 @@ def main() -> None:
|
|||
print(f"Selected temporary directory {temp_dir.name}")
|
||||
|
||||
if len(sys.argv) >= 2:
|
||||
orig_repo = git.Repo(sys.argv[1])
|
||||
print(f"Fetching updates on {orig_repo.git_dir}")
|
||||
orig_repo.remote("origin").fetch()
|
||||
repo = git.Repo(sys.argv[1])
|
||||
print(f"Fetching updates on {repo.git_dir}")
|
||||
repo.remote("origin").fetch()
|
||||
else:
|
||||
print("Cloning source repo")
|
||||
orig_repo = git.Repo.clone_from(
|
||||
"https://git.FreeBSD.org/src.git", to_path=os.path.join(temp_dir.name, "orig")
|
||||
repo = git.Repo.clone_from(
|
||||
"https://git.FreeBSD.org/src.git", to_path=temp_dir.name
|
||||
)
|
||||
|
||||
supported_refs = request_supported_refs()
|
||||
print(f"Supported refs are: {' '.join(supported_refs)}")
|
||||
|
||||
print("Doing git crimes, do not run `git worktree prune` until after script finishes!")
|
||||
workdir = os.path.join(temp_dir.name, "work")
|
||||
git.cmd.Git(orig_repo.git_dir).worktree("add", "--orphan", workdir)
|
||||
|
||||
# Have to create object before removing .git otherwise it will complain
|
||||
repo = git.Repo(workdir)
|
||||
repo.git.set_persistent_git_options(git_dir=repo.git_dir)
|
||||
# Remove so that nix hash doesn't see the file
|
||||
os.remove(os.path.join(workdir, ".git"))
|
||||
|
||||
print(f"Working in directory {repo.working_dir} with git directory {repo.git_dir}")
|
||||
|
||||
print(f"git directory {repo.git_dir}")
|
||||
|
||||
try:
|
||||
with open(os.path.join(BASE_DIR, "versions.json"), "r") as f:
|
||||
|
|
@ -191,10 +186,10 @@ def main() -> None:
|
|||
)
|
||||
versions[fullname] = result
|
||||
|
||||
|
||||
with open(os.path.join(BASE_DIR, "versions.json"), "w") as out:
|
||||
json.dump(versions, out, sort_keys=True, indent=2)
|
||||
out.write("\n")
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
{
|
||||
"main": {
|
||||
"hash": "sha256-xsgY5Ex/B5ngOTa5OZRauSaSYvET5lWI7veJRrSq1oY=",
|
||||
"hash": "sha256-f69U2FX+3GQimdUJunWApFseeKJhzjgNw4h26ZBPiT0=",
|
||||
"ref": "main",
|
||||
"refType": "branch",
|
||||
"rev": "c5773d366ecc5271b9bd6e5506c00fb3520f19ae",
|
||||
"rev": "9357c694e8dca627c25b15529e8435b2ab3dd48b",
|
||||
"supported": false,
|
||||
"version": {
|
||||
"branch": "CURRENT",
|
||||
"major": 15,
|
||||
"minor": 0,
|
||||
"reldate": "1500035",
|
||||
"reldate": "1500036",
|
||||
"release": "15.0-CURRENT",
|
||||
"revision": "15.0",
|
||||
"type": "FreeBSD",
|
||||
|
|
@ -161,7 +161,7 @@
|
|||
}
|
||||
},
|
||||
"release/14.2.0": {
|
||||
"hash": "sha256-qZkeuUZbuPOvXZBgP5x6Hii1YN7XdDJzwZeYacIR5BI=",
|
||||
"hash": "sha256-q5/0pKRlmzZUN+VMwUImkve+NTT1+O68YlwgXp2/aQM=",
|
||||
"ref": "release/14.2.0",
|
||||
"refType": "tag",
|
||||
"rev": "c8918d6c7412fce87922e9bd7e4f5c7d7ca96eb7",
|
||||
|
|
@ -308,7 +308,7 @@
|
|||
"ref": "releng/14.1",
|
||||
"refType": "branch",
|
||||
"rev": "f389e68ca980b7e053a34d9eddde89b4c2a1ee6c",
|
||||
"supported": true,
|
||||
"supported": false,
|
||||
"version": {
|
||||
"branch": "RELEASE-p8",
|
||||
"major": 14,
|
||||
|
|
@ -322,7 +322,7 @@
|
|||
}
|
||||
},
|
||||
"releng/14.2": {
|
||||
"hash": "sha256-XP8BFnXvziaC9wOJj8q31UZXFqCUE7WQ5FdJHEZWGbg=",
|
||||
"hash": "sha256-iSeTif/cEkqhIQ5hSNF+Rx7dORU13Bc0Dk8Zv4TYWtA=",
|
||||
"ref": "releng/14.2",
|
||||
"refType": "branch",
|
||||
"rev": "ac2cbb46b5f1efa7f7b5d4eb15631337329ec5b2",
|
||||
|
|
@ -340,10 +340,10 @@
|
|||
}
|
||||
},
|
||||
"stable/13": {
|
||||
"hash": "sha256-J9SJKeR6Den3Sep2o4r0cqIDd2V5gbY0Ow9eP69Ny0o=",
|
||||
"hash": "sha256-rex3CUXyyNDz/TouNqlzZFo6DAwSOKXTdC5HaRQoKKc=",
|
||||
"ref": "stable/13",
|
||||
"refType": "branch",
|
||||
"rev": "a8431b47adae8f8b731206dc38d82b2245ad245e",
|
||||
"rev": "64f5a71c1cb79c09e50a37ddbe958224bb64add4",
|
||||
"supported": true,
|
||||
"version": {
|
||||
"branch": "STABLE",
|
||||
|
|
@ -357,10 +357,10 @@
|
|||
}
|
||||
},
|
||||
"stable/14": {
|
||||
"hash": "sha256-tleB6J5Cg1SIN2LCfvV3Cfp4Lxx65UHmiILpin6UYGY=",
|
||||
"hash": "sha256-s+pj8LttAnKNNohUsJNOJRFmLg6x06tlkJn3xQnFQiY=",
|
||||
"ref": "stable/14",
|
||||
"refType": "branch",
|
||||
"rev": "6e510d8fbaf8d91da235fe28250cd48124edda9f",
|
||||
"rev": "e6a470ffcbd708cf404472bb871c2cb76eaa7b39",
|
||||
"supported": true,
|
||||
"version": {
|
||||
"branch": "STABLE",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue