feat: establish standalone Argand Site Registry
This commit is contained in:
commit
2a0fe1714b
SSH key fingerprint:
SHA256:2te+ycJIQON/Wo/dH6+ZkFSQ4HnHWpetV2azx9E65dQ
60 changed files with 10494 additions and 0 deletions
211
scripts/source_release.py
Normal file
211
scripts/source_release.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Create deterministic source releases; verify them against an external receipt pin."""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tomllib
|
||||
import zlib
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
ARCHIVE = "source.tar.gz"
|
||||
RECEIPT = "RELEASE.json"
|
||||
SCHEMA = "argand.site-source-release/v1"
|
||||
MAX_BYTES = 32 * 1024 * 1024
|
||||
MAX_FILES = 2000
|
||||
MAX_TAR_BYTES = MAX_BYTES + MAX_FILES * 1024 + 10240
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError("duplicate JSON key")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def read_bounded(path, maximum):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError(f"expected a regular file: {path.name}")
|
||||
with path.open("rb") as stream:
|
||||
data = stream.read(maximum + 1)
|
||||
if len(data) > maximum:
|
||||
raise ValueError("release exceeds size limit")
|
||||
return data
|
||||
|
||||
|
||||
def safe_path(name):
|
||||
path = PurePosixPath(name)
|
||||
if not name or path.is_absolute() or str(path) != name:
|
||||
raise ValueError("noncanonical archive path")
|
||||
if any(part in ("..", ".git", "target", "build", "dist", "data", "cache",
|
||||
"__pycache__") for part in path.parts):
|
||||
raise ValueError("excluded archive path")
|
||||
if any(ord(char) < 32 for char in name) or "\\" in name:
|
||||
raise ValueError("unsafe archive path")
|
||||
allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml",
|
||||
".service", ".timer"}
|
||||
if path.suffix not in allowed and path.name != ".gitignore" and name not in (
|
||||
"LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore"):
|
||||
raise ValueError(f"file is outside the source release allowlist: {name}")
|
||||
|
||||
|
||||
def git(root, *args):
|
||||
return subprocess.check_output(["git", "-C", str(root), *args])
|
||||
|
||||
|
||||
def write_new(path, data):
|
||||
with path.open("xb") as stream:
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def sync_directory(path):
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def create(root, output):
|
||||
root = root.resolve()
|
||||
if git(root, "rev-parse", "--show-toplevel").decode().strip() != str(root):
|
||||
raise ValueError("run from the source repository root")
|
||||
if git(root, "status", "--porcelain", "--untracked-files=all"):
|
||||
raise ValueError("source release requires a clean committed tree")
|
||||
commit = git(root, "rev-parse", "HEAD").decode().strip()
|
||||
tree = git(root, "rev-parse", f"{commit}^{{tree}}").decode().strip()
|
||||
manifest = git(root, "show", f"{commit}:Cargo.toml")
|
||||
version = tomllib.loads(manifest.decode())["workspace"]["package"]["version"]
|
||||
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?", version):
|
||||
raise ValueError("unsupported release version")
|
||||
prefix = f"argand-site-registry-{version}/"
|
||||
files = []
|
||||
buffer = io.BytesIO()
|
||||
total = 0
|
||||
with gzip.GzipFile(fileobj=buffer, mode="wb", filename="", mtime=0, compresslevel=9) as compressed:
|
||||
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.USTAR_FORMAT) as archive:
|
||||
for entry in git(root, "ls-tree", "-rz", commit).split(b"\0"):
|
||||
if not entry:
|
||||
continue
|
||||
header, raw_path = entry.split(b"\t", 1)
|
||||
mode, kind, blob = header.decode().split()
|
||||
name = raw_path.decode()
|
||||
safe_path(name)
|
||||
if kind != "blob" or mode not in ("100644", "100755"):
|
||||
raise ValueError("source releases cannot contain links or submodules")
|
||||
size = int(git(root, "cat-file", "-s", blob))
|
||||
total += size
|
||||
if total > MAX_BYTES or len(files) >= MAX_FILES:
|
||||
raise ValueError("source tree exceeds release limits")
|
||||
data = git(root, "cat-file", "blob", blob)
|
||||
info = tarfile.TarInfo(prefix + name)
|
||||
info.size, info.mode = size, int(mode[-3:], 8)
|
||||
archive.addfile(info, io.BytesIO(data))
|
||||
files.append({"path": name, "bytes": size, "mode": info.mode,
|
||||
"sha256": digest(data)})
|
||||
required = {"LICENSE", "Cargo.lock", "Cargo.toml", "UPSTREAM.json",
|
||||
"crates/argand-site-registry/LICENSE_SOURCES.md"}
|
||||
if not required.issubset({item["path"] for item in files}):
|
||||
raise ValueError("source release is missing required license or build inputs")
|
||||
payload = buffer.getvalue()
|
||||
receipt = {"schema": SCHEMA, "commit": commit, "tree": tree, "version": version,
|
||||
"archive": ARCHIVE, "prefix": prefix, "bytes": len(payload),
|
||||
"sha256": digest(payload), "files": files,
|
||||
"packager": {"python": platform.python_version(),
|
||||
"zlib": zlib.ZLIB_RUNTIME_VERSION}}
|
||||
encoded = (json.dumps(receipt, indent=2, sort_keys=True) + "\n").encode()
|
||||
output.mkdir(mode=0o700, parents=False) # Never overwrite an earlier release.
|
||||
write_new(output / ARCHIVE, payload)
|
||||
write_new(output / RECEIPT, encoded) # Completion receipt is written last.
|
||||
sync_directory(output)
|
||||
sync_directory(output.parent)
|
||||
return digest(encoded)
|
||||
|
||||
|
||||
def verify(output, pin):
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", pin):
|
||||
raise ValueError("supply a full externally trusted receipt SHA-256")
|
||||
encoded = read_bounded(output / RECEIPT, 1024 * 1024)
|
||||
if digest(encoded) != pin:
|
||||
raise ValueError("source receipt pin mismatch")
|
||||
receipt = json.loads(encoded, object_pairs_hook=unique_object)
|
||||
if receipt["schema"] != SCHEMA or receipt["archive"] != ARCHIVE:
|
||||
raise ValueError("unsupported source release")
|
||||
if not re.fullmatch(r"argand-site-registry-[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?/",
|
||||
receipt["prefix"]):
|
||||
raise ValueError("unsafe archive prefix")
|
||||
payload = read_bounded(output / ARCHIVE, MAX_BYTES)
|
||||
if len(payload) != receipt["bytes"] or digest(payload) != receipt["sha256"]:
|
||||
raise ValueError("source archive digest mismatch")
|
||||
files = receipt["files"]
|
||||
expected = {item["path"]: item for item in files}
|
||||
if len(expected) != len(files) or not 1 <= len(files) <= MAX_FILES:
|
||||
raise ValueError("duplicate or excessive source entries")
|
||||
for name in expected:
|
||||
safe_path(name)
|
||||
seen, total = set(), 0
|
||||
# Bound expansion before tarfile parses potentially large extended headers.
|
||||
with gzip.GzipFile(fileobj=io.BytesIO(payload)) as compressed:
|
||||
expanded = compressed.read(MAX_TAR_BYTES + 1)
|
||||
if len(expanded) > MAX_TAR_BYTES:
|
||||
raise ValueError("expanded archive exceeds limit")
|
||||
with tarfile.open(fileobj=io.BytesIO(expanded), mode="r:") as archive:
|
||||
for member in archive:
|
||||
if not member.isfile() or not member.name.startswith(receipt["prefix"]):
|
||||
raise ValueError("unsafe archive member")
|
||||
name = member.name.removeprefix(receipt["prefix"])
|
||||
safe_path(name)
|
||||
if name in seen or name not in expected:
|
||||
raise ValueError("duplicate or unexpected archive member")
|
||||
item = expected[name]
|
||||
total += member.size
|
||||
if total > MAX_BYTES or member.size != item["bytes"]:
|
||||
raise ValueError("archive size mismatch")
|
||||
if member.mode not in (0o644, 0o755) or member.mode != item["mode"]:
|
||||
raise ValueError("archive mode mismatch")
|
||||
stream = archive.extractfile(member)
|
||||
if stream is None or digest(stream.read(MAX_BYTES + 1)) != item["sha256"]:
|
||||
raise ValueError("source file digest mismatch")
|
||||
seen.add(name)
|
||||
if seen != expected.keys():
|
||||
raise ValueError("archive omits source files")
|
||||
return receipt
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
make = commands.add_parser("create")
|
||||
make.add_argument("--output", type=Path, required=True)
|
||||
check = commands.add_parser("verify")
|
||||
check.add_argument("--release", type=Path, required=True)
|
||||
check.add_argument("--pin", required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == "create":
|
||||
result = {"release": str(args.output), "pin": create(Path.cwd(), args.output)}
|
||||
else:
|
||||
result = verify(args.release, args.pin)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
except (OSError, ValueError, KeyError, TypeError, tarfile.TarError,
|
||||
subprocess.CalledProcessError) as error:
|
||||
parser.exit(1, f"source release refused: {error}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue