#!/usr/bin/env python3 """Create deterministic source releases; verify them against an external receipt pin.""" import argparse import gzip import hashlib import io import json import os import platform import re import subprocess import tarfile import tomllib import zlib from pathlib import Path, PurePosixPath ARCHIVE = "source.tar.gz" RECEIPT = "RELEASE.json" SCHEMA = "argand.site-source-release/v1" MAX_BYTES = 32 * 1024 * 1024 MAX_FILES = 2000 MAX_TAR_BYTES = MAX_BYTES + MAX_FILES * 1024 + 10240 def digest(data): return hashlib.sha256(data).hexdigest() def unique_object(pairs): result = {} for key, value in pairs: if key in result: raise ValueError("duplicate JSON key") result[key] = value return result def read_bounded(path, maximum): if path.is_symlink() or not path.is_file(): raise ValueError(f"expected a regular file: {path.name}") with path.open("rb") as stream: data = stream.read(maximum + 1) if len(data) > maximum: raise ValueError("release exceeds size limit") return data def safe_path(name): path = PurePosixPath(name) if not name or path.is_absolute() or str(path) != name: raise ValueError("noncanonical archive path") if any(part in ("..", ".git", "target", "build", "dist", "data", "cache", "__pycache__") for part in path.parts): raise ValueError("excluded archive path") if any(ord(char) < 32 for char in name) or "\\" in name: raise ValueError("unsafe archive path") allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml", ".service", ".timer"} if path.suffix not in allowed and path.name not in (".gitignore", "Dockerfile") and name not in ( "LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore"): raise ValueError(f"file is outside the source release allowlist: {name}") def git(root, *args): return subprocess.check_output(["git", "-C", str(root), *args]) def write_new(path, data): with path.open("xb") as stream: stream.write(data) stream.flush() os.fsync(stream.fileno()) def sync_directory(path): descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(descriptor) finally: os.close(descriptor) def create(root, output): root = root.resolve() if git(root, "rev-parse", "--show-toplevel").decode().strip() != str(root): raise ValueError("run from the source repository root") if git(root, "status", "--porcelain", "--untracked-files=all"): raise ValueError("source release requires a clean committed tree") commit = git(root, "rev-parse", "HEAD").decode().strip() tree = git(root, "rev-parse", f"{commit}^{{tree}}").decode().strip() manifest = git(root, "show", f"{commit}:Cargo.toml") version = tomllib.loads(manifest.decode())["workspace"]["package"]["version"] if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?", version): raise ValueError("unsupported release version") prefix = f"argand-site-registry-{version}/" files = [] buffer = io.BytesIO() total = 0 with gzip.GzipFile(fileobj=buffer, mode="wb", filename="", mtime=0, compresslevel=9) as compressed: with tarfile.open(fileobj=compressed, mode="w", format=tarfile.USTAR_FORMAT) as archive: for entry in git(root, "ls-tree", "-rz", commit).split(b"\0"): if not entry: continue header, raw_path = entry.split(b"\t", 1) mode, kind, blob = header.decode().split() name = raw_path.decode() safe_path(name) if kind != "blob" or mode not in ("100644", "100755"): raise ValueError("source releases cannot contain links or submodules") size = int(git(root, "cat-file", "-s", blob)) total += size if total > MAX_BYTES or len(files) >= MAX_FILES: raise ValueError("source tree exceeds release limits") data = git(root, "cat-file", "blob", blob) info = tarfile.TarInfo(prefix + name) info.size, info.mode = size, int(mode[-3:], 8) archive.addfile(info, io.BytesIO(data)) files.append({"path": name, "bytes": size, "mode": info.mode, "sha256": digest(data)}) required = {"LICENSE", "Cargo.lock", "Cargo.toml", "UPSTREAM.json", "crates/argand-site-registry/LICENSE_SOURCES.md"} if not required.issubset({item["path"] for item in files}): raise ValueError("source release is missing required license or build inputs") payload = buffer.getvalue() receipt = {"schema": SCHEMA, "commit": commit, "tree": tree, "version": version, "archive": ARCHIVE, "prefix": prefix, "bytes": len(payload), "sha256": digest(payload), "files": files, "packager": {"python": platform.python_version(), "zlib": zlib.ZLIB_RUNTIME_VERSION}} encoded = (json.dumps(receipt, indent=2, sort_keys=True) + "\n").encode() output.mkdir(mode=0o700, parents=False) # Never overwrite an earlier release. write_new(output / ARCHIVE, payload) write_new(output / RECEIPT, encoded) # Completion receipt is written last. sync_directory(output) sync_directory(output.parent) return digest(encoded) def verify(output, pin): if not re.fullmatch(r"[0-9a-f]{64}", pin): raise ValueError("supply a full externally trusted receipt SHA-256") encoded = read_bounded(output / RECEIPT, 1024 * 1024) if digest(encoded) != pin: raise ValueError("source receipt pin mismatch") receipt = json.loads(encoded, object_pairs_hook=unique_object) if receipt["schema"] != SCHEMA or receipt["archive"] != ARCHIVE: raise ValueError("unsupported source release") if not re.fullmatch(r"argand-site-registry-[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?/", receipt["prefix"]): raise ValueError("unsafe archive prefix") payload = read_bounded(output / ARCHIVE, MAX_BYTES) if len(payload) != receipt["bytes"] or digest(payload) != receipt["sha256"]: raise ValueError("source archive digest mismatch") files = receipt["files"] expected = {item["path"]: item for item in files} if len(expected) != len(files) or not 1 <= len(files) <= MAX_FILES: raise ValueError("duplicate or excessive source entries") for name in expected: safe_path(name) seen, total = set(), 0 # Bound expansion before tarfile parses potentially large extended headers. with gzip.GzipFile(fileobj=io.BytesIO(payload)) as compressed: expanded = compressed.read(MAX_TAR_BYTES + 1) if len(expanded) > MAX_TAR_BYTES: raise ValueError("expanded archive exceeds limit") with tarfile.open(fileobj=io.BytesIO(expanded), mode="r:") as archive: for member in archive: if not member.isfile() or not member.name.startswith(receipt["prefix"]): raise ValueError("unsafe archive member") name = member.name.removeprefix(receipt["prefix"]) safe_path(name) if name in seen or name not in expected: raise ValueError("duplicate or unexpected archive member") item = expected[name] total += member.size if total > MAX_BYTES or member.size != item["bytes"]: raise ValueError("archive size mismatch") if member.mode not in (0o644, 0o755) or member.mode != item["mode"]: raise ValueError("archive mode mismatch") stream = archive.extractfile(member) if stream is None or digest(stream.read(MAX_BYTES + 1)) != item["sha256"]: raise ValueError("source file digest mismatch") seen.add(name) if seen != expected.keys(): raise ValueError("archive omits source files") return receipt def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) make = commands.add_parser("create") make.add_argument("--output", type=Path, required=True) check = commands.add_parser("verify") check.add_argument("--release", type=Path, required=True) check.add_argument("--pin", required=True) args = parser.parse_args() try: if args.command == "create": result = {"release": str(args.output), "pin": create(Path.cwd(), args.output)} else: result = verify(args.release, args.pin) print(json.dumps(result, sort_keys=True)) except (OSError, ValueError, KeyError, TypeError, tarfile.TarError, subprocess.CalledProcessError) as error: parser.exit(1, f"source release refused: {error}\n") if __name__ == "__main__": main()