214 lines
8.9 KiB
Python
214 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Create deterministic source releases; verify them against an external receipt pin."""
|
|
|
|
import argparse
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import platform
|
|
import re
|
|
import subprocess
|
|
import tarfile
|
|
import tomllib
|
|
import zlib
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
ARCHIVE = "source.tar.gz"
|
|
RECEIPT = "RELEASE.json"
|
|
SCHEMA = "argand.site-source-release/v1"
|
|
MAX_BYTES = 32 * 1024 * 1024
|
|
MAX_FILES = 2000
|
|
MAX_TAR_BYTES = MAX_BYTES + MAX_FILES * 1024 + 10240
|
|
|
|
|
|
def digest(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def unique_object(pairs):
|
|
result = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError("duplicate JSON key")
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def read_bounded(path, maximum):
|
|
if path.is_symlink() or not path.is_file():
|
|
raise ValueError(f"expected a regular file: {path.name}")
|
|
with path.open("rb") as stream:
|
|
data = stream.read(maximum + 1)
|
|
if len(data) > maximum:
|
|
raise ValueError("release exceeds size limit")
|
|
return data
|
|
|
|
|
|
def safe_path(name):
|
|
path = PurePosixPath(name)
|
|
if not name or path.is_absolute() or str(path) != name:
|
|
raise ValueError("noncanonical archive path")
|
|
if any(part in ("..", ".git", "target", "build", "dist", "data", "cache",
|
|
"__pycache__") for part in path.parts):
|
|
raise ValueError("excluded archive path")
|
|
if any(ord(char) < 32 for char in name) or "\\" in name:
|
|
raise ValueError("unsafe archive path")
|
|
allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml",
|
|
".service", ".timer"}
|
|
if path.suffix not in allowed and path.name not in (
|
|
".gitignore", "Dockerfile") and name not in (
|
|
"LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore",
|
|
"crates/argand-site-registry/examples/observer.env",
|
|
"crates/argand-site-registry/tests/fixtures/v03-contract.json"):
|
|
raise ValueError(f"file is outside the source release allowlist: {name}")
|
|
|
|
|
|
def git(root, *args):
|
|
return subprocess.check_output(["git", "-C", str(root), *args])
|
|
|
|
|
|
def write_new(path, data):
|
|
with path.open("xb") as stream:
|
|
stream.write(data)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
|
|
|
|
def sync_directory(path):
|
|
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def create(root, output):
|
|
root = root.resolve()
|
|
if git(root, "rev-parse", "--show-toplevel").decode().strip() != str(root):
|
|
raise ValueError("run from the source repository root")
|
|
if git(root, "status", "--porcelain", "--untracked-files=all"):
|
|
raise ValueError("source release requires a clean committed tree")
|
|
commit = git(root, "rev-parse", "HEAD").decode().strip()
|
|
tree = git(root, "rev-parse", f"{commit}^{{tree}}").decode().strip()
|
|
manifest = git(root, "show", f"{commit}:Cargo.toml")
|
|
version = tomllib.loads(manifest.decode())["workspace"]["package"]["version"]
|
|
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?", version):
|
|
raise ValueError("unsupported release version")
|
|
prefix = f"argand-site-registry-{version}/"
|
|
files = []
|
|
buffer = io.BytesIO()
|
|
total = 0
|
|
with gzip.GzipFile(fileobj=buffer, mode="wb", filename="", mtime=0, compresslevel=9) as compressed:
|
|
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.USTAR_FORMAT) as archive:
|
|
for entry in git(root, "ls-tree", "-rz", commit).split(b"\0"):
|
|
if not entry:
|
|
continue
|
|
header, raw_path = entry.split(b"\t", 1)
|
|
mode, kind, blob = header.decode().split()
|
|
name = raw_path.decode()
|
|
safe_path(name)
|
|
if kind != "blob" or mode not in ("100644", "100755"):
|
|
raise ValueError("source releases cannot contain links or submodules")
|
|
size = int(git(root, "cat-file", "-s", blob))
|
|
total += size
|
|
if total > MAX_BYTES or len(files) >= MAX_FILES:
|
|
raise ValueError("source tree exceeds release limits")
|
|
data = git(root, "cat-file", "blob", blob)
|
|
info = tarfile.TarInfo(prefix + name)
|
|
info.size, info.mode = size, int(mode[-3:], 8)
|
|
archive.addfile(info, io.BytesIO(data))
|
|
files.append({"path": name, "bytes": size, "mode": info.mode,
|
|
"sha256": digest(data)})
|
|
required = {"LICENSE", "Cargo.lock", "Cargo.toml", "UPSTREAM.json",
|
|
"crates/argand-site-registry/LICENSE_SOURCES.md"}
|
|
if not required.issubset({item["path"] for item in files}):
|
|
raise ValueError("source release is missing required license or build inputs")
|
|
payload = buffer.getvalue()
|
|
receipt = {"schema": SCHEMA, "commit": commit, "tree": tree, "version": version,
|
|
"archive": ARCHIVE, "prefix": prefix, "bytes": len(payload),
|
|
"sha256": digest(payload), "files": files,
|
|
"packager": {"python": platform.python_version(),
|
|
"zlib": zlib.ZLIB_RUNTIME_VERSION}}
|
|
encoded = (json.dumps(receipt, indent=2, sort_keys=True) + "\n").encode()
|
|
output.mkdir(mode=0o700, parents=False) # Never overwrite an earlier release.
|
|
write_new(output / ARCHIVE, payload)
|
|
write_new(output / RECEIPT, encoded) # Completion receipt is written last.
|
|
sync_directory(output)
|
|
sync_directory(output.parent)
|
|
return digest(encoded)
|
|
|
|
|
|
def verify(output, pin):
|
|
if not re.fullmatch(r"[0-9a-f]{64}", pin):
|
|
raise ValueError("supply a full externally trusted receipt SHA-256")
|
|
encoded = read_bounded(output / RECEIPT, 1024 * 1024)
|
|
if digest(encoded) != pin:
|
|
raise ValueError("source receipt pin mismatch")
|
|
receipt = json.loads(encoded, object_pairs_hook=unique_object)
|
|
if receipt["schema"] != SCHEMA or receipt["archive"] != ARCHIVE:
|
|
raise ValueError("unsupported source release")
|
|
if not re.fullmatch(r"argand-site-registry-[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?/",
|
|
receipt["prefix"]):
|
|
raise ValueError("unsafe archive prefix")
|
|
payload = read_bounded(output / ARCHIVE, MAX_BYTES)
|
|
if len(payload) != receipt["bytes"] or digest(payload) != receipt["sha256"]:
|
|
raise ValueError("source archive digest mismatch")
|
|
files = receipt["files"]
|
|
expected = {item["path"]: item for item in files}
|
|
if len(expected) != len(files) or not 1 <= len(files) <= MAX_FILES:
|
|
raise ValueError("duplicate or excessive source entries")
|
|
for name in expected:
|
|
safe_path(name)
|
|
seen, total = set(), 0
|
|
# Bound expansion before tarfile parses potentially large extended headers.
|
|
with gzip.GzipFile(fileobj=io.BytesIO(payload)) as compressed:
|
|
expanded = compressed.read(MAX_TAR_BYTES + 1)
|
|
if len(expanded) > MAX_TAR_BYTES:
|
|
raise ValueError("expanded archive exceeds limit")
|
|
with tarfile.open(fileobj=io.BytesIO(expanded), mode="r:") as archive:
|
|
for member in archive:
|
|
if not member.isfile() or not member.name.startswith(receipt["prefix"]):
|
|
raise ValueError("unsafe archive member")
|
|
name = member.name.removeprefix(receipt["prefix"])
|
|
safe_path(name)
|
|
if name in seen or name not in expected:
|
|
raise ValueError("duplicate or unexpected archive member")
|
|
item = expected[name]
|
|
total += member.size
|
|
if total > MAX_BYTES or member.size != item["bytes"]:
|
|
raise ValueError("archive size mismatch")
|
|
if member.mode not in (0o644, 0o755) or member.mode != item["mode"]:
|
|
raise ValueError("archive mode mismatch")
|
|
stream = archive.extractfile(member)
|
|
if stream is None or digest(stream.read(MAX_BYTES + 1)) != item["sha256"]:
|
|
raise ValueError("source file digest mismatch")
|
|
seen.add(name)
|
|
if seen != expected.keys():
|
|
raise ValueError("archive omits source files")
|
|
return receipt
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
make = commands.add_parser("create")
|
|
make.add_argument("--output", type=Path, required=True)
|
|
check = commands.add_parser("verify")
|
|
check.add_argument("--release", type=Path, required=True)
|
|
check.add_argument("--pin", required=True)
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.command == "create":
|
|
result = {"release": str(args.output), "pin": create(Path.cwd(), args.output)}
|
|
else:
|
|
result = verify(args.release, args.pin)
|
|
print(json.dumps(result, sort_keys=True))
|
|
except (OSError, ValueError, KeyError, TypeError, tarfile.TarError,
|
|
subprocess.CalledProcessError) as error:
|
|
parser.exit(1, f"source release refused: {error}\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|