feat: establish standalone Argand Site Registry
This commit is contained in:
commit
2a0fe1714b
SSH key fingerprint:
SHA256:2te+ycJIQON/Wo/dH6+ZkFSQ4HnHWpetV2azx9E65dQ
60 changed files with 10494 additions and 0 deletions
28
scripts/check.sh
Normal file
28
scripts/check.sh
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
#!/usr/bin/env bash
|
||||
# By Nic Weyand! Run from a checkout or an extracted source release.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||
export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-2}"
|
||||
export CARGO_TERM_COLOR=never
|
||||
registry_check_root="${ARGAND_REGISTRY_CHECK_OUTPUT:-$(mktemp -d -t site-registry-check.XXXXXXXX)}"
|
||||
mkdir -p "$registry_check_root"
|
||||
registry_check_root="$(cd "$registry_check_root" && pwd)"
|
||||
if [[ -e "$registry_check_root/fixture" ]]; then
|
||||
echo 'Choose a new check output directory; fixture evidence is never overwritten.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cargo fmt --all -- --check
|
||||
cargo check --workspace --all-targets --locked --offline
|
||||
cargo clippy --workspace --all-targets --locked --offline -- -D warnings
|
||||
ARGAND_REGISTRY_E2E_OUTPUT="$registry_check_root/fixture" \
|
||||
cargo test --workspace --locked --offline
|
||||
RUSTDOCFLAGS="${RUSTDOCFLAGS:-} -D warnings" \
|
||||
cargo doc --workspace --no-deps --locked --offline
|
||||
python3 -m unittest discover -s tests -v
|
||||
cargo build --workspace --bins --examples --locked --offline
|
||||
cargo metadata --no-deps --format-version 1 --locked --offline > "$registry_check_root/metadata.json"
|
||||
registry_target="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["target_directory"])' "$registry_check_root/metadata.json")"
|
||||
python3 scripts/check_consumers.py --binary "$registry_target/debug/argand-site-registry" \
|
||||
--rust-example "$registry_target/debug/examples/lookup" --fixture "$registry_check_root/fixture"
|
||||
echo "All checks passed. Synthetic fixture evidence: $registry_check_root"
|
||||
58
scripts/check_consumers.py
Normal file
58
scripts/check_consumers.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Compare shipped native CLI, Rust library and Python example on authored fixtures."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run(command):
|
||||
result = subprocess.run(command, check=True, capture_output=True, text=True, timeout=60)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--binary", type=Path, required=True)
|
||||
parser.add_argument("--rust-example", type=Path, required=True)
|
||||
parser.add_argument("--fixture", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
example = Path(__file__).resolve().parents[1] / "examples" / "lookup.py"
|
||||
generation = args.fixture / "candidate"
|
||||
# This local test generated the fixture itself. Production pins come from
|
||||
# an authenticated publisher, as explained in the consumer documentation.
|
||||
pin = hashlib.sha256((generation / "COMPLETE.json").read_bytes()).hexdigest()
|
||||
for query in ("FB", "Atlas", "Café Atlas", "$(echo unsafe); `id`"):
|
||||
shared = ["--generation", str(generation), "--pin", pin, "--query", query]
|
||||
cli = run([str(args.binary), "lookup", *shared, "--limit", "100"])
|
||||
rust = run([str(args.rust_example), *shared])
|
||||
python = run([sys.executable, str(example), "--binary", str(args.binary), *shared])
|
||||
if cli != rust or cli != python:
|
||||
raise ValueError(f"consumer output differs for {query!r}")
|
||||
if not cli["attribution"]:
|
||||
raise ValueError("consumer lost attribution")
|
||||
if query == "FB":
|
||||
candidate = cli["candidates"][0]
|
||||
if candidate["canonical_name"] != "Facebook" or not candidate["provenance"]:
|
||||
raise ValueError("name/provenance contract mismatch")
|
||||
if candidate["web_property"]["domain"]["registrable_domain"] != "facebook.com":
|
||||
raise ValueError("domain contract mismatch")
|
||||
if query == "Atlas":
|
||||
urls = {candidate["url"] for candidate in cli["candidates"]}
|
||||
if urls != {"https://atlas.example.com/", "https://atlas.example.co.uk/",
|
||||
"https://atlas.example.de/"}:
|
||||
raise ValueError("regional properties were lost")
|
||||
bad = ["--generation", str(generation), "--pin", "0" * 64, "--query", "FB"]
|
||||
for command in ([str(args.binary), "lookup", *bad], [str(args.rust_example), *bad],
|
||||
[sys.executable, str(example), "--binary", str(args.binary), *bad]):
|
||||
result = subprocess.run(command, capture_output=True, timeout=60)
|
||||
if result.returncode == 0:
|
||||
raise ValueError("consumer accepted an untrusted generation")
|
||||
print("Native CLI, Rust and Python consumer parity passed, including trust failures.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
211
scripts/source_release.py
Normal file
211
scripts/source_release.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Create deterministic source releases; verify them against an external receipt pin."""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tomllib
|
||||
import zlib
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
ARCHIVE = "source.tar.gz"
|
||||
RECEIPT = "RELEASE.json"
|
||||
SCHEMA = "argand.site-source-release/v1"
|
||||
MAX_BYTES = 32 * 1024 * 1024
|
||||
MAX_FILES = 2000
|
||||
MAX_TAR_BYTES = MAX_BYTES + MAX_FILES * 1024 + 10240
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError("duplicate JSON key")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def read_bounded(path, maximum):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError(f"expected a regular file: {path.name}")
|
||||
with path.open("rb") as stream:
|
||||
data = stream.read(maximum + 1)
|
||||
if len(data) > maximum:
|
||||
raise ValueError("release exceeds size limit")
|
||||
return data
|
||||
|
||||
|
||||
def safe_path(name):
|
||||
path = PurePosixPath(name)
|
||||
if not name or path.is_absolute() or str(path) != name:
|
||||
raise ValueError("noncanonical archive path")
|
||||
if any(part in ("..", ".git", "target", "build", "dist", "data", "cache",
|
||||
"__pycache__") for part in path.parts):
|
||||
raise ValueError("excluded archive path")
|
||||
if any(ord(char) < 32 for char in name) or "\\" in name:
|
||||
raise ValueError("unsafe archive path")
|
||||
allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml",
|
||||
".service", ".timer"}
|
||||
if path.suffix not in allowed and path.name != ".gitignore" and name not in (
|
||||
"LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore"):
|
||||
raise ValueError(f"file is outside the source release allowlist: {name}")
|
||||
|
||||
|
||||
def git(root, *args):
|
||||
return subprocess.check_output(["git", "-C", str(root), *args])
|
||||
|
||||
|
||||
def write_new(path, data):
|
||||
with path.open("xb") as stream:
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def sync_directory(path):
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def create(root, output):
|
||||
root = root.resolve()
|
||||
if git(root, "rev-parse", "--show-toplevel").decode().strip() != str(root):
|
||||
raise ValueError("run from the source repository root")
|
||||
if git(root, "status", "--porcelain", "--untracked-files=all"):
|
||||
raise ValueError("source release requires a clean committed tree")
|
||||
commit = git(root, "rev-parse", "HEAD").decode().strip()
|
||||
tree = git(root, "rev-parse", f"{commit}^{{tree}}").decode().strip()
|
||||
manifest = git(root, "show", f"{commit}:Cargo.toml")
|
||||
version = tomllib.loads(manifest.decode())["workspace"]["package"]["version"]
|
||||
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?", version):
|
||||
raise ValueError("unsupported release version")
|
||||
prefix = f"argand-site-registry-{version}/"
|
||||
files = []
|
||||
buffer = io.BytesIO()
|
||||
total = 0
|
||||
with gzip.GzipFile(fileobj=buffer, mode="wb", filename="", mtime=0, compresslevel=9) as compressed:
|
||||
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.USTAR_FORMAT) as archive:
|
||||
for entry in git(root, "ls-tree", "-rz", commit).split(b"\0"):
|
||||
if not entry:
|
||||
continue
|
||||
header, raw_path = entry.split(b"\t", 1)
|
||||
mode, kind, blob = header.decode().split()
|
||||
name = raw_path.decode()
|
||||
safe_path(name)
|
||||
if kind != "blob" or mode not in ("100644", "100755"):
|
||||
raise ValueError("source releases cannot contain links or submodules")
|
||||
size = int(git(root, "cat-file", "-s", blob))
|
||||
total += size
|
||||
if total > MAX_BYTES or len(files) >= MAX_FILES:
|
||||
raise ValueError("source tree exceeds release limits")
|
||||
data = git(root, "cat-file", "blob", blob)
|
||||
info = tarfile.TarInfo(prefix + name)
|
||||
info.size, info.mode = size, int(mode[-3:], 8)
|
||||
archive.addfile(info, io.BytesIO(data))
|
||||
files.append({"path": name, "bytes": size, "mode": info.mode,
|
||||
"sha256": digest(data)})
|
||||
required = {"LICENSE", "Cargo.lock", "Cargo.toml", "UPSTREAM.json",
|
||||
"crates/argand-site-registry/LICENSE_SOURCES.md"}
|
||||
if not required.issubset({item["path"] for item in files}):
|
||||
raise ValueError("source release is missing required license or build inputs")
|
||||
payload = buffer.getvalue()
|
||||
receipt = {"schema": SCHEMA, "commit": commit, "tree": tree, "version": version,
|
||||
"archive": ARCHIVE, "prefix": prefix, "bytes": len(payload),
|
||||
"sha256": digest(payload), "files": files,
|
||||
"packager": {"python": platform.python_version(),
|
||||
"zlib": zlib.ZLIB_RUNTIME_VERSION}}
|
||||
encoded = (json.dumps(receipt, indent=2, sort_keys=True) + "\n").encode()
|
||||
output.mkdir(mode=0o700, parents=False) # Never overwrite an earlier release.
|
||||
write_new(output / ARCHIVE, payload)
|
||||
write_new(output / RECEIPT, encoded) # Completion receipt is written last.
|
||||
sync_directory(output)
|
||||
sync_directory(output.parent)
|
||||
return digest(encoded)
|
||||
|
||||
|
||||
def verify(output, pin):
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", pin):
|
||||
raise ValueError("supply a full externally trusted receipt SHA-256")
|
||||
encoded = read_bounded(output / RECEIPT, 1024 * 1024)
|
||||
if digest(encoded) != pin:
|
||||
raise ValueError("source receipt pin mismatch")
|
||||
receipt = json.loads(encoded, object_pairs_hook=unique_object)
|
||||
if receipt["schema"] != SCHEMA or receipt["archive"] != ARCHIVE:
|
||||
raise ValueError("unsupported source release")
|
||||
if not re.fullmatch(r"argand-site-registry-[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?/",
|
||||
receipt["prefix"]):
|
||||
raise ValueError("unsafe archive prefix")
|
||||
payload = read_bounded(output / ARCHIVE, MAX_BYTES)
|
||||
if len(payload) != receipt["bytes"] or digest(payload) != receipt["sha256"]:
|
||||
raise ValueError("source archive digest mismatch")
|
||||
files = receipt["files"]
|
||||
expected = {item["path"]: item for item in files}
|
||||
if len(expected) != len(files) or not 1 <= len(files) <= MAX_FILES:
|
||||
raise ValueError("duplicate or excessive source entries")
|
||||
for name in expected:
|
||||
safe_path(name)
|
||||
seen, total = set(), 0
|
||||
# Bound expansion before tarfile parses potentially large extended headers.
|
||||
with gzip.GzipFile(fileobj=io.BytesIO(payload)) as compressed:
|
||||
expanded = compressed.read(MAX_TAR_BYTES + 1)
|
||||
if len(expanded) > MAX_TAR_BYTES:
|
||||
raise ValueError("expanded archive exceeds limit")
|
||||
with tarfile.open(fileobj=io.BytesIO(expanded), mode="r:") as archive:
|
||||
for member in archive:
|
||||
if not member.isfile() or not member.name.startswith(receipt["prefix"]):
|
||||
raise ValueError("unsafe archive member")
|
||||
name = member.name.removeprefix(receipt["prefix"])
|
||||
safe_path(name)
|
||||
if name in seen or name not in expected:
|
||||
raise ValueError("duplicate or unexpected archive member")
|
||||
item = expected[name]
|
||||
total += member.size
|
||||
if total > MAX_BYTES or member.size != item["bytes"]:
|
||||
raise ValueError("archive size mismatch")
|
||||
if member.mode not in (0o644, 0o755) or member.mode != item["mode"]:
|
||||
raise ValueError("archive mode mismatch")
|
||||
stream = archive.extractfile(member)
|
||||
if stream is None or digest(stream.read(MAX_BYTES + 1)) != item["sha256"]:
|
||||
raise ValueError("source file digest mismatch")
|
||||
seen.add(name)
|
||||
if seen != expected.keys():
|
||||
raise ValueError("archive omits source files")
|
||||
return receipt
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
make = commands.add_parser("create")
|
||||
make.add_argument("--output", type=Path, required=True)
|
||||
check = commands.add_parser("verify")
|
||||
check.add_argument("--release", type=Path, required=True)
|
||||
check.add_argument("--pin", required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == "create":
|
||||
result = {"release": str(args.output), "pin": create(Path.cwd(), args.output)}
|
||||
else:
|
||||
result = verify(args.release, args.pin)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
except (OSError, ValueError, KeyError, TypeError, tarfile.TarError,
|
||||
subprocess.CalledProcessError) as error:
|
||||
parser.exit(1, f"source release refused: {error}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue