From e26efc19fa7f73e63cd98cb32b446d1fe10eed40 Mon Sep 17 00:00:00 2001 From: nicweyand Date: Sun, 13 Sep 2026 12:22:05 -0400 Subject: [PATCH] release: implement site registry v0.4 trust pipeline --- CHANGELOG.md | 26 + Cargo.lock | 266 +++- Cargo.toml | 2 +- README.md | 278 ++-- SECURITY.md | 7 +- crates/argand-site-registry/Cargo.toml | 2 + .../argand-site-registry/LICENSE_SOURCES.md | 8 +- crates/argand-site-registry/README.md | 372 +++-- .../argand-site-registry-observe@.service | 39 + .../argand-site-registry-observe@.timer | 12 + .../examples/observer.env | 3 + .../argand-site-registry/examples/update.toml | 26 + .../argand-site-registry/migrations/004.sql | 37 + .../argand-site-registry/migrations/005.sql | 25 + crates/argand-site-registry/src/build.rs | 176 ++- crates/argand-site-registry/src/bundle.rs | 248 ++++ crates/argand-site-registry/src/catalog.rs | 65 +- crates/argand-site-registry/src/cli.rs | 884 +++++++++++- crates/argand-site-registry/src/coverage.rs | 576 ++++++++ crates/argand-site-registry/src/crux.rs | 13 +- crates/argand-site-registry/src/diff.rs | 38 +- crates/argand-site-registry/src/download.rs | 12 +- crates/argand-site-registry/src/evidence.rs | 2 +- crates/argand-site-registry/src/generation.rs | 24 +- crates/argand-site-registry/src/identity.rs | 182 ++- crates/argand-site-registry/src/lib.rs | 7 + crates/argand-site-registry/src/model.rs | 112 +- .../argand-site-registry/src/observation.rs | 474 ++++++- crates/argand-site-registry/src/observer.rs | 1243 +++++++++++++++++ crates/argand-site-registry/src/policy.rs | 287 ++++ crates/argand-site-registry/src/query.rs | 3 + crates/argand-site-registry/src/queue.rs | 639 +++++++++ crates/argand-site-registry/src/release.rs | 202 ++- crates/argand-site-registry/src/resolution.rs | 205 ++- crates/argand-site-registry/src/revocation.rs | 459 ++++++ crates/argand-site-registry/src/store.rs | 26 +- crates/argand-site-registry/src/update.rs | 175 ++- crates/argand-site-registry/src/vote.rs | 881 ++++++++++++ crates/argand-site-registry/tests/cli.rs | 42 +- .../argand-site-registry/tests/common/mod.rs | 7 +- .../tests/compatibility.rs | 51 + crates/argand-site-registry/tests/coverage.rs | 133 ++ crates/argand-site-registry/tests/failures.rs | 45 +- .../tests/fixtures/v03-contract.json | 21 + crates/argand-site-registry/tests/identity.rs | 7 +- crates/argand-site-registry/tests/registry.rs | 8 +- crates/argand-site-registry/tests/v04.rs | 1109 +++++++++++++++ docs/ARCHITECTURE.md | 83 ++ docs/CONSUMERS.md | 135 +- docs/EVALUATION.md | 40 +- docs/FORMATS.md | 61 + docs/INDEX.md | 15 +- docs/MIGRATING-0.4.md | 53 + docs/PUBLISHING.md | 140 +- docs/RELEASING.md | 6 +- docs/SECURITY-REVIEW-0.4.md | 76 + docs/TRUST.md | 154 +- docs/adr/0001-typed-source-coverage.md | 26 + docs/adr/0002-granular-trust-subjects.md | 23 + .../0003-votes-revocations-and-publishers.md | 38 + docs/adr/0004-active-and-audit-views.md | 29 + docs/adr/0005-full-delta-release-identity.md | 30 + docs/adr/0006-source-lineage.md | 24 + docs/adr/0007-distribution-and-embedding.md | 27 + .../plans/2026-09-13-v0.4-and-beyond.md | 880 ++++++++++++ scripts/source_release.py | 7 +- tests/test_source_release.py | 8 +- 67 files changed, 10686 insertions(+), 628 deletions(-) create mode 100644 crates/argand-site-registry/examples/argand-site-registry-observe@.service create mode 100644 crates/argand-site-registry/examples/argand-site-registry-observe@.timer create mode 100644 crates/argand-site-registry/examples/observer.env create mode 100644 crates/argand-site-registry/migrations/004.sql create mode 100644 crates/argand-site-registry/migrations/005.sql create mode 100644 crates/argand-site-registry/src/bundle.rs create mode 100644 crates/argand-site-registry/src/coverage.rs create mode 100644 crates/argand-site-registry/src/observer.rs create mode 100644 crates/argand-site-registry/src/policy.rs create mode 100644 crates/argand-site-registry/src/queue.rs create mode 100644 crates/argand-site-registry/src/revocation.rs create mode 100644 crates/argand-site-registry/src/vote.rs create mode 100644 crates/argand-site-registry/tests/compatibility.rs create mode 100644 crates/argand-site-registry/tests/coverage.rs create mode 100644 crates/argand-site-registry/tests/fixtures/v03-contract.json create mode 100644 crates/argand-site-registry/tests/v04.rs create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/FORMATS.md create mode 100644 docs/MIGRATING-0.4.md create mode 100644 docs/SECURITY-REVIEW-0.4.md create mode 100644 docs/adr/0001-typed-source-coverage.md create mode 100644 docs/adr/0002-granular-trust-subjects.md create mode 100644 docs/adr/0003-votes-revocations-and-publishers.md create mode 100644 docs/adr/0004-active-and-audit-views.md create mode 100644 docs/adr/0005-full-delta-release-identity.md create mode 100644 docs/adr/0006-source-lineage.md create mode 100644 docs/adr/0007-distribution-and-embedding.md create mode 100644 docs/superpowers/plans/2026-09-13-v0.4-and-beyond.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b598422..c89661b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## 0.4.0 - 2026-09-13 + +- Add typed full/partition/delta source coverage, active-record masking, a + receipt-bound coverage digest, active export and complete audit export. +- Separate name, website-edge and entity-equivalence identities so aliases cannot + inherit routes and unrelated metadata does not invalidate stable edges. +- Replace latest-review admission with authenticated, policy-bound reviewer votes, + independent identity/group/physical-key quorum, trusted writer acceptance time, + bounded expiry, risk-class threshold overrides and sticky explicit revocation + supersession. +- Bind strict reviewer trust roots and review policy into releases and enforce + publisher identity and physical-key separation. +- Store immutable subject-bound observation batches and add exact/reverse lookup, + bounded DNS-pinned candidate observation, cache-only replay and structured HTTP, + redirect, canonical, hreflang, JSON-LD, sitemap, country, DNS, TLS and failure + evidence. +- Add deterministic evidence bundles, risk-ordered review queues, drift classes, + revocation-candidate export, schema migrations 4 and 5, 0.3 compatibility + guidance, architecture decisions and strict end-to-end security fixtures. +- Add publisher-signed cumulative emergency revocation feeds with seven-day + freshness, rollback-safe continuity, block-only application to older compatible + registries, and reinstatement validation against an exact full generation. +- Add safe scheduled typed-snapshot supersession and a serialized, bandwidth-capped + observer service; reject compressed bodies, redirect loops and duplicate update + coordinates before acquisition. + ## 0.3.0 - 2026-09-13 - Snapshot authenticated SQLite bytes into a private file before queries and sign diff --git a/Cargo.lock b/Cargo.lock index 5533a49..898fe01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,17 +78,18 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "argand-atomic" -version = "0.3.0" +version = "0.4.0" dependencies = [ "tempfile", ] [[package]] name = "argand-site-registry" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "argand-atomic", + "base64", "bzip2", "chrono", "clap", @@ -99,6 +100,7 @@ dependencies = [ "publicsuffix", "reqwest", "rusqlite", + "scraper", "serde", "serde_json", "sha2", @@ -354,6 +356,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "cssparser" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9cdaae01d5ed7882b04d795e7f752f46ff52d2fa3b50a20d28c464510bba98" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a" +dependencies = [ + "quote", + "syn 3.0.3", +] + [[package]] name = "csv" version = "1.4.0" @@ -375,6 +400,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -396,12 +442,33 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "ego-tree" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04dc5a38e4f151a79d9f2451ae6037fb6eaf5cba34771f44781f80e508498e3" + [[package]] name = "equivalent" version = "1.0.2" @@ -553,6 +620,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -613,6 +689,16 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "html5ever" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8" +dependencies = [ + "log", + "markup5ever", +] + [[package]] name = "http" version = "1.5.0" @@ -992,6 +1078,17 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "markup5ever" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1019,6 +1116,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "num-traits" version = "0.2.19" @@ -1075,6 +1178,59 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1096,6 +1252,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1447,6 +1609,21 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -1470,6 +1647,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adfa1c298912827b8a28b223b3b874357397ae706e6190acd9bf28cee99114d" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.28" @@ -1540,6 +1736,15 @@ dependencies = [ "serde", ] +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1589,6 +1794,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -1629,6 +1840,30 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1707,6 +1942,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08" +dependencies = [ + "new_debug_unreachable", +] + [[package]] name = "thiserror" version = "2.0.20" @@ -1933,6 +2177,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "untrusted" version = "0.9.0" @@ -2089,6 +2339,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + [[package]] name = "webpki-root-certs" version = "1.0.9" diff --git a/Cargo.toml b/Cargo.toml index 67416bb..c10eebc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "3" members = ["crates/argand-atomic", "crates/argand-site-registry"] [workspace.package] -version = "0.3.0" +version = "0.4.0" authors = ["Nic Weyand"] edition = "2024" license = "AGPL-3.0-or-later" diff --git a/README.md b/README.md index dd0e4fb..f1b9fbb 100644 --- a/README.md +++ b/README.md @@ -1,69 +1,57 @@ # Argand Site Registry -Argand Site Registry connects an organization, product or other named entity to -its official websites. It is built for navigational search and for choosing the -right regional site without hiding where each claim came from. - -Given `facebook`, a registry can return: +Argand Site Registry is a Rust toolkit for building a trustworthy map from names +to entities to official websites. It is useful for navigational search, regional +site selection, link directories, and any application that needs to explain why +it trusts a destination. ```text -facebook -> Facebook (Wikidata Q355) -> https://www.facebook.com/ - registrable domain: facebook.com +facebook -> Facebook (Wikidata Q355) -> https://www.facebook.com/ + hostname: www.facebook.com + registrable domain: facebook.com + public suffix: com ``` -An entity may have several legitimate properties. For example, one Amazon entity -can carry separately evidenced properties for `amazon.com`, `amazon.co.uk` and -`amazon.de`. The registry never joins entities merely because their names or -hostnames look alike. +The repository contains the library, CLI, schemas, migrations, and synthetic +fixtures. It does not contain a preapproved production dataset. A publisher must +import source evidence, collect signed reviews, and distribute a signed registry +generation. -This repository provides the Rust library, command-line tools and synthetic test -fixtures. It does **not** publish a ready-to-use approved-link dataset. A registry -publisher imports source data, reviews destinations and distributes a signed -generation; a consumer verifies that generation before using it. +## The basic idea -## What it answers +The registry keeps four things separate: -The two main query operations have different purposes: +1. **Assertions:** Wikidata, Curlie, and other reviewed sources say that a name or + website is associated with an entity. +2. **Observations:** a bounded observer records redirects, canonical links, + hreflang, JSON-LD `sameAs`, sitemaps, DNS, TLS, and failures. These are evidence, + not proof of ownership. +3. **Votes:** authenticated reviewers approve or revoke one exact name, website + edge, or entity equivalence under a versioned policy. +4. **Releases:** a separate publisher signs an immutable generation. Consumers + verify it before resolving names. -| Operation | Answer | Trust behavior | -| --- | --- | --- | -| `lookup` | What entities and websites do the sources associate with this name? | Returns evidence for inspection, including conflicts and unreviewed claims. | -| `resolve` | Which destination is approved for this name, locale and country? | Returns only an unambiguous, signed-review-backed, unexpired destination; otherwise it abstains. | +`lookup` shows source evidence and conflicts. `resolve` returns a destination only +when both the name binding and website edge satisfy the configured review policy. +Otherwise it returns a typed abstention such as `no_active_name_review`, +`no_active_review`, or `ambiguous_destination`. -The CLI can also inspect an entity by stable ID, reverse-lookup a URL or domain, -show source-separated popularity signals, inspect redacted Curlie categories, -compare generations and evaluate a pinned generation against JSONL judgments. - -Source confidence is evidence metadata, not a malware-safety score. An imported -official-site claim cannot become a `resolve` destination until an authorized -reviewer approves that exact claim. Applications should still apply their own -security and content policy. - -## How a registry is built +One entity can have several independently reviewed regional properties: ```text -source downloads -> provenance-preserving imports -> immutable candidate - -> signed human reviews -> rebuilt generation -> signed activation - -> lookup / regional resolve +Example Store + primary -> https://example.com/ + country=GB -> https://example.co.uk/ + country=DE -> https://example.de/ ``` -| Stage | Main commands | What changes | -| --- | --- | --- | -| Acquire | `download`, `crux-download`, `manifest` | Stores source bytes and a receipt describing their origin. | -| Import | `import` | Streams normalized facts into the local SQLite writer database. | -| Build | `build` | Produces a new immutable, content-pinned generation. | -| Review | `review`, `equivalence` | Records signed destination or entity-equivalence decisions against exact evidence fingerprints. | -| Publish | `sign`, `activate` | Verifies reviewer trust, signs a generation and atomically selects it. | -| Consume | `lookup`, `resolve`, `lookup-web`, `entity` | Reads a verified generation without changing it. | +The relationship must come from evidence and review. Similar names, ccTLDs, and +hostnames never merge entities or create regional relationships by themselves. -Automated updates stop after building a candidate. They cannot approve a link, -sign a release or activate it. +## Install and try it -## Try it locally - -Linux is the currently validated platform. You need Rust 1.97 or newer, a C -compiler, CMake, Perl and OpenSSH (`ssh-keygen`). Python 3.11 or newer is needed -for release checks and the Python example. SQLite is compiled into the binary. +Linux is the validated platform. You need Rust 1.97 or newer, a C toolchain, +CMake, Perl, Python 3.11 or newer, and OpenSSH `ssh-keygen`. ```bash git clone https://git.argand.org/nicweyand/argand-site-registry.git @@ -73,85 +61,142 @@ cargo build --release --locked --offline -p argand-site-registry ./target/release/argand-site-registry --help ``` -To see the complete lifecycle without downloading provider data, run the native -fixture in a new directory outside the checkout: +Run the complete source-shaped fixture without downloading provider data: ```bash ARGAND_REGISTRY_E2E_OUTPUT=/tmp/site-registry-example \ cargo test -p argand-site-registry --test cli --locked --offline -- --nocapture ``` -The fixture imports source-shaped Wikidata, Majestic Million, CrUX, Curlie and -Public Suffix List records. It proves repeatable imports, alias lookup, explicit -entity equivalence, destination review, signing, activation, revocation and -rollback protection. Its keys and approvals are disposable test material and -must never be used for a real registry. - -Production inputs belong in a configurable data/cache directory outside the -source checkout. Installation and tests do not download datasets or start -scheduled jobs. - -## Use a generation from Rust or Python - -A consumer needs the generation directory and a trusted pin for its -`COMPLETE.json` receipt. Obtain that pin through a trusted publisher channel, or -verify the publisher signature with an independently configured key. A hash found -beside an untrusted download does not authenticate the download. - -The [Rust example](crates/argand-site-registry/examples/lookup.rs) reuses the -native reader. The [Python example](examples/lookup.py) calls the native CLI so it -keeps the same validation and response contract: +Run the strict 0.4 trust and observer acceptance flow: ```bash -cargo run --locked --offline -p argand-site-registry --example lookup -- \ - --generation /data/registry/generation \ - --pin "$REGISTRY_TRUSTED_PIN" \ - --query facebook +cargo test -p argand-site-registry --test v04 --locked --offline -- --nocapture +``` -python3 examples/lookup.py \ - --binary ./target/release/argand-site-registry \ - --generation /data/registry/generation \ - --pin "$REGISTRY_TRUSTED_PIN" \ +The first fixture imports synthetic Wikidata, Majestic Million, CrUX, Curlie, +and Public Suffix List inputs. The second proves two-reviewer name and edge votes, +policy epochs, stale evidence, observation replay, drift, sticky revocation, +publisher separation, signed activation, and resistance to signature tampering. +All keys and approvals created by tests are disposable. + +## Build and use a registry + +Production data and keys belong outside the checkout. A typical lifecycle is: + +```text +download/manifest -> import -> build candidate -> inspect/observe + -> prepare and sign votes -> rebuild -> evaluate -> sign -> activate +``` + +The strict default policy requires two independent reviewer identities and groups +for names, website edges, and entity equivalences. A build binds both the policy +and exact reviewer trust file into its receipt. + +```bash +argand-site-registry build \ + --database /data/site-registry/import.sqlite \ + --output /data/site-registry/candidate \ + --reviewer-trust /secure/site-registry/reviewer-allowed-signers + +PIN=$(sha256sum /data/site-registry/candidate/COMPLETE.json | cut -d' ' -f1) + +argand-site-registry lookup \ + --generation /data/site-registry/candidate --pin "$PIN" --query facebook + +argand-site-registry review-queue \ + --generation /data/site-registry/candidate --pin "$PIN" \ + --at 2026-09-13T00:00:00Z +``` + +For a queued name or edge, prepare canonical vote JSON and sign its exact bytes: + +```bash +argand-site-registry prepare-vote \ + --generation /data/site-registry/candidate --pin "$PIN" \ + --subject-kind edge --fingerprint "$EDGE_FINGERPRINT" \ + --decision approve --reviewer reviewer-one \ + --reason "Verified exact entity, URL, and global role" \ + --reviewed-at 2026-09-13T00:00:00Z \ + --expires-at 2026-10-13T00:00:00Z --role primary \ + --output /secure/site-registry/edge-vote.json + +ssh-keygen -Y sign -n argand-site-registry-vote \ + -f /secure/site-registry/reviewer-one /secure/site-registry/edge-vote.json + +argand-site-registry vote \ + --database /data/site-registry/import.sqlite \ + --generation /data/site-registry/candidate --pin "$PIN" \ + --decision /secure/site-registry/edge-vote.json \ + --signature /secure/site-registry/edge-vote.json.sig \ + --allowed-reviewers /secure/site-registry/reviewer-allowed-signers \ + --identity reviewer-one +``` + +Repeat independently for the second reviewer and for the selected name binding, +then rebuild. `prepare-equivalence-vote`, `verify-equivalence-vote`, and +`equivalence-vote` provide the same flow for explicit cross-source entity links. +The [operator guide](crates/argand-site-registry/README.md) documents source +downloads, typed full/partition/delta coverage, observation commands, regional +roles, release signing, activation, and recovery. + +Consumers should use `resolve`, not the first result from `lookup`: + +```bash +argand-site-registry resolve \ + --generation /data/site-registry/reviewed --pin "$REVIEWED_PIN" \ --query facebook ``` -Do not turn `lookup.candidates[0]` into an automatic redirect. Use `resolve` and -preserve `destination: null` as a deliberate abstention. The -[consumer contract](docs/CONSUMERS.md) documents the Rust API, CLI JSON, signed -generation files, compatibility rules and Argand's pinned integration. +The [Rust example](crates/argand-site-registry/examples/lookup.rs) reuses a +verified reader. The [Python example](examples/lookup.py) invokes the same native +CLI and preserves its JSON contract. -## Operate or contribute +## Sources and licenses -The [operator guide](crates/argand-site-registry/README.md) contains the complete -commands for acquiring each source, importing large files, reviewing regional -properties, signing releases, scheduling candidate updates and recovering state. -CrUX acquisition always requires explicit credentials and a billing cap. - -The registry currently supports these source adapters: - -| Source | Purpose | Data license | +| Source | Consumed evidence | Data license | | --- | --- | --- | -| Wikidata | Entity names, aliases, official websites and locale/country evidence | CC0 1.0 Universal | -| Majestic Million | Domain popularity rank | CC BY 3.0 Unported | -| Chrome UX Report (CrUX) | Popular-origin rank bucket | CC BY 4.0 International | -| Curlie | Human-curated names and categories | CC BY 3.0 Unported | -| Public Suffix List | Public-suffix and registrable-domain parsing | Mozilla Public License 2.0 | +| Wikidata | IDs, labels, aliases, P856 statements, selected locale/country metadata | CC0 1.0 Universal | +| Majestic Million | source-specific domain rank and supplied metrics | CC BY 3.0 Unported | +| Chrome UX Report | origin popularity bucket, month, optional audience country | CC BY 4.0 International | +| Curlie | site titles, categories, descriptions retained for audit | CC BY 3.0 Unported | +| Public Suffix List | ICANN and PRIVATE suffix rules | MPL 2.0 | -Read [LICENSE_SOURCES.md](LICENSE_SOURCES.md) before redistributing data. Curlie -attribution applies to names and categories as well as descriptions. The registry -keeps each source logically separate and retains provenance, licenses, retrieval -timestamps, source identifiers and conflicting evidence. +Popularity never proves identity or ownership. Curlie attribution applies to +names and categories as well as descriptions; copied descriptions are redacted +unless the caller explicitly exports them and satisfies the display obligations. +Read [LICENSE_SOURCES.md](LICENSE_SOURCES.md) before distributing provider data. -For the security model and release process, see: +Cloudflare Radar, default Tranco, Cisco Umbrella, arbitrary mirrors, and sources +without verified commercial reuse rights are unsupported. Adding a provider +requires an explicit adapter, current format inspection, provenance, rights +review, attribution rules, and tests. -- [Trust and evidence policy](docs/TRUST.md) -- [Publishing runbook](docs/PUBLISHING.md) -- [Security policy](SECURITY.md) -- [Governance](GOVERNANCE.md) -- [Full documentation index](docs/INDEX.md) +## Safety and reproducibility -Pull requests may add evidence or code, but cannot directly approve a production -destination. +- Raw sources remain logically separate and content-addressed outside Git. +- Every imported or derived fact keeps source, source identifier, license, + retrieval time, selector, and confidence. +- Typed coverage declares full snapshots, partitions, deltas, bases, and exact + supersession; conflicts fail closed. +- Active export contains selected, nonrejected facts. Audit export also retains + superseded and rejected evidence. +- Votes bind assertion, evidence-bundle, policy, reviewer, scope, and expiry. +- Revocations remain sticky until every member of a fresh quorum explicitly + supersedes them. +- Small publisher-signed cumulative revocation feeds can block a compatible + pinned offline registry before its replacement generation arrives. They expire + after seven days and cannot restore a route across generations. +- The observer follows at most five redirects, uses public DNS pinning, blocks + private and link-local networks, rejects HTTPS downgrade and nondefault ports, + and caps headers, bodies, time, and extracted links. +- Automated updates may download, import, observe, and build candidates. They do + not approve, renew, sign, activate, or silently choose a destination. + +The code is **AGPL-3.0-or-later**. Provider data retains its own license. See the +[documentation index](docs/INDEX.md), [architecture](docs/ARCHITECTURE.md), +[trust model](docs/TRUST.md), [publishing runbook](docs/PUBLISHING.md), and +[consumer contract](docs/CONSUMERS.md). ## Development @@ -160,14 +205,7 @@ cargo fetch --locked bash scripts/check.sh ``` -The check covers formatting, all-target compilation, strict Clippy, Rust tests, -API documentation, Python release tests and native Rust/Python consumer parity. -[RELEASING.md](docs/RELEASING.md) explains deterministic source archives and -independent signature verification; [VALIDATION.md](docs/VALIDATION.md) records -the current independent build and acceptance evidence. - -The code is licensed under **AGPL-3.0-or-later**; see [LICENSE](LICENSE). Provider -data keeps its own license. [UPSTREAM.json](UPSTREAM.json) records the signed -Argand extraction revision and original file hashes. Argand pins the signed -`v0.3.0` release by full Git revision and no longer carries an embedded source -copy. +The gate covers formatting, all-target compilation, strict Clippy, Rust tests, +API documentation, Python release checks, deterministic packaging, and native +Rust/Python consumer parity. Security validation is recorded separately in +[docs/VALIDATION.md](docs/VALIDATION.md). diff --git a/SECURITY.md b/SECURITY.md index 200049d..b473691 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,8 +14,11 @@ affected generation. Use inert or local fixtures where possible. Publishers should append a revocation for a suspected malicious destination, build and inspect a replacement generation with the full review history, sign it with a trusted key and deliver its new pin to consumers. Consumers must refresh -their verified registry and any derived routing catalogue. Merely appending a -local revocation does not notify running applications or invalidate their caches. +their verified registry and any derived routing catalogue. A seven-day emergency +feed can add blocks to an older compatible generation, but only the exact full +generation can clear one after the consumer recomputes the signed superseding +quorum. Merely appending a local revocation does not notify running applications +or invalidate their caches. For signer compromise, remove that signer from consumer trust files through an independent authenticated channel, investigate affected releases and rotate the diff --git a/crates/argand-site-registry/Cargo.toml b/crates/argand-site-registry/Cargo.toml index 1c4c675..2506971 100644 --- a/crates/argand-site-registry/Cargo.toml +++ b/crates/argand-site-registry/Cargo.toml @@ -10,6 +10,7 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true argand-atomic = { path = "../argand-atomic" } +base64 = "0.22.1" bzip2 = "0.6.1" chrono.workspace = true clap.workspace = true @@ -19,6 +20,7 @@ libc.workspace = true publicsuffix = "=2.3.0" reqwest.workspace = true rusqlite = { version = "=0.40.2", features = ["bundled"] } +scraper = "0.27.0" serde.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/argand-site-registry/LICENSE_SOURCES.md b/crates/argand-site-registry/LICENSE_SOURCES.md index 057a839..eb293b7 100644 --- a/crates/argand-site-registry/LICENSE_SOURCES.md +++ b/crates/argand-site-registry/LICENSE_SOURCES.md @@ -1,6 +1,6 @@ # Site Registry source licenses -Reviewed against the primary distribution and licensing pages on 2026-09-12. +Reviewed against the primary distribution and licensing pages on 2026-09-13. The Rust code uses the workspace's **AGPL-3.0-or-later** license. Imported data keeps its own licenses; neither the code license nor a merged export relicenses it. Commercial reuse is supported subject to the following obligations. A provider's @@ -13,6 +13,7 @@ listing is evidence of an assertion, not a guarantee of ownership or safety. | Chrome UX Report (CrUX), Google | [CC BY 4.0 International](https://creativecommons.org/licenses/by/4.0/), [`CC-BY-4.0`](https://developer.chrome.com/docs/crux/methodology) | [Monthly BigQuery dataset](https://developer.chrome.com/docs/crux/bigquery/): `origin`, `experimental.popularity.rank`, observation month, optional audience-country dataset code. The adapter produces `origin,rank,yyyymm,country_code` CSV. Rank is a coarse bucket, not a precise visit count. Audience country is not website jurisdiction. No API key or OAuth token is retained. | | Curlie | [CC BY 3.0 Unported](https://creativecommons.org/licenses/by/3.0/), [`CC-BY-3.0`](https://curlie.org/docs/en/license.html), including the attribution placement prescribed on that page | [Format documentation](https://curlie.org/docs/en/rdf.html), [official download redirect](https://curlie.org/directory-dl), currently [Passau-hosted archive](https://share.innkube.fim.uni-passau.de/curlie-rdf/curlie-rdf-all.tar.gz). Despite its RDF name, the current archive contains **literal TSV**. Content: URL, title, description, category ID. Structure: category ID, full category path, entry count, description, latitude, longitude. Archive notices are retained. | | Public Suffix List contributors | [Mozilla Public License 2.0](https://mozilla.org/MPL/2.0/), [`MPL-2.0`](https://publicsuffix.org/list/public_suffix_list.dat) | [Official list](https://publicsuffix.org/list/public_suffix_list.dat). All ICANN and PRIVATE rules, wildcard/exception rules, version/commit comments and notices. Used for hostname, registrable-domain and public-suffix derivations. Download at most once per day. | +| Argand candidate observer | [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/), `CC0-1.0` | Local host-side observations authored by the registry publisher: HTTP status and redirect targets, canonical/hreflang/JSON-LD/sitemap/country-selector targets, public DNS-set hash, TLS leaf-certificate hash, bounded failure class, content hash and selectors. These records describe a capture; they do not incorporate page prose or prove ownership. | ## Attribution and distribution @@ -47,6 +48,11 @@ listing is evidence of an assertion, not a guarantee of ownership or safety. and `facts`; exports include the PSL fact and original download locator. Changes to covered PSL source files must remain available under MPL 2.0. The Rust `publicsuffix` parser is MIT/Apache-2.0; that is separate from the list. +* **Argand observer:** locally produced observation metadata is dedicated under + CC0. The fetched page remains subject to its own rights. The default observer + stores only a bounded body in the local replay cache and emits normalized link, + status, hash and failure metadata. Publishers control and document retention of + local cache bodies; generated registry releases do not contain them. Every generation contains this document and `ATTRIBUTION.json`, both hash-bound by its signed completion receipt. Exports carry source manifests, licenses, diff --git a/crates/argand-site-registry/README.md b/crates/argand-site-registry/README.md index db345e9..163f776 100644 --- a/crates/argand-site-registry/README.md +++ b/crates/argand-site-registry/README.md @@ -53,28 +53,39 @@ of current source sizes. Increase a cap only after checking available storage. export ARGAND_SITE_DATA="$HOME/.local/share/argand-site-registry" mkdir -p "$ARGAND_SITE_DATA" +cat > "$ARGAND_SITE_DATA/full-coverage.json" <<'JSON' +{"collection":"default","kind":"full","partition":null,"base":null,"sequence":null,"supersedes":[]} +JSON +cat > "$ARGAND_SITE_DATA/wikidata-selection-coverage.json" <<'JSON' +{"collection":"entity-selections","kind":"partition","partition":"facebook-amazon","base":null,"sequence":null,"supersedes":[]} +JSON + argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \ --source psl --format psl-text \ --url https://publicsuffix.org/list/public_suffix_list.dat \ --snapshot "$(date -u +%F)" --scope full --maximum-bytes 1000000 \ + --coverage "$ARGAND_SITE_DATA/full-coverage.json" \ > "$ARGAND_SITE_DATA/psl-download.json" argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \ --source wikidata --format wikidata-entities \ --url 'https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q355%7CQ3884&format=json&maxlag=5' \ --snapshot "$(date -u +%F)" --scope selection:facebook-amazon \ + --coverage "$ARGAND_SITE_DATA/wikidata-selection-coverage.json" \ --maximum-bytes 5000000 > "$ARGAND_SITE_DATA/wikidata-download.json" argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \ --source majestic --format majestic-csv \ --url https://downloads.majestic.com/majestic_million.csv \ --snapshot "$(date -u +%F)" --scope full --maximum-bytes 250000000 \ + --coverage "$ARGAND_SITE_DATA/full-coverage.json" \ > "$ARGAND_SITE_DATA/majestic-download.json" argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \ --source curlie --format curlie-tar-gz \ --url https://curlie.org/directory-dl \ --snapshot "$(date -u +%F)" --scope full --maximum-bytes 1000000000 \ + --coverage "$ARGAND_SITE_DATA/full-coverage.json" \ > "$ARGAND_SITE_DATA/curlie-download.json" for source in psl wikidata majestic curlie; do @@ -84,6 +95,13 @@ for source in psl wikidata majestic curlie; do done ``` +The reusable `default` collection is safe because coverage graphs are separated +by provider. For a replacement, copy the preceding manifest ID into the new +coverage object's `supersedes` array. For a delta, also set `kind: "delta"`, +`base` to that ID, and a consecutive positive `sequence`. Use distinct stable +`partition` names only for provider-declared disjoint subsets. The build rejects +ambiguous or overlapping typed coverage. + For a full Wikidata dump, select a real dump URL from the official [download index](https://dumps.wikimedia.org/wikidatawiki/entities/), then use `--format wikidata-dump --compression gzip` (or `bzip2`) and `--scope full`. @@ -134,7 +152,15 @@ Create `crux-request.json` with your project and explicit limits: "month": "202608", "country": null, "maximum_bytes_billed": 1000000000, - "maximum_output_bytes": 500000000 + "maximum_output_bytes": 500000000, + "coverage": { + "collection": "monthly-origins", + "kind": "partition", + "partition": "global", + "base": null, + "sequence": null, + "supersedes": [] + } } ``` @@ -158,224 +184,192 @@ exact CSV projection can instead use `manifest --source crux --format crux-csv retrieval time, query/snapshot identity and appropriate `monthly:YYYYMM:country` scope. The token is never written into a manifest. -## Build, look up and review +Keep a partition coordinate stable across refreshes, such as `global` or `GB`. +The month belongs in the source snapshot identity. With scheduled typed +supersession enabled, the next month then replaces the same audience partition +instead of accumulating stale popularity facts. + +## Build, inspect and review + +The strict default requires an independently maintained OpenSSH reviewer trust +file and two independent approvals for each name, edge, and equivalence. Build a +candidate and inspect its deterministic work queue: ```bash argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \ - --output "$ARGAND_SITE_DATA/generation-1" > "$ARGAND_SITE_DATA/build-1.json" -export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-1.json")" -argand-site-registry lookup --generation "$ARGAND_SITE_DATA/generation-1" \ + --output "$ARGAND_SITE_DATA/candidate" \ + --reviewer-trust /secure/reviewer-allowed-signers \ + > "$ARGAND_SITE_DATA/candidate.json" +export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/candidate.json")" + +argand-site-registry lookup --generation "$ARGAND_SITE_DATA/candidate" \ --pin "$ARGAND_SITE_PIN" --query facebook -argand-site-registry lookup --generation "$ARGAND_SITE_DATA/generation-1" \ - --pin "$ARGAND_SITE_PIN" --query amazon --limit 100 +argand-site-registry review-queue --generation "$ARGAND_SITE_DATA/candidate" \ + --pin "$ARGAND_SITE_PIN" --at 2026-09-13T00:00:00Z ``` -The real-source acceptance run produced: +A source-shaped Facebook result retains `https://www.facebook.com/`, its +Wikidata Q355 identity, the `facebook.com` registrable domain, all relevant source +facts, and source-separated popularity. An entity can carry `example.com`, +`example.co.uk`, and `example.de` only when source evidence attaches each property +to that exact entity. PSL parsing returns `example.co.uk`, not `co.uk`, as the +registrable domain. -| Query | Entity | Example properties | Registrable domains | -| --- | --- | --- | --- | -| `facebook` | Facebook (Q355) | `https://www.facebook.com/`, `https://m.facebook.com/` | `facebook.com` | -| `amazon` | Amazon (Q3884) | `https://www.amazon.com/`, `https://www.amazon.co.uk/`, `https://www.amazon.de/` | `amazon.com`, `amazon.co.uk`, `amazon.de` | - -These properties were asserted on the **same Wikidata entity**. Hostname -resemblance did not establish the relationship. Imported qualifiers remain in -`evidence` and `property_scopes`; unknown locale/country remains null. Names, -aliases and entity metadata carry their own fact-level source declarations. -Regional locale/country and role are explicit reviewed assertions. They are -separate from entity headquarters, ccTLD spelling and CrUX audience country. - -Inspect the full statements, references, names/aliases, hostname spelling and -independent current ownership/role evidence. A review JSON has this shape: - -```json -{ - "fingerprint": "COPY_THE_EXACT_64_CHARACTER_FINGERPRINT_FROM_LOOKUP", - "decision": "approve", - "reviewer": "operator identity", - "reason": "How entity ownership and this exact destination role were verified", - "evidence": "An immutable capture identifier or evidence digest", - "reviewed_at": "2026-09-12T12:00:00Z", - "expires_at": "2026-10-12T12:00:00Z", - "role": "regional", - "locale": "", - "country": "GB" -} -``` - -Replace the example evidence and dates; approvals expire within 90 days. Use -`role: "primary"` for the independently verified default and `country: "DE"` -for a separately verified German regional property. A country-scoped review -can leave locale empty. If both are specified, both must match the request. -Sign the exact decision bytes under the dedicated reviewer namespace. The reviewer -identity must match the JSON and an independently maintained OpenSSH allowed-signers -file. The release signing key may be separate from reviewer keys. +Prepare canonical vote JSON for the exact queued name or edge. The command fills +the current evidence-bundle and policy digests. Never hand-copy an earlier digest. ```bash -ssh-keygen -Y sign -n argand-site-registry-review \ - -f /secure/reviewer-key review.json -argand-site-registry review --database "$ARGAND_SITE_DATA/import.sqlite" \ - --generation "$ARGAND_SITE_DATA/generation-1" --pin "$ARGAND_SITE_PIN" \ - --decision review.json --signature review.json.sig \ - --allowed-reviewers /secure/reviewer-allowed-signers \ - --identity operator@example.org -argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \ - --output "$ARGAND_SITE_DATA/generation-2" > "$ARGAND_SITE_DATA/build-2.json" -export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-2.json")" -argand-site-registry resolve --generation "$ARGAND_SITE_DATA/generation-2" \ - --pin "$ARGAND_SITE_PIN" --query amazon --country GB +argand-site-registry prepare-vote \ + --generation "$ARGAND_SITE_DATA/candidate" --pin "$ARGAND_SITE_PIN" \ + --subject-kind edge --fingerprint "$EDGE_FINGERPRINT" \ + --decision approve --reviewer reviewer-one \ + --reason "Verified entity, URL, and exact role from retained evidence" \ + --reviewed-at 2026-09-13T00:00:00Z \ + --expires-at 2026-10-13T00:00:00Z --role primary \ + --output /secure/edge-vote.json +ssh-keygen -Y sign -n argand-site-registry-vote \ + -f /secure/reviewer-one /secure/edge-vote.json +argand-site-registry verify-vote \ + --generation "$ARGAND_SITE_DATA/candidate" --pin "$ARGAND_SITE_PIN" \ + --decision /secure/edge-vote.json --signature /secure/edge-vote.json.sig \ + --allowed-reviewers /secure/reviewer-allowed-signers --identity reviewer-one +argand-site-registry vote --database "$ARGAND_SITE_DATA/import.sqlite" \ + --generation "$ARGAND_SITE_DATA/candidate" --pin "$ARGAND_SITE_PIN" \ + --decision /secure/edge-vote.json --signature /secure/edge-vote.json.sig \ + --allowed-reviewers /secure/reviewer-allowed-signers --identity reviewer-one ``` -After the corresponding real reviews, GB selects the reviewed UK property; -DE selects the reviewed German property; otherwise an explicitly reviewed -primary may be used. Unknown, expired, tied or entity-ambiguous requests return -`"destination": null` with `status` and rejection counts. Result limits never hide -ambiguity. Name/alias changes, -changed statements/revisions, URLs or normalization evidence invalidate reviews. -Deprecated, end-dated and non-value statements remain audit evidence and cannot -be admitted. An unchanged PSL file with a new retrieval time preserves reviews. +Repeat with another identity, group, and physical key. Prepare separate name votes +for the exact label or alias used by the query. A regional edge approval uses +`--role regional` plus `--country GB`, `--locale en-GB`, or both. A primary edge +is an unscoped global fallback. If both country and locale are set, both must +match. The resolver abstains on missing quorum, ambiguity, expiry, revocation, +stale evidence, stale policy, or equal destinations. + +The writer assigns `accepted_at`. Effective validity begins at the later of that +time and signed `reviewed_at`, and ends no later than 90 days after acceptance. +A revocation has no expiry. Every approval in a new quorum must pass each active +revocation ID with `--supersedes`; ordinary later approvals remain blocked. + +### Observe an existing candidate + +The observer is candidate-only and does not grant approval. It pins public DNS per +hop, rejects private/link-local addresses, credentials, nondefault ports, HTTPS +downgrade, compressed response bodies, oversized headers/bodies and more than five redirects. Store its cache +outside Git, replay it without network access, import the immutable batch, and +rebuild: + +```bash +argand-site-registry observe \ + --generation "$ARGAND_SITE_DATA/candidate" --pin "$ARGAND_SITE_PIN" \ + --fingerprint "$EDGE_FINGERPRINT" \ + --capture "$ARGAND_SITE_DATA/captures/run-1" \ + --output "$ARGAND_SITE_DATA/observations/run-1.jsonl" \ + --manifest-output "$ARGAND_SITE_DATA/observations/run-1.source.json" +argand-site-registry observation-import \ + --database "$ARGAND_SITE_DATA/import.sqlite" \ + --generation "$ARGAND_SITE_DATA/candidate" --pin "$ARGAND_SITE_PIN" \ + --input "$ARGAND_SITE_DATA/observations/run-1.jsonl" \ + --manifest "$ARGAND_SITE_DATA/observations/run-1.source.json" +``` + +`observe-replay` reproduces JSONL from the cache without a request. +`observations` looks up an exact subject, `observation-lookup` searches exact URLs +or domains, and `drift` compares the latest two batches. A new observation changes +the evidence bundle and requires fresh review; it never auto-renews a vote. +For scheduled runs, `{timestamp}` in the three `observe` output paths expands once +to a nanosecond UTC coordinate. The template service and timer in `examples/` +serialize enabled fingerprints with a runtime lock, cap response bandwidth, and +run one candidate per process; operators still import and +review the produced batch separately. ## Release, export, update and recovery -When two providers describe the same navigational entity, `lookup` deliberately -shows both source IDs. Connect them only after reviewing their identities: +Preview an entity pair, then use the dedicated equivalence vote commands. Two +similar names or domains remain separate until the equivalence quorum passes: ```bash -argand-site-registry equivalence --generation "$ARGAND_SITE_DATA/generation-2" \ - --pin "$ARGAND_SITE_PIN" --left SOURCE_ENTITY_ID --right OTHER_SOURCE_ENTITY_ID +argand-site-registry equivalence --generation "$ARGAND_SITE_DATA/candidate" \ + --pin "$ARGAND_SITE_PIN" --left SOURCE_ENTITY_ID --right OTHER_ENTITY_ID +argand-site-registry prepare-equivalence-vote \ + --generation "$ARGAND_SITE_DATA/candidate" --pin "$ARGAND_SITE_PIN" \ + --left SOURCE_ENTITY_ID --right OTHER_ENTITY_ID --decision approve \ + --reviewer reviewer-one --reason "Same entity under both source IDs" \ + --reviewed-at 2026-09-13T00:00:00Z \ + --expires-at 2026-10-13T00:00:00Z --output /secure/equivalence-vote.json ``` -Use the returned fingerprint in a review JSON with `role: "unspecified"`, empty -locale/country, a reason, immutable identity evidence and an expiry. Sign -`identity-review.json` with the reviewer namespace and repeat the command with -`--database`, `--decision`, `--signature`, `--allowed-reviewers` and `--identity`, -then rebuild. `resolve` follows only active, explicitly -reviewed equivalences and includes their provenance. Each destination still -needs its own review. The original IDs, raw ambiguity counts and conflicting -assertions remain visible. Changed names or website assertions invalidate the -identity decision; identity revocations use the same append-only release log. -Operator-authored decisions are published under CC0-1.0, separately from source -data licenses. - -Each generation contains `registry.sqlite`, `LICENSE_SOURCES.md`, -`ATTRIBUTION.json` and a hash-binding `COMPLETE.json`. Distribute all four together. -Keep the import database, review history and cached source bytes for recovery. -The SQLite file includes raw relevant records and descriptions for audit; public -consumers must obey the source attribution requirements. `export` streams -source-bearing JSONL and omits descriptions by default: +After appending complete quorums, rebuild and inspect `stats`, `diff`, +`review-queue`, and `evaluate`. `export` writes selected, nonrejected facts. +`export-audit` also writes superseded and rejected facts. Both preserve provenance +and attribution, redact Curlie descriptions by default, and are evidence exports +rather than admitted-route lists. ```bash -argand-site-registry export --generation "$ARGAND_SITE_DATA/generation-2" \ - --pin "$ARGAND_SITE_PIN" --output "$ARGAND_SITE_DATA/assertions.jsonl" -argand-site-registry sign --generation "$ARGAND_SITE_DATA/generation-2" \ - --pin "$ARGAND_SITE_PIN" --key /secure/registry-signing-key \ - --allowed-reviewers /secure/reviewer-allowed-signers -argand-site-registry activate --generation "$ARGAND_SITE_DATA/generation-2" \ +argand-site-registry export --generation "$ARGAND_SITE_DATA/reviewed" \ + --pin "$REVIEWED_PIN" --output "$ARGAND_SITE_DATA/active.jsonl" +argand-site-registry export-audit --generation "$ARGAND_SITE_DATA/reviewed" \ + --pin "$REVIEWED_PIN" --output "$ARGAND_SITE_DATA/audit.jsonl" +argand-site-registry sign --generation "$ARGAND_SITE_DATA/reviewed" \ + --pin "$REVIEWED_PIN" --key /secure/publisher-key \ + --allowed-reviewers /secure/reviewer-allowed-signers \ + --identity registry-publisher +argand-site-registry activate --generation "$ARGAND_SITE_DATA/reviewed" \ --current "$ARGAND_SITE_DATA/current.json" \ - --allowed-signers /secure/registry-allowed-signers \ + --allowed-signers /secure/publisher-allowed-signers \ --allowed-reviewers /secure/reviewer-allowed-signers \ --identity registry-publisher ``` -Before signing, the CLI replays every stored reviewer signature against the supplied -reviewer trust file and rejects missing, forged or altered proofs. Use an existing -operator-controlled SSH release key. The external release allowed-signers -file follows OpenSSH syntax: `registry-publisher ssh-ed25519 PUBLIC_KEY`. Neither -keys nor the trust file should come from the downloaded dataset. Consumers can -open `Registry::open(path, trusted_receipt_sha256)` once and reuse its indexed -queries, or verify both a publisher and the retained reviewer proofs with -`release::verify_signed` first. A hash proves -integrity only relative to a trusted pin. Signature verification authenticates -the publisher, not the truth of a source assertion. +Strict signing rejects a publisher identity or physical key used for any reviewer vote. +Activation re-verifies the publisher, reviewer trust digest, every retained vote, +and revocation continuity. Rollback is allowed only when the target retains every +distributed legacy and vote revocation. -`diff --old PATH --old-pin HASH --new PATH --new-pin HASH` streams typed added, -removed and changed source selections, entities, names, properties, edges, -popularity observations, reviews and equivalences. `verify --generation PATH ---pin HASH` checks every artifact -bound by the receipt. To revoke, append a review with `decision: "revoke"`, then -rebuild, sign and activate. Re-activating an older signed generation supports -rollback **only if it retains every distributed revocation**. Otherwise rebuild -the older source selection with the current review log; never edit generations. +For emergency delivery, `export-revocations` creates a cumulative feed at an +explicit time. `sign-revocations` recomputes it from the pinned generation before +using the publisher's separate SSH key. `verify-revocations` checks its signature, +compatibility, effective time, seven-day refresh deadline, and optional prior-feed +continuity. Cross-generation feeds can add blocks but cannot restore a route. A consumer may +pass `--revocations`, `--revocation-signature`, `--allowed-publishers`, and +`--publisher-identity` to `resolve` so the signed block applies before a full +replacement generation is installed. After bootstrap, also pass the last accepted +feed through `--previous-revocations` and `--previous-revocation-signature` to +reject a replacement that drops retained revocations. -The [update configuration](examples/update.toml) and [systemd service/timer](examples/) -provide weekly candidate refreshes without a resident daemon. Set absolute paths -and an installed executable path. TOML paths do not expand environment variables. -`update --config /etc/argand-site-registry.toml` downloads/imports all configured -sources and builds only after they succeed. The same inputs and review log reuse -the same generation. A nonzero exit is a failed refresh; the active pointer stays -intact. Failed `pending-*` builds can be inspected before explicitly removing -that incomplete directory. Acquisition, import and update operations take local -locks; use one writer and keep old complete generations for rollback. - -Downloads permit only the reviewed HTTPS source endpoints, validate each -redirect, bound bytes and bind range resumes to strong ETags. Chunked/validatorless -responses safely restart on interruption. PSL network attempts are limited to -once per 24 hours per cache. CrUX is opt-in and may use `{previous_month}` in -update configuration; monthly data may not yet be published on the first day. -Wikidata source snapshot labels support `{date}` and `{month}` in scheduled -downloads. No scheduled job signs, approves, renews approvals or activates links. +The [update configuration](examples/update.toml) and +systemd examples refresh candidates without a resident daemon. Its explicit +`auto_supersede_typed_snapshots = true` setting replaces only the current frontier +with the same provider, collection and full/partition coordinate; deltas and +coverage-layout changes still require exact operator-supplied bases. Update jobs may +download, import and build. Schedule observation commands independently according +to risk. Use separate cache/capture/output paths and process concurrency limits; +one observer invocation handles one candidate with explicit body, redirect and +time bounds. No scheduled command approves, renews, signs or activates. ## Storage and operating limits -Migrations `migrations/001.sql`, `002.sql` and `003.sql` own schema version 3. `sources`, `records` and -`facts` preserve snapshot/native IDs, licenses, retrieval times and confidence; -`reviews` and their cryptographic `review_auth` proofs are append-only. Complete source selection is latest retrieval time per -provider/scope, with digest as the deterministic tie break. Use the **same scope** -for a replacement snapshot, and separate scopes for deliberate independent -selections. History and conflicts remain stored. A failed source cannot replace -a complete one. Avoid overlapping full/partial scopes unless both evidences are -intended to remain active. +Migrations 001 through 005 own writer schema 5. Source manifests v2 declare typed +full, partition, or delta coverage. Deltas name an exact base, consecutive sequence +and superseded object. Ambiguous coverage, overlap, cycles, gaps, cross-provider +supersession and duplicate native records fail the build. V1 manifests remain +isolated by provider and scope for compatibility. -Derived tables are `selected_sources`, `entities`, `names`, `properties`, `edges`, -`popularity` and `rejected`. Entity IDs derive from source/native IDs; URL IDs -derive from strict normalized URLs. Equal source entities merge across snapshots -and equal URLs share a property. Explicit `equivalences` connect reviewed -cross-source identities while preserving both IDs. Names are never an identity -join. The canonical label rule prefers labels, then English, -then language/text order. Original labels/aliases are kept. Popularity has its -own source, target, observation period and audience scope and never creates an -ownership edge. All derivations bind input fact IDs, PSL identity and the -`argand.site-rules/v3` contract through their generation receipt. A v1 or v2 writer store -migrates in place while retaining review history. Any legacy unauthenticated -decision makes release signing fail closed; start a reviewed v3 store from the -pinned source inputs rather than deleting historical decisions. +Each generation contains `registry.sqlite`, `LICENSE_SOURCES.md`, +`ATTRIBUTION.json`, and `COMPLETE.json`, plus an optional publisher signature. The +receipt binds database bytes, source selection, policy, reviewer trust, licenses, +attribution, and decision-time contract. Keep source objects, writer state, +generations, pins, trust files and signatures for recovery. -Imports use transactions of 256 relevant records with durable replay checkpoints. -Restart replays the compressed stream and skips committed records. The exact open -descriptor stream is hashed, and any integrity or resource failure removes all -partially committed rows for that source. Large source records are capped at 16 MiB; -imports also have finite expanded-byte, record and database-growth ceilings. Override -them with `import --maximum-expanded-bytes`, `--maximum-records` and -`--maximum-database-growth-bytes` when a reviewed source requires different bounds. -SQLite has an 8 MiB page cache and disk-backed sorts. -Builds stream a canonical sorted copy and never load the full registry into RAM. -Names and entity metadata exposed by lookup are capped at 256 facts each, with -uncapped totals; the complete assertions remain available in the database/export. -Lookup returns at most 100 edges, reports all pre-limit ambiguity counts and caps -aggregate serialized candidate data at 64 MiB. Typed diff events are capped at -16 MiB and the full stream at 1 GiB; descriptions are redacted and the header -contains source attribution. -The full store/history and each generation consume disk; there is no automatic -pruning. These bounds are not a full-dump throughput claim. +Imports are streaming, transactional, resumable and idempotent. Defaults bound +expanded bytes, record size/count, database growth, query output and diff output. +Use command-line overrides only after checking the real object and local capacity. +Raw datasets, credentials, signing keys and production review logs do not belong +in this repository. -The `observation` module validates and deterministically normalizes future crawler -evidence for redirects, canonical links, hreflang, JSON-LD sameAs, sitemaps and -country selectors, with capture IDs, hashes, rights and confidence. It does not -crawl, admit a new source or automatically infer ownership. - -Use `entity`, `lookup-web`, `popularity`, `category` and `stats` for reverse/audit -views. Curlie descriptions remain redacted on the category surface. The -`evaluate` command accepts bounded JSONL judgments; see the repository -[evaluation guide](../../docs/EVALUATION.md) and [publisher runbook](../../docs/PUBLISHING.md). - -Source vandalism, compromised publishers and a domain changing ownership cannot -be eliminated by hashes or popularity. Review expiration, exact evidence binding, -signed releases, explicit revocations and conservative abstention contain those -risks. Protect the writer database, signing key and consumer trust configuration. -Do not feed raw `lookup` candidates straight into an automatic redirect consumer. - -If an import fails, fix the input/format or reuse the matching original source -manifest, then rerun the same import. Do not edit digests to make corrupted data -pass. A source host/schema change needs an adapter review. HTTP 403/429 is a -source-access failure; reuse an authorized retained snapshot or retry according -to the provider's policy. A null resolution means evidence/review is missing, -expired or ambiguous; `lookup` explains which assertions are involved. +`lookup`, reverse lookup, popularity and categories are audit surfaces. Popularity, +TLS, DNS, redirects, `sameAs`, ccTLD spelling and source confidence do not prove +ownership or safety. Use `resolve` for navigation and keep a null destination as +an intentional abstention. See [TRUST.md](../../docs/TRUST.md), +[PUBLISHING.md](../../docs/PUBLISHING.md), and +[LICENSE_SOURCES.md](LICENSE_SOURCES.md). diff --git a/crates/argand-site-registry/examples/argand-site-registry-observe@.service b/crates/argand-site-registry/examples/argand-site-registry-observe@.service new file mode 100644 index 0000000..5245579 --- /dev/null +++ b/crates/argand-site-registry/examples/argand-site-registry-observe@.service @@ -0,0 +1,39 @@ +# By Nic Weyand! One bounded candidate fingerprint per instance; no approval authority. +[Unit] +Description=Observe Argand Site Registry candidate %i +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=argand-site-registry +Group=argand-site-registry +StateDirectory=argand-site-registry +RuntimeDirectory=argand-site-registry-observer +EnvironmentFile=/etc/argand-site-registry-observer.env +ExecStartPre=/usr/bin/mkdir -p /var/lib/argand-site-registry/captures /var/lib/argand-site-registry/observations +ExecStart=/usr/bin/flock --nonblock /run/argand-site-registry-observer/observer.lock /usr/local/bin/argand-site-registry observe --generation ${ARGAND_REGISTRY_GENERATION} --pin ${ARGAND_REGISTRY_PIN} --fingerprint %i --capture /var/lib/argand-site-registry/captures/%i-{timestamp} --output /var/lib/argand-site-registry/observations/%i-{timestamp}.jsonl --manifest-output /var/lib/argand-site-registry/observations/%i-{timestamp}.source.json --maximum-body-bytes 2097152 --maximum-redirects 5 --timeout-seconds 20 --maximum-bytes-per-second 1048576 +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +PrivateDevices=true +ProtectSystem=strict +ProtectHome=true +ProtectClock=true +ProtectControlGroups=true +ProtectKernelLogs=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectProc=invisible +ProcSubset=pid +ReadWritePaths=/var/lib/argand-site-registry +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +RestrictNamespaces=true +RestrictRealtime=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true +CapabilityBoundingSet= +AmbientCapabilities= +SystemCallArchitectures=native +TimeoutStartSec=3min diff --git a/crates/argand-site-registry/examples/argand-site-registry-observe@.timer b/crates/argand-site-registry/examples/argand-site-registry-observe@.timer new file mode 100644 index 0000000..68d38ea --- /dev/null +++ b/crates/argand-site-registry/examples/argand-site-registry-observe@.timer @@ -0,0 +1,12 @@ +# By Nic Weyand! Enable only for explicitly reviewed candidate fingerprints. +[Unit] +Description=Refresh Argand Site Registry observation %i daily + +[Timer] +OnCalendar=daily +RandomizedDelaySec=2h +Persistent=true +Unit=argand-site-registry-observe@%i.service + +[Install] +WantedBy=timers.target diff --git a/crates/argand-site-registry/examples/observer.env b/crates/argand-site-registry/examples/observer.env new file mode 100644 index 0000000..19ccbb6 --- /dev/null +++ b/crates/argand-site-registry/examples/observer.env @@ -0,0 +1,3 @@ +# Refresh these public candidate coordinates before each scheduled run. +ARGAND_REGISTRY_GENERATION=/var/lib/argand-site-registry/generations/candidate-RECEIPT_PIN +ARGAND_REGISTRY_PIN=RECEIPT_PIN diff --git a/crates/argand-site-registry/examples/update.toml b/crates/argand-site-registry/examples/update.toml index 94a31cf..881b918 100644 --- a/crates/argand-site-registry/examples/update.toml +++ b/crates/argand-site-registry/examples/update.toml @@ -2,6 +2,10 @@ cache = "/var/lib/argand-site-registry/cache" database = "/var/lib/argand-site-registry/import.sqlite" generations = "/var/lib/argand-site-registry/generations" +reviewer_trust = "/etc/argand-site-registry/reviewer-allowed-signers" +# This authorizes each scheduled full/partition download to supersede the exact +# current frontier for the same provider, collection and partition coordinate. +auto_supersede_typed_snapshots = true [[downloads]] source = "psl" @@ -10,6 +14,10 @@ url = "https://publicsuffix.org/list/public_suffix_list.dat" snapshot = "{date}" scope = "full" maximum_bytes = 1000000 +[downloads.coverage] +collection = "default" +kind = "full" +supersedes = [] [[downloads]] source = "wikidata" @@ -18,6 +26,11 @@ url = "https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q355%7CQ3884& snapshot = "{date}" scope = "selection:facebook-amazon" maximum_bytes = 5000000 +[downloads.coverage] +collection = "entity-selections" +kind = "partition" +partition = "facebook-amazon" +supersedes = [] [[downloads]] source = "majestic" @@ -26,6 +39,10 @@ url = "https://downloads.majestic.com/majestic_million.csv" snapshot = "{date}" scope = "full" maximum_bytes = 250000000 +[downloads.coverage] +collection = "default" +kind = "full" +supersedes = [] [[downloads]] source = "curlie" @@ -34,6 +51,10 @@ url = "https://curlie.org/directory-dl" snapshot = "{date}" scope = "full" maximum_bytes = 1000000000 +[downloads.coverage] +collection = "default" +kind = "full" +supersedes = [] # Optional pinned acquisitions; repeat [[inputs]] for each source. # [[inputs]] @@ -48,3 +69,8 @@ maximum_bytes = 1000000000 # country = "GB" # maximum_bytes_billed = 1000000000 # maximum_output_bytes = 500000000 +# [crux.coverage] +# collection = "monthly-origins" +# kind = "partition" +# partition = "GB" +# supersedes = [] diff --git a/crates/argand-site-registry/migrations/004.sql b/crates/argand-site-registry/migrations/004.sql new file mode 100644 index 0000000..cfae63d --- /dev/null +++ b/crates/argand-site-registry/migrations/004.sql @@ -0,0 +1,37 @@ +-- By Nic Weyand! ADR 0002/0003: granular accepted-time, authenticated reviewer votes. +CREATE TABLE votes ( + sequence INTEGER PRIMARY KEY, + id TEXT NOT NULL UNIQUE, + fingerprint TEXT NOT NULL, + subject_kind TEXT NOT NULL CHECK(subject_kind IN ('name','edge','equivalence')), + decision TEXT NOT NULL CHECK(decision IN ('approve','revoke')), + reviewer TEXT NOT NULL, + reason TEXT NOT NULL, + evidence_bundle TEXT NOT NULL, + policy_sha256 TEXT NOT NULL, + reviewed_at TEXT NOT NULL, + expires_at TEXT, + role TEXT NOT NULL, + locale TEXT NOT NULL, + country TEXT NOT NULL, + supersedes_json TEXT NOT NULL, + accepted_at TEXT NOT NULL, + document_json BLOB NOT NULL +) STRICT; +CREATE INDEX vote_subject ON votes(subject_kind,fingerprint,sequence); +CREATE INDEX vote_reviewer ON votes(reviewer,sequence); +CREATE TABLE vote_auth ( + sequence INTEGER PRIMARY KEY REFERENCES votes(sequence), + signer TEXT NOT NULL, + signature_sha256 TEXT NOT NULL, + namespace TEXT NOT NULL CHECK(namespace='argand-site-registry-vote'), + decision_sha256 TEXT NOT NULL, + key_sha256 TEXT NOT NULL, + signature BLOB NOT NULL +) STRICT; +CREATE TRIGGER vote_no_update BEFORE UPDATE ON votes BEGIN SELECT RAISE(ABORT,'votes are append-only'); END; +CREATE TRIGGER vote_no_delete BEFORE DELETE ON votes BEGIN SELECT RAISE(ABORT,'votes are append-only'); END; +CREATE TRIGGER vote_auth_no_update BEFORE UPDATE ON vote_auth BEGIN SELECT RAISE(ABORT,'vote authentication is immutable'); END; +CREATE TRIGGER vote_auth_no_delete BEFORE DELETE ON vote_auth BEGIN SELECT RAISE(ABORT,'vote authentication is immutable'); END; +UPDATE registry_metadata SET rules='argand.site-rules/v4' WHERE singleton=1; +PRAGMA user_version=4; diff --git a/crates/argand-site-registry/migrations/005.sql b/crates/argand-site-registry/migrations/005.sql new file mode 100644 index 0000000..01f2b14 --- /dev/null +++ b/crates/argand-site-registry/migrations/005.sql @@ -0,0 +1,25 @@ +-- By Nic Weyand! ADR 0002: store immutable, subject-bound observation batches. +CREATE TABLE observation_batches ( + id TEXT PRIMARY KEY, + source TEXT NOT NULL, + retrieved_at TEXT NOT NULL, + manifest_json TEXT NOT NULL, + records INTEGER NOT NULL DEFAULT 0 CHECK(records>=0), + complete INTEGER NOT NULL DEFAULT 0 CHECK(complete IN (0,1)) +) STRICT; +CREATE TABLE observations ( + fingerprint TEXT PRIMARY KEY, + batch_id TEXT NOT NULL REFERENCES observation_batches(id), + subject_kind TEXT NOT NULL CHECK(subject_kind IN ('name','edge','equivalence')), + subject_fingerprint TEXT NOT NULL, + document_json TEXT NOT NULL +) STRICT; +CREATE INDEX observation_subject ON observations(subject_kind,subject_fingerprint,fingerprint); +CREATE TRIGGER observation_batch_open_insert BEFORE INSERT ON observation_batches WHEN NEW.records<>0 OR NEW.complete<>0 BEGIN SELECT RAISE(ABORT,'observation batch must be created open'); END; +CREATE TRIGGER observation_batch_no_update BEFORE UPDATE OF id,source,retrieved_at,manifest_json ON observation_batches BEGIN SELECT RAISE(ABORT,'observation batch identity is immutable'); END; +CREATE TRIGGER observation_batch_finish_once BEFORE UPDATE OF records,complete ON observation_batches WHEN OLD.complete<>0 OR NEW.complete<>1 OR NEW.records<=0 OR NEW.records<>(SELECT count(*) FROM observations WHERE batch_id=OLD.id) BEGIN SELECT RAISE(ABORT,'observation batch can complete only once with its exact record count'); END; +CREATE TRIGGER observation_batch_no_delete BEFORE DELETE ON observation_batches BEGIN SELECT RAISE(ABORT,'observation batches are immutable'); END; +CREATE TRIGGER observation_insert_open BEFORE INSERT ON observations WHEN NOT EXISTS(SELECT 1 FROM observation_batches WHERE id=NEW.batch_id AND complete=0) BEGIN SELECT RAISE(ABORT,'observations require an open batch'); END; +CREATE TRIGGER observation_no_update BEFORE UPDATE ON observations BEGIN SELECT RAISE(ABORT,'observations are immutable'); END; +CREATE TRIGGER observation_no_delete BEFORE DELETE ON observations BEGIN SELECT RAISE(ABORT,'observations are immutable'); END; +PRAGMA user_version=5; diff --git a/crates/argand-site-registry/src/build.rs b/crates/argand-site-registry/src/build.rs index 8f1d1ad..78475e3 100644 --- a/crates/argand-site-registry/src/build.rs +++ b/crates/argand-site-registry/src/build.rs @@ -16,11 +16,9 @@ use std::{ }; const PROJECTIONS: &str = " -CREATE TABLE selected_sources(id TEXT PRIMARY KEY REFERENCES sources(id)) STRICT; -INSERT INTO selected_sources SELECT id FROM ( - SELECT id,row_number() OVER(PARTITION BY source,scope ORDER BY retrieved_at DESC,id DESC) AS position - FROM sources WHERE complete=1) WHERE position=1; -CREATE TABLE names(entity TEXT NOT NULL,key TEXT NOT NULL,text TEXT NOT NULL,language TEXT NOT NULL,kind TEXT NOT NULL,fact TEXT NOT NULL REFERENCES facts(id),PRIMARY KEY(entity,fact)) STRICT; +CREATE TABLE selected_sources(id TEXT PRIMARY KEY REFERENCES sources(id),precedence INTEGER NOT NULL CHECK(precedence>=0),partition TEXT NOT NULL) STRICT; +CREATE TABLE active_records(source_id TEXT NOT NULL,ordinal INTEGER NOT NULL,PRIMARY KEY(source_id,ordinal),FOREIGN KEY(source_id,ordinal) REFERENCES records(source_id,ordinal)) STRICT; +CREATE TABLE names(entity TEXT NOT NULL,key TEXT NOT NULL,text TEXT NOT NULL,language TEXT NOT NULL,kind TEXT NOT NULL,fact TEXT NOT NULL REFERENCES facts(id),fingerprint TEXT NOT NULL UNIQUE,PRIMARY KEY(entity,fact)) STRICT; CREATE INDEX name_lookup ON names(key,entity); CREATE TABLE entities(id TEXT PRIMARY KEY,canonical_name TEXT NOT NULL,names_fingerprint TEXT NOT NULL) STRICT; CREATE TABLE properties(id TEXT PRIMARY KEY,url TEXT NOT NULL UNIQUE,hostname TEXT NOT NULL,domain TEXT NOT NULL,suffix TEXT NOT NULL,derived_json TEXT NOT NULL) STRICT; @@ -30,13 +28,14 @@ CREATE INDEX edge_entity ON edges(entity,property); CREATE TABLE popularity(fact TEXT PRIMARY KEY REFERENCES facts(id),source TEXT NOT NULL,target TEXT NOT NULL,hostname TEXT NOT NULL,domain TEXT NOT NULL,value TEXT NOT NULL,derived_json TEXT NOT NULL) STRICT; CREATE INDEX popularity_host ON popularity(hostname,source); CREATE TABLE rejected(fact TEXT PRIMARY KEY REFERENCES facts(id),reason TEXT NOT NULL) STRICT; +CREATE TABLE review_policy(singleton INTEGER PRIMARY KEY CHECK(singleton=1),id TEXT NOT NULL,document TEXT NOT NULL) STRICT; "; /// Completion receipt. Authenticity needs an external digest or trusted signature. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Receipt { - /// `argand.site-registry/v1`. + /// `argand.site-registry/v2`. pub schema: String, /// Parser/derivation contract. pub rules: String, @@ -50,6 +49,21 @@ pub struct Receipt { pub psl_source: String, /// Active source snapshot declarations. pub sources: Vec, + /// Exact policy compiled by consumers. + #[serde(default)] + pub review_policy: crate::policy::ReviewPolicy, + /// Digest of the canonical review-policy JSON. + #[serde(default)] + pub review_policy_sha256: String, + /// Digest of exact allowed-reviewer file bytes for strict policies. + #[serde(default)] + pub reviewer_trust_sha256: String, + /// Digest of the complete selected source coverage graph. + #[serde(default)] + pub coverage_sha256: String, + /// Trusted writer-acceptance and bounded-expiry contract. + #[serde(default)] + pub decision_time_policy: String, /// Distinct entity count. pub entities: u64, /// Strict URL identity count. @@ -65,13 +79,55 @@ pub struct Receipt { /// # Errors /// Rejects existing destinations, incomplete PSL, corrupt stores, and I/O failures. pub fn build(db: &Connection, output: &Path) -> anyhow::Result { + build_with_policy(db, output, &crate::policy::ReviewPolicy::reference()) +} + +/// Builds a generation under an explicit, receipt-authenticated review policy. +/// +/// # Errors +/// Rejects invalid policy, existing destinations, corrupt stores, and I/O failures. +pub fn build_with_policy( + db: &Connection, + output: &Path, + policy: &crate::policy::ReviewPolicy, +) -> anyhow::Result { + build_with_policy_and_trust(db, output, policy, None) +} + +/// Builds with an external reviewer trust root bound into the immutable receipt. +/// +/// # Errors +/// Rejects strict policies without trust roots, malformed roots, invalid policy, +/// existing destinations, corrupt stores, and I/O failures. +pub fn build_with_policy_and_trust( + db: &Connection, + output: &Path, + policy: &crate::policy::ReviewPolicy, + reviewer_trust: Option<&Path>, +) -> anyhow::Result { + policy.validate()?; + let reviewer_trust_sha256 = reviewer_trust + .map(|path| crate::ssh::sealed_input(path, 1024 * 1024)) + .transpose()? + .map(|input| crate::digest(&input.bytes)) + .unwrap_or_default(); + ensure!( + policy.allow_legacy_reviews || crate::model::valid_digest(&reviewer_trust_sha256), + "strict review policy requires an exact reviewer trust root" + ); fs::create_dir(output).context("generation destination must not exist")?; let path = output.join("registry.sqlite"); let snapshot = store::open(&path)?; copy_canonical(db, &snapshot)?; snapshot.execute_batch(PROJECTIONS)?; - let (psl_id,text): (String,String)=snapshot.query_row("SELECT f.source_id,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl' ORDER BY f.source_id LIMIT 1",[],|r| Ok((r.get(0)?,r.get(1)?))).context("import a complete PSL snapshot first")?; - let psl_count:u64=snapshot.query_row("SELECT count(*) FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl'",[],|r|store::unsigned(r,0))?; + crate::coverage::project(&snapshot)?; + let policy_id = policy.id()?; + snapshot.execute( + "INSERT INTO review_policy VALUES(1,?1,?2)", + params![policy_id, serde_json::to_string(policy)?], + )?; + let (psl_id,text): (String,String)=snapshot.query_row("SELECT f.source_id,f.value FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.predicate='psl' ORDER BY f.source_id LIMIT 1",[],|r| Ok((r.get(0)?,r.get(1)?))).context("import a complete PSL snapshot first")?; + let psl_count:u64=snapshot.query_row("SELECT count(*) FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.predicate='psl'",[],|r|store::unsigned(r,0))?; ensure!(psl_count == 1, "exactly one active PSL snapshot required"); let psl_text: String = serde_json::from_str(&text)?; let normalizer = Normalizer::new(psl_text.as_bytes(), psl_id.clone())?; @@ -93,8 +149,9 @@ pub fn build(db: &Connection, output: &Path) -> anyhow::Result { .map(|s| Ok(serde_json::from_str(&s?)?)) .collect::>>()?; drop(statement); + let coverage_sha256 = crate::coverage::projection_digest(&snapshot)?; let mut receipt = Receipt { - schema: "argand.site-registry/v1".into(), + schema: "argand.site-registry/v2".into(), rules: store::RULE_VERSION.into(), database_sha256: String::new(), licenses_sha256: crate::digest(crate::release::LICENSES.as_bytes()), @@ -103,6 +160,11 @@ pub fn build(db: &Connection, output: &Path) -> anyhow::Result { )?), psl_source: psl_id, sources, + review_policy: policy.clone(), + review_policy_sha256: policy_id, + reviewer_trust_sha256, + coverage_sha256, + decision_time_policy: "argand.site-decision-time/v1".into(), entities: count(&snapshot, "entities")?, properties: count(&snapshot, "properties")?, edges: count(&snapshot, "edges")?, @@ -158,6 +220,8 @@ fn copy_canonical(source: &Connection, destination: &Connection) -> anyhow::Resu "SELECT * FROM equivalences ORDER BY fingerprint", 5, ), + ("votes", "SELECT * FROM votes ORDER BY sequence", 17), + ("vote_auth", "SELECT * FROM vote_auth ORDER BY sequence", 7), ]; for (table, select, columns) in tables { let placeholders = (1..=columns) @@ -175,6 +239,46 @@ fn copy_canonical(source: &Connection, destination: &Connection) -> anyhow::Resu insert.execute(rusqlite::params_from_iter(values))?; } } + let mut batch_rows = source.prepare( + "SELECT id,source,retrieved_at,manifest_json,records FROM observation_batches WHERE complete=1 ORDER BY id", + )?; + let completed_batches = batch_rows + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, i64>(4)?, + )) + })? + .collect::, _>>()?; + for (id, provider, retrieved_at, manifest, _) in &completed_batches { + destination.execute( + "INSERT INTO observation_batches(id,source,retrieved_at,manifest_json) VALUES(?1,?2,?3,?4)", + params![id, provider, retrieved_at, manifest], + )?; + } + let mut observation_rows = source.prepare( + "SELECT o.* FROM observations o JOIN observation_batches b ON b.id=o.batch_id WHERE b.complete=1 ORDER BY o.fingerprint", + )?; + let mut rows = observation_rows.query([])?; + let mut insert = destination.prepare("INSERT INTO observations VALUES(?1,?2,?3,?4,?5)")?; + while let Some(row) = rows.next()? { + let values = (0..5) + .map(|index| row.get::<_, rusqlite::types::Value>(index)) + .collect::, _>>()?; + insert.execute(rusqlite::params_from_iter(values))?; + } + drop(insert); + drop(rows); + drop(observation_rows); + for (id, _, _, _, records) in completed_batches { + destination.execute( + "UPDATE observation_batches SET records=?2,complete=1 WHERE id=?1", + params![id, records], + )?; + } destination_transaction.commit()?; source_transaction.commit()?; Ok(()) @@ -189,7 +293,7 @@ fn count(db: &Connection, table: &str) -> anyhow::Result { } fn project_names(db: &Connection) -> anyhow::Result<()> { - let mut stmt=db.prepare("SELECT f.id,f.subject,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='name' ORDER BY f.subject,f.id")?; + let mut stmt=db.prepare("SELECT f.id,f.subject,f.value,s.source,r.native_id,f.selector FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal JOIN sources s ON s.id=f.source_id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE f.predicate='name' ORDER BY f.subject,f.id")?; let mut rows = stmt.query([])?; while let Some(row) = rows.next()? { let (id, subject, raw): (String, String, String) = (row.get(0)?, row.get(1)?, row.get(2)?); @@ -197,16 +301,22 @@ fn project_names(db: &Connection) -> anyhow::Result<()> { let text = value["text"].as_str().context("name text missing")?; match name_key(text) { Ok(key) => { + let language = value["language"].as_str().unwrap_or("und"); + let kind = value["kind"].as_str().unwrap_or("label"); + let fingerprint = crate::digest(&serde_json::to_vec(&( + "argand.site-name/v1", + &subject, + &key, + text, + language, + kind, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + ))?); db.execute( - "INSERT INTO names VALUES(?1,?2,?3,?4,?5,?6)", - params![ - subject, - key, - text, - value["language"].as_str().unwrap_or("und"), - value["kind"].as_str().unwrap_or("label"), - id - ], + "INSERT INTO names VALUES(?1,?2,?3,?4,?5,?6,?7)", + params![subject, key, text, language, kind, id, fingerprint], )?; } Err(error) => { @@ -218,7 +328,7 @@ fn project_names(db: &Connection) -> anyhow::Result<()> { } } // All entities with website assertions exist even when names are absent. - let mut entities=db.prepare("SELECT DISTINCT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate IN('website','name') ORDER BY f.subject")?; + let mut entities=db.prepare("SELECT DISTINCT f.subject FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.predicate IN('website','name') ORDER BY f.subject")?; for subject in entities.query_map([], |r| r.get::<_, String>(0))? { let subject = subject?; let mut names=db.prepare("SELECT DISTINCT text,language,kind FROM names WHERE entity=?1 ORDER BY CASE WHEN kind='label' THEN 0 ELSE 1 END,CASE WHEN language='en' THEN 0 ELSE 1 END,language,text")?; @@ -242,7 +352,7 @@ fn project_names(db: &Connection) -> anyhow::Result<()> { } fn project_facts(db: &Connection, normalizer: &Normalizer) -> anyhow::Result<()> { - let mut stmt=db.prepare("SELECT f.id,f.subject,f.predicate,f.value,s.source,f.selector,r.native_id FROM facts f JOIN sources s ON s.id=f.source_id JOIN selected_sources a ON a.id=s.id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE f.predicate IN('website','popularity') ORDER BY f.subject,f.id")?; + let mut stmt=db.prepare("SELECT f.id,f.subject,f.predicate,f.value,s.source,f.selector,r.native_id FROM facts f JOIN sources s ON s.id=f.source_id JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE f.predicate IN('website','popularity') ORDER BY f.subject,f.id")?; let mut rows = stmt.query([])?; while let Some(row) = rows.next()? { let id: String = row.get(0)?; @@ -335,17 +445,12 @@ fn project_edge( serde_json::to_string(&property)? ], )?; - let names: String = db.query_row( - "SELECT names_fingerprint FROM entities WHERE id=?1", - [entity], - |r| r.get(0), - )?; let relation = if source == "wikidata" { "asserted_official" } else { "directory_listing" }; - let evidence = json!({"source":source,"native_id":native,"selector":selector,"assertion":value,"names_fingerprint":names,"normalization":store::RULE_VERSION}); + let evidence = json!({"source":source,"native_id":native,"selector":selector,"assertion":value,"normalization":store::RULE_VERSION}); let mut identity_property = property.clone(); // Bind reviews to the normalization result. The complete PSL identity remains // on the property, while comment-only or unrelated rule changes do not force @@ -353,9 +458,13 @@ fn project_edge( identity_property.domain.psl_source.clear(); identity_property.domain.psl_sha256.clear(); let fingerprint = crate::digest(&serde_json::to_vec(&( + "argand.site-edge/v2", entity, &identity_property, - &evidence, + source, + native, + selector, + material_website_assertion(value), ))?); // End-dated assertions stay as historical evidence, never current destinations. // Future/partial starts require the operator to inspect the retained qualifiers. @@ -364,3 +473,14 @@ fn project_edge( db.execute("INSERT INTO edges VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(fingerprint) DO UPDATE SET facts=json_insert(edges.facts,'$[#]',?8)",params![fingerprint,entity,property.id,relation,serde_json::to_string(&vec![id])?,serde_json::to_string(&evidence)?,eligible,id])?; Ok(()) } + +fn material_website_assertion(value: &Value) -> Value { + let mut material = value.clone(); + if let Some(object) = material.as_object_mut() { + object.remove("revision"); + if let Some(statement) = object.get_mut("statement").and_then(Value::as_object_mut) { + statement.remove("references"); + } + } + material +} diff --git a/crates/argand-site-registry/src/bundle.rs b/crates/argand-site-registry/src/bundle.rs new file mode 100644 index 0000000..814e745 --- /dev/null +++ b/crates/argand-site-registry/src/bundle.rs @@ -0,0 +1,248 @@ +// By Nic Weyand! +//! Deterministic, bounded evidence bundles referenced by reviewer votes. + +use crate::{policy::SubjectKind, query::Registry}; +use anyhow::{Context, ensure}; +use rusqlite::OptionalExtension; +use serde::Serialize; +use serde_json::{Value, json}; + +const MAXIMUM_BUNDLE_BYTES: usize = 16 * 1024 * 1024; + +/// Exact evidence presented for one granular review subject. +#[derive(Clone, Debug, Serialize)] +pub struct EvidenceBundle { + /// `argand.site-evidence-bundle/v1`. + pub schema: String, + /// Content digest over the remaining fields. + pub id: String, + /// Granular assertion type. + pub subject_kind: SubjectKind, + /// Stable material assertion fingerprint. + pub fingerprint: String, + /// Complete current source evidence needed for the decision. + pub evidence: Value, + /// Current crawler observations; empty until an observation batch is imported. + pub observations: Vec, +} + +impl EvidenceBundle { + fn new( + subject_kind: SubjectKind, + fingerprint: &str, + evidence: Value, + observations: Vec, + ) -> anyhow::Result { + ensure!( + crate::model::valid_digest(fingerprint), + "invalid evidence subject fingerprint" + ); + let schema = "argand.site-evidence-bundle/v1".to_owned(); + let id = crate::digest(&serde_json::to_vec(&( + &schema, + subject_kind, + fingerprint, + &evidence, + &observations, + ))?); + let bundle = Self { + schema, + id, + subject_kind, + fingerprint: fingerprint.into(), + evidence, + observations, + }; + ensure!( + serde_json::to_vec(&bundle)?.len() <= MAXIMUM_BUNDLE_BYTES, + "evidence bundle exceeds 16 MiB" + ); + Ok(bundle) + } +} + +/// Builds current evidence for a projected name or website edge. +/// +/// # Errors +/// Rejects missing/mismatched subjects, oversized evidence, or corrupt registry data. +pub fn build( + registry: &Registry, + subject_kind: SubjectKind, + fingerprint: &str, +) -> anyhow::Result { + match subject_kind { + SubjectKind::Name => name(registry, fingerprint), + SubjectKind::Edge => edge(registry, fingerprint), + SubjectKind::Equivalence => { + anyhow::bail!("equivalence evidence needs the exact proposed entity pair") + } + } +} + +fn name(registry: &Registry, fingerprint: &str) -> anyhow::Result { + let row: Option<(String, String, String, String, String, String)> = registry + .db + .query_row( + "SELECT entity,key,text,language,kind,fact FROM names WHERE fingerprint=?1", + [fingerprint], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, + ) + .optional()?; + let (entity, key, text, language, kind, fact) = row.context("name fingerprint is absent")?; + let conflicts = name_conflicts(registry, &key, fingerprint)?; + EvidenceBundle::new( + SubjectKind::Name, + fingerprint, + json!({"entity":entity,"key":key,"text":text,"language":language,"kind":kind,"provenance":crate::evidence::fact(®istry.db,&fact)?,"conflicts":conflicts}), + observations(registry, fingerprint)?, + ) +} + +fn edge(registry: &Registry, fingerprint: &str) -> anyhow::Result { + let row: Option<(String, String, String, String, bool)> = registry + .db + .query_row( + "SELECT e.entity,p.derived_json,e.relation,e.evidence,e.eligible FROM edges e JOIN properties p ON p.id=e.property WHERE e.fingerprint=?1", + [fingerprint], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + ) + .optional()?; + let (entity, property, relation, evidence, eligible) = + row.context("edge fingerprint is absent")?; + let facts: String = registry.db.query_row( + "SELECT facts FROM edges WHERE fingerprint=?1", + [fingerprint], + |row| row.get(0), + )?; + let mut provenance = Vec::new(); + for fact in serde_json::from_str::>(&facts)? { + provenance.push(crate::evidence::fact(®istry.db, &fact)?); + } + let property_value = serde_json::from_str::(&property)?; + let domain = property_value + .pointer("/domain/registrable_domain") + .and_then(Value::as_str) + .context("edge property has no registrable domain")?; + let conflicts = edge_conflicts(registry, domain, fingerprint)?; + let drift = crate::queue::drift(registry, fingerprint)?; + EvidenceBundle::new( + SubjectKind::Edge, + fingerprint, + json!({"entity":entity,"web_property":property_value,"relation":relation,"eligible":eligible,"assertion":serde_json::from_str::(&evidence)?,"provenance":provenance,"conflicts":conflicts,"drift":drift}), + observations(registry, fingerprint)?, + ) +} + +fn name_conflicts(registry: &Registry, key: &str, fingerprint: &str) -> anyhow::Result { + let total: u64 = registry.db.query_row( + "SELECT count(*) FROM names WHERE key=?1 AND fingerprint<>?2", + rusqlite::params![key, fingerprint], + |row| crate::store::unsigned(row, 0), + )?; + let mut statement = registry.db.prepare( + "SELECT fingerprint,entity,text,language,kind,fact FROM names WHERE key=?1 AND fingerprint<>?2 ORDER BY fingerprint LIMIT 256", + )?; + let conflicts = statement + .query_map(rusqlite::params![key, fingerprint], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + )) + })? + .map(|row| { + let (fingerprint, entity, text, language, kind, fact) = row?; + Ok(json!({"fingerprint":fingerprint,"entity":entity,"text":text,"language":language,"kind":kind,"provenance":crate::evidence::fact(®istry.db,&fact)?})) + }) + .collect::>>()?; + Ok(json!({"total":total,"truncated":total>256,"items":conflicts})) +} + +fn edge_conflicts(registry: &Registry, domain: &str, fingerprint: &str) -> anyhow::Result { + let from = + "FROM edges e JOIN properties p ON p.id=e.property WHERE p.domain=?1 AND e.fingerprint<>?2"; + let total: u64 = registry.db.query_row( + &format!("SELECT count(*) {from}"), + rusqlite::params![domain, fingerprint], + |row| crate::store::unsigned(row, 0), + )?; + let mut statement = registry.db.prepare(&format!( + "SELECT e.fingerprint,e.entity,p.url,e.relation,e.facts,e.evidence {from} ORDER BY e.fingerprint LIMIT 256" + ))?; + let conflicts = statement + .query_map(rusqlite::params![domain, fingerprint], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + )) + })? + .map(|row| { + let (fingerprint, entity, url, relation, facts, assertion) = row?; + Ok(json!({"fingerprint":fingerprint,"entity":entity,"url":url,"relation":relation,"facts":serde_json::from_str::(&facts)?,"assertion":serde_json::from_str::(&assertion)?})) + }) + .collect::>>()?; + Ok(json!({"total":total,"truncated":total>256,"items":conflicts})) +} + +/// Builds current evidence for an exact proposed equivalence. +/// +/// # Errors +/// Rejects a stale proposal or oversized/corrupt evidence. +pub fn equivalence( + registry: &Registry, + pair: &crate::identity::Equivalence, +) -> anyhow::Result { + let current = crate::identity::propose(registry, &pair.entities[0], &pair.entities[1])?; + ensure!( + current.fingerprint == pair.fingerprint, + "equivalence evidence is stale" + ); + EvidenceBundle::new( + SubjectKind::Equivalence, + &pair.fingerprint, + serde_json::to_value(current)?, + observations(registry, &pair.fingerprint)?, + ) +} + +fn observations(registry: &Registry, fingerprint: &str) -> anyhow::Result> { + let exists: bool = registry.db.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='observations')", + [], + |row| row.get(0), + )?; + if !exists { + return Ok(Vec::new()); + } + let mut statement = registry.db.prepare( + "SELECT document_json FROM observations WHERE subject_fingerprint=?1 ORDER BY fingerprint", + )?; + statement + .query_map([fingerprint], |row| row.get::<_, String>(0))? + .map(|row| Ok(serde_json::from_str(&row?)?)) + .collect() +} diff --git a/crates/argand-site-registry/src/catalog.rs b/crates/argand-site-registry/src/catalog.rs index 184b986..0847e0d 100644 --- a/crates/argand-site-registry/src/catalog.rs +++ b/crates/argand-site-registry/src/catalog.rs @@ -24,6 +24,14 @@ pub struct RegistryStats { pub source_snapshots: u64, /// Active source selections used for projections. pub selected_sources: u64, + /// Complete snapshots retained for audit but excluded from active projections. + pub superseded_source_snapshots: u64, + /// Incomplete snapshots; immutable generations always report zero. + pub incomplete_source_snapshots: u64, + /// Conflicting snapshots; successful immutable generations always report zero. + pub conflicting_source_snapshots: u64, + /// Active source records after delta masking. + pub active_records: u64, /// Retained source records. pub records: u64, /// Retained source facts. @@ -32,10 +40,26 @@ pub struct RegistryStats { pub reviews: u64, /// Decisions carrying verified reviewer authentication. pub authenticated_reviews: u64, + /// Immutable v0.4 votes. + pub votes: u64, + /// Votes carrying retained authentication. + pub authenticated_votes: u64, + /// Sticky revocation votes retained in the generation. + pub vote_revocations: u64, /// Explicit cross-source identity proposals. pub equivalences: u64, /// Current approvals expiring during the next seven days. pub approvals_expiring_within_seven_days: u64, + /// v0.4 approval votes requesting expiry during the next seven days. + pub vote_approvals_expiring_within_seven_days: u64, + /// Complete immutable observation batches. + pub observation_batches: u64, + /// Subject-bound observations. + pub observations: u64, + /// Receipt-authenticated review policy digest. + pub review_policy_sha256: String, + /// Receipt-authenticated source coverage selection digest. + pub coverage_sha256: String, /// Receipt-level entity count. pub entities: u64, /// Receipt-level strict URL count. @@ -163,18 +187,45 @@ impl Registry { params![now.to_rfc3339(), deadline.to_rfc3339()], |row| crate::store::unsigned(row, 0), )?; + let vote_approvals_expiring_within_seven_days = self.db.query_row( + "SELECT count(*) FROM votes WHERE decision='approve' AND expires_at>?1 AND expires_at<=?2", + params![now.to_rfc3339(), deadline.to_rfc3339()], + |row| crate::store::unsigned(row, 0), + )?; + let source_snapshots = count("sources")?; + let selected_sources = count("selected_sources")?; Ok(RegistryStats { registry: self.identity.clone(), rules: self.receipt.rules.clone(), database_bytes: page_count.saturating_mul(page_size), - source_snapshots: count("sources")?, - selected_sources: count("selected_sources")?, + source_snapshots, + selected_sources, + superseded_source_snapshots: source_snapshots.saturating_sub(selected_sources), + incomplete_source_snapshots: 0, + conflicting_source_snapshots: 0, + active_records: count("active_records")?, records: count("records")?, facts: count("facts")?, reviews: count("reviews")?, authenticated_reviews: count("review_auth")?, + votes: count("votes")?, + authenticated_votes: count("vote_auth")?, + vote_revocations: self.db.query_row( + "SELECT count(*) FROM votes WHERE decision='revoke'", + [], + |row| crate::store::unsigned(row, 0), + )?, equivalences: count("equivalences")?, approvals_expiring_within_seven_days, + vote_approvals_expiring_within_seven_days, + observation_batches: self.db.query_row( + "SELECT count(*) FROM observation_batches WHERE complete=1", + [], + |row| crate::store::unsigned(row, 0), + )?, + observations: count("observations")?, + review_policy_sha256: self.receipt.review_policy_sha256.clone(), + coverage_sha256: self.receipt.coverage_sha256.clone(), entities: self.receipt.entities, properties: self.receipt.properties, edges: self.receipt.edges, @@ -350,7 +401,7 @@ impl Registry { && category_id.bytes().all(|byte| byte.is_ascii_digit()), "invalid category ID" ); - let mut statement = self.db.prepare("SELECT f.id FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category' AND json_extract(f.value,'$.category_id')=?1 ORDER BY f.id")?; + let mut statement = self.db.prepare("SELECT f.id FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.predicate='category' AND json_extract(f.value,'$.category_id')=?1 ORDER BY f.id")?; let mut metadata = Vec::new(); let mut output_bytes = 0; for id in statement.query_map([category_id], |row| row.get::<_, String>(0))? { @@ -362,8 +413,8 @@ impl Registry { crate::query::account_output(&mut output_bytes, &fact)?; metadata.push(fact); } - let total_members = self.db.query_row("SELECT count(*) FROM edges e WHERE e.entity IN(SELECT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category_membership' AND json_extract(f.value,'$.category_id')=?1)",[category_id],|row|crate::store::unsigned(row,0))?; - let members = self.candidates_for("SELECT e.fingerprint FROM edges e WHERE e.entity IN(SELECT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category_membership' AND json_extract(f.value,'$.category_id')=?1) ORDER BY e.entity,e.fingerprint LIMIT ?2",params![category_id,limit])?; + let total_members = self.db.query_row("SELECT count(*) FROM edges e WHERE e.entity IN(SELECT f.subject FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.predicate='category_membership' AND json_extract(f.value,'$.category_id')=?1)",[category_id],|row|crate::store::unsigned(row,0))?; + let members = self.candidates_for("SELECT e.fingerprint FROM edges e WHERE e.entity IN(SELECT f.subject FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.predicate='category_membership' AND json_extract(f.value,'$.category_id')=?1) ORDER BY e.entity,e.fingerprint LIMIT ?2",params![category_id,limit])?; Ok(CategoryLookup { category_id: category_id.into(), metadata, @@ -374,8 +425,8 @@ impl Registry { }) } - fn normalizer(&self) -> anyhow::Result { - let (source, raw): (String, String) = self.db.query_row("SELECT f.source_id,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl' ORDER BY f.source_id LIMIT 1",[],|row|Ok((row.get(0)?,row.get(1)?))).context("generation has no PSL")?; + pub(crate) fn normalizer(&self) -> anyhow::Result { + let (source, raw): (String, String) = self.db.query_row("SELECT f.source_id,f.value FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.predicate='psl' ORDER BY f.source_id LIMIT 1",[],|row|Ok((row.get(0)?,row.get(1)?))).context("generation has no PSL")?; let text: String = serde_json::from_str(&raw)?; Normalizer::new(text.as_bytes(), source) } diff --git a/crates/argand-site-registry/src/cli.rs b/crates/argand-site-registry/src/cli.rs index cea8e90..a40aaff 100644 --- a/crates/argand-site-registry/src/cli.rs +++ b/crates/argand-site-registry/src/cli.rs @@ -20,6 +20,21 @@ struct Args { command: Command, } +#[derive(Clone, Copy, clap::ValueEnum)] +enum DirectSubjectKind { + Name, + Edge, +} + +impl From for registry::policy::SubjectKind { + fn from(value: DirectSubjectKind) -> Self { + match value { + DirectSubjectKind::Name => Self::Name, + DirectSubjectKind::Edge => Self::Edge, + } + } +} + #[derive(Subcommand)] enum Command { /// Preview an explicit entity equivalence, or record its exact review JSON. @@ -43,6 +58,71 @@ enum Command { #[arg(long, requires = "decision")] identity: Option, }, + /// Record a v0.4 quorum vote for an exact entity equivalence. + EquivalenceVote { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + left: String, + #[arg(long)] + right: String, + #[arg(long)] + database: PathBuf, + #[arg(long)] + decision: PathBuf, + #[arg(long)] + signature: PathBuf, + #[arg(long)] + allowed_reviewers: PathBuf, + #[arg(long)] + identity: String, + }, + /// Prepare exact entity-equivalence vote JSON for detached SSH signing. + PrepareEquivalenceVote { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + left: String, + #[arg(long)] + right: String, + #[arg(long, value_enum)] + decision: registry::vote::VoteDecision, + #[arg(long)] + reviewer: String, + #[arg(long)] + reason: String, + #[arg(long)] + reviewed_at: chrono::DateTime, + #[arg(long)] + expires_at: Option>, + #[arg(long)] + supersedes: Vec, + #[arg(long)] + output: PathBuf, + }, + /// Verify exact signed entity-equivalence vote bytes without appending them. + VerifyEquivalenceVote { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + left: String, + #[arg(long)] + right: String, + #[arg(long)] + decision: PathBuf, + #[arg(long)] + signature: PathBuf, + #[arg(long)] + allowed_reviewers: PathBuf, + #[arg(long)] + identity: String, + }, /// Download one allowlisted source into an immutable local cache. Download { #[arg(long)] @@ -61,6 +141,9 @@ enum Command { scope: String, #[arg(long)] maximum_bytes: u64, + /// JSON file containing a typed coverage declaration. + #[arg(long)] + coverage: Option, }, /// Download paginated `CrUX` data using an explicit billing configuration JSON. CruxDownload { @@ -89,6 +172,9 @@ enum Command { scope: String, #[arg(long)] retrieved_at: chrono::DateTime, + /// JSON file containing a typed coverage declaration. + #[arg(long)] + coverage: Option, }, /// Import a complete pinned source, resuming committed record batches. Import { @@ -111,6 +197,12 @@ enum Command { database: PathBuf, #[arg(long)] output: PathBuf, + /// Explicit review policy JSON; strict Argand reference policy by default. + #[arg(long)] + policy: Option, + /// Exact SSH allowed-signers trust root; required by strict policies. + #[arg(long)] + reviewer_trust: Option, }, /// Audit an exact name or alias; includes ambiguity counts and attribution. Lookup { @@ -174,6 +266,37 @@ enum Command { #[arg(long)] pin: String, }, + /// Emit deterministic risk-ordered work under the generation policy. + ReviewQueue { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long, default_value_t = 100)] + limit: u32, + #[arg(long, default_value_t = 100_000)] + maximum_subjects: u32, + #[arg(long)] + at: Option>, + }, + /// Classify retained observations for one exact website assertion. + Drift { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + fingerprint: String, + }, + /// Export material observation failures as signed-revocation candidates. + RevocationCandidates { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long, default_value_t = 100_000)] + maximum_subjects: u32, + }, /// Replay bounded JSONL judgments and report accuracy and native latency. Evaluate { #[arg(long)] @@ -199,6 +322,20 @@ enum Command { locale: Option, #[arg(long)] country: Option, + /// Publisher-signed emergency feed applied before destination selection. + #[arg(long, requires_all = ["revocation_signature", "allowed_publishers", "publisher_identity"])] + revocations: Option, + #[arg(long, requires = "revocations")] + revocation_signature: Option, + #[arg(long, requires = "revocations")] + allowed_publishers: Option, + #[arg(long, requires = "revocations")] + publisher_identity: Option, + /// Last accepted feed, used to reject a replacement that drops revocations. + #[arg(long, requires_all = ["revocations", "previous_revocation_signature"])] + previous_revocations: Option, + #[arg(long, requires_all = ["revocations", "previous_revocations"])] + previous_revocation_signature: Option, }, /// Append an exact assertion approval or revocation from a review JSON file. Review { @@ -217,6 +354,168 @@ enum Command { #[arg(long)] identity: String, }, + /// Show the exact evidence bundle a reviewer vote must reference. + Evidence { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long, value_enum)] + subject_kind: DirectSubjectKind, + #[arg(long)] + fingerprint: String, + }, + /// Declare exact local JSONL observation bytes and rights metadata. + ObservationManifest { + #[arg(long)] + input: PathBuf, + #[arg(long)] + output: PathBuf, + #[arg(long)] + source: String, + #[arg(long)] + source_url: String, + #[arg(long)] + license: String, + #[arg(long)] + license_url: String, + #[arg(long)] + retrieved_at: chrono::DateTime, + }, + /// Import a complete pinned observation batch against a verified candidate. + ObservationImport { + #[arg(long)] + database: PathBuf, + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + input: PathBuf, + #[arg(long)] + manifest: PathBuf, + }, + /// Inspect observations attached to one exact review subject. + Observations { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + fingerprint: String, + #[arg(long, default_value_t = 100)] + limit: u32, + }, + /// Reverse lookup observations by exact URL, hostname, or registrable domain. + ObservationLookup { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + target: String, + #[arg(long, default_value_t = 100)] + limit: u32, + }, + /// Fetch one existing candidate through the bounded, DNS-pinned observer. + Observe { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + fingerprint: String, + #[arg(long)] + capture: PathBuf, + #[arg(long)] + output: PathBuf, + #[arg(long)] + manifest_output: PathBuf, + #[arg(long, default_value_t = 2 * 1024 * 1024)] + maximum_body_bytes: u64, + #[arg(long, default_value_t = 5)] + maximum_redirects: u8, + #[arg(long, default_value_t = 20)] + timeout_seconds: u64, + #[arg(long, default_value_t = 1024 * 1024)] + maximum_bytes_per_second: u64, + }, + /// Replay one immutable observer cache without network access. + ObserveReplay { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + capture: PathBuf, + #[arg(long)] + output: PathBuf, + #[arg(long)] + manifest_output: PathBuf, + }, + /// Append an authenticated v0.4 name or edge vote. + Vote { + #[arg(long)] + database: PathBuf, + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + decision: PathBuf, + #[arg(long)] + signature: PathBuf, + #[arg(long)] + allowed_reviewers: PathBuf, + #[arg(long)] + identity: String, + }, + /// Prepare exact name/edge vote JSON for detached SSH signing. + PrepareVote { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long, value_enum)] + subject_kind: DirectSubjectKind, + #[arg(long)] + fingerprint: String, + #[arg(long, value_enum)] + decision: registry::vote::VoteDecision, + #[arg(long)] + reviewer: String, + #[arg(long)] + reason: String, + #[arg(long)] + reviewed_at: chrono::DateTime, + #[arg(long)] + expires_at: Option>, + #[arg(long, default_value = "unspecified")] + role: String, + #[arg(long, default_value = "")] + locale: String, + #[arg(long, default_value = "")] + country: String, + #[arg(long)] + supersedes: Vec, + #[arg(long)] + output: PathBuf, + }, + /// Verify exact signed name/edge vote bytes without appending them. + VerifyVote { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + decision: PathBuf, + #[arg(long)] + signature: PathBuf, + #[arg(long)] + allowed_reviewers: PathBuf, + #[arg(long)] + identity: String, + }, /// Export facts as streaming JSONL with source licenses and attribution. Export { #[arg(long)] @@ -228,6 +527,66 @@ enum Command { #[arg(long)] include_descriptions: bool, }, + /// Export active and superseded facts with explicit selection state for audit. + ExportAudit { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + output: PathBuf, + #[arg(long)] + include_descriptions: bool, + }, + /// Export cumulative emergency revocation state at an explicit time. + ExportRevocations { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + effective_at: chrono::DateTime, + #[arg(long)] + output: PathBuf, + }, + /// Sign an exact revocation feed after checking it against its generation. + SignRevocations { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + input: PathBuf, + #[arg(long)] + output: PathBuf, + #[arg(long)] + key: PathBuf, + #[arg(long)] + allowed_reviewers: PathBuf, + #[arg(long)] + identity: String, + }, + /// Verify and compatibility-check a publisher-signed emergency feed. + VerifyRevocations { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + input: PathBuf, + #[arg(long)] + signature: PathBuf, + #[arg(long)] + allowed_publishers: PathBuf, + #[arg(long)] + identity: String, + #[arg(long, requires = "previous_signature")] + previous: Option, + #[arg(long, requires = "previous")] + previous_signature: Option, + #[arg(long)] + at: Option>, + }, /// Show added/removed evidence fingerprints without loading either registry. Diff { #[arg(long)] @@ -256,6 +615,9 @@ enum Command { key: PathBuf, #[arg(long)] allowed_reviewers: PathBuf, + /// Required by policies that separate release publishers and reviewers. + #[arg(long)] + identity: Option, }, /// Verify publisher signature and atomically activate (also supports rollback). Activate { @@ -283,13 +645,22 @@ pub(super) async fn run() -> anyhow::Result<()> { command @ (Command::Download { .. } | Command::CruxDownload { .. } | Command::Manifest { .. } - | Command::Equivalence { .. }) => acquire(command).await?, + | Command::ObservationManifest { .. } + | Command::Equivalence { .. } + | Command::EquivalenceVote { .. } + | Command::Observe { .. }) => acquire(command).await?, command @ (Command::Lookup { .. } | Command::Entity { .. } | Command::LookupWeb { .. } | Command::Popularity { .. } | Command::Category { .. } | Command::Stats { .. } + | Command::ReviewQueue { .. } + | Command::Drift { .. } + | Command::RevocationCandidates { .. } + | Command::Evidence { .. } + | Command::Observations { .. } + | Command::ObservationLookup { .. } | Command::Evaluate { .. } | Command::Resolve { .. }) => inspect(command)?, Command::Import { @@ -307,9 +678,167 @@ pub(super) async fn run() -> anyhow::Result<()> { maximum_records, maximum_database_growth_bytes, )?, - Command::Build { database, output } => { + Command::ObservationImport { + database, + generation, + pin, + input, + manifest, + } => { + let manifest = registry::read_json(&manifest)?; + serde_json::json!({"observation_batch_id":registry::observation::import( + ®istry::store::open(&database)?, + &Registry::open(&generation,&pin)?, + &manifest, + &input, + )?,"rebuild_required":true}) + } + Command::PrepareVote { + generation, + pin, + subject_kind, + fingerprint, + decision, + reviewer, + reason, + reviewed_at, + expires_at, + role, + locale, + country, + supersedes, + output, + } => { + let subject_kind = subject_kind.into(); + let registry = Registry::open(&generation, &pin)?; + let evidence = registry::bundle::build(®istry, subject_kind, &fingerprint)?; + let vote = registry::vote::Vote { + schema: "argand.site-vote/v1".into(), + subject_kind, + fingerprint, + decision, + reviewer, + reason, + evidence_bundle: evidence.id, + policy: registry.receipt.review_policy_sha256.clone(), + reviewed_at, + expires_at, + role, + locale, + country, + supersedes, + }; + vote.validate()?; + argand_atomic::create_durable(&output, &serde_json::to_vec_pretty(&vote)?)?; + serde_json::json!({"vote":output,"signature_namespace":registry::vote::SIGNATURE_NAMESPACE}) + } + Command::PrepareEquivalenceVote { + generation, + pin, + left, + right, + decision, + reviewer, + reason, + reviewed_at, + expires_at, + supersedes, + output, + } => { + let registry = Registry::open(&generation, &pin)?; + let pair = registry::identity::propose(®istry, &left, &right)?; + let evidence = registry::bundle::equivalence(®istry, &pair)?; + let vote = registry::vote::Vote { + schema: "argand.site-vote/v1".into(), + subject_kind: registry::policy::SubjectKind::Equivalence, + fingerprint: pair.fingerprint, + decision, + reviewer, + reason, + evidence_bundle: evidence.id, + policy: registry.receipt.review_policy_sha256.clone(), + reviewed_at, + expires_at, + role: "unspecified".into(), + locale: String::new(), + country: String::new(), + supersedes, + }; + vote.validate()?; + argand_atomic::create_durable(&output, &serde_json::to_vec_pretty(&vote)?)?; + serde_json::json!({"vote":output,"signature_namespace":registry::vote::SIGNATURE_NAMESPACE}) + } + Command::VerifyEquivalenceVote { + generation, + pin, + left, + right, + decision, + signature, + allowed_reviewers, + identity, + } => { + let registry = Registry::open(&generation, &pin)?; + let pair = registry::identity::propose(®istry, &left, &right)?; + let (vote, authentication) = + registry::vote::authenticate(&decision, &signature, &allowed_reviewers, &identity)?; + registry::vote::verify_equivalence_authenticated( + ®istry, + &pair, + &vote, + &authentication, + )?; + serde_json::json!({"verified":true,"vote_id":registry::file_digest(&decision)?,"reviewer":identity}) + } + Command::VerifyVote { + generation, + pin, + decision, + signature, + allowed_reviewers, + identity, + } => { + let (vote, authentication) = + registry::vote::authenticate(&decision, &signature, &allowed_reviewers, &identity)?; + registry::vote::verify_authenticated( + &Registry::open(&generation, &pin)?, + &vote, + &authentication, + )?; + serde_json::json!({"verified":true,"vote_id":registry::file_digest(&decision)?,"reviewer":identity}) + } + Command::ObserveReplay { + generation, + pin, + capture, + output, + manifest_output, + } => { + let manifest = registry::observer::replay( + &Registry::open(&generation, &pin)?, + &capture, + &output, + &manifest_output, + )?; + serde_json::json!({"capture":capture,"output":output,"manifest_output":manifest_output,"manifest":manifest}) + } + Command::Build { + database, + output, + policy, + reviewer_trust, + } => { ensure!(database.is_file(), "import database does not exist"); - let receipt = registry::build::build(®istry::store::open(&database)?, &output)?; + let policy = policy + .map(|path| registry::read_json(&path)) + .transpose()? + .unwrap_or_else(registry::policy::ReviewPolicy::reference); + let receipt = registry::build::build_with_policy_and_trust( + ®istry::store::open(&database)?, + &output, + &policy, + reviewer_trust.as_deref(), + )?; serde_json::json!({"receipt":receipt,"pin":registry::file_digest(&output.join("COMPLETE.json"))?,"generation":output}) } Command::Review { @@ -329,6 +858,23 @@ pub(super) async fn run() -> anyhow::Result<()> { &allowed_reviewers, &identity, )?, + Command::Vote { + database, + generation, + pin, + decision, + signature, + allowed_reviewers, + identity, + } => record_vote( + &database, + &generation, + &pin, + &decision, + &signature, + &allowed_reviewers, + &identity, + )?, Command::Export { generation, pin, @@ -342,6 +888,90 @@ pub(super) async fn run() -> anyhow::Result<()> { )?; serde_json::json!({"export":output}) } + Command::ExportAudit { + generation, + pin, + output, + include_descriptions, + } => { + registry::release::export_audit( + &Registry::open(&generation, &pin)?, + &output, + include_descriptions, + )?; + serde_json::json!({"audit_export":output}) + } + Command::ExportRevocations { + generation, + pin, + effective_at, + output, + } => { + registry::revocation::export( + &Registry::open(&generation, &pin)?, + &output, + effective_at, + )?; + serde_json::json!({"revocation_feed":output,"signature_namespace":registry::revocation::SIGNATURE_NAMESPACE}) + } + Command::SignRevocations { + generation, + pin, + input, + output, + key, + allowed_reviewers, + identity, + } => { + registry::revocation::sign( + &Registry::open(&generation, &pin)?, + &input, + &output, + &key, + &allowed_reviewers, + &identity, + )?; + serde_json::json!({"revocation_feed":input,"signature":output,"publisher":identity}) + } + Command::VerifyRevocations { + generation, + pin, + input, + signature, + allowed_publishers, + identity, + previous, + previous_signature, + at, + } => { + let registry = Registry::open(&generation, &pin)?; + let now = at.unwrap_or_else(chrono::Utc::now); + let previous = previous + .as_deref() + .zip(previous_signature.as_deref()) + .map(|(feed, signature)| { + registry::revocation::verify( + ®istry, + feed, + signature, + &allowed_publishers, + &identity, + now, + None, + ) + }) + .transpose()?; + let verified = registry::revocation::verify( + ®istry, + &input, + &signature, + &allowed_publishers, + &identity, + now, + previous.as_ref(), + )?; + serde_json::json!({"verified":true,"sha256":verified.sha256,"publisher":verified.publisher,"feed":verified.feed()}) + } Command::Diff { old, old_pin, @@ -363,8 +993,13 @@ pub(super) async fn run() -> anyhow::Result<()> { pin, key, allowed_reviewers, + identity, } => { - registry::release::sign(&generation, &key, &pin, &allowed_reviewers)?; + if let Some(identity) = identity { + registry::release::sign_as(&generation, &key, &pin, &allowed_reviewers, &identity)?; + } else { + registry::release::sign(&generation, &key, &pin, &allowed_reviewers)?; + } serde_json::json!({"signed":generation}) } Command::Activate { @@ -410,6 +1045,26 @@ fn record_review( ) } +fn record_vote( + database: &std::path::Path, + generation: &std::path::Path, + pin: &str, + decision: &std::path::Path, + signature: &std::path::Path, + allowed_reviewers: &std::path::Path, + identity: &str, +) -> anyhow::Result { + let (vote, authentication) = + registry::vote::authenticate(decision, signature, allowed_reviewers, identity)?; + let id = registry::vote::record_authenticated( + ®istry::store::open(database)?, + &Registry::open(generation, pin)?, + &vote, + &authentication, + )?; + Ok(serde_json::json!({"vote_id":id,"authenticated_reviewer":identity,"rebuild_required":true})) +} + fn activate_release( generation: &std::path::Path, current: &std::path::Path, @@ -465,6 +1120,7 @@ fn write_json(value: &serde_json::Value) -> anyhow::Result<()> { Ok(()) } +#[allow(clippy::too_many_lines)] // One exhaustive read-only command dispatch table is easier to audit. fn inspect(command: Command) -> anyhow::Result { Ok(match command { Command::Lookup { @@ -500,6 +1156,64 @@ fn inspect(command: Command) -> anyhow::Result { Command::Stats { generation, pin } => { serde_json::to_value(Registry::open(&generation, &pin)?.stats(chrono::Utc::now())?)? } + Command::ReviewQueue { + generation, + pin, + limit, + maximum_subjects, + at, + } => serde_json::to_value(registry::queue::review_queue( + &Registry::open(&generation, &pin)?, + at.unwrap_or_else(chrono::Utc::now), + limit, + maximum_subjects, + )?)?, + Command::Drift { + generation, + pin, + fingerprint, + } => serde_json::to_value(registry::queue::drift( + &Registry::open(&generation, &pin)?, + &fingerprint, + )?)?, + Command::RevocationCandidates { + generation, + pin, + maximum_subjects, + } => serde_json::to_value(registry::queue::revocation_candidates( + &Registry::open(&generation, &pin)?, + maximum_subjects, + )?)?, + Command::Evidence { + generation, + pin, + subject_kind, + fingerprint, + } => serde_json::to_value(registry::bundle::build( + &Registry::open(&generation, &pin)?, + subject_kind.into(), + &fingerprint, + )?)?, + Command::Observations { + generation, + pin, + fingerprint, + limit, + } => serde_json::to_value(registry::observation::lookup( + &Registry::open(&generation, &pin)?, + &fingerprint, + limit, + )?)?, + Command::ObservationLookup { + generation, + pin, + target, + limit, + } => serde_json::to_value(registry::observation::reverse_lookup( + &Registry::open(&generation, &pin)?, + &target, + limit, + )?)?, Command::Evaluate { generation, pin, @@ -518,16 +1232,70 @@ fn inspect(command: Command) -> anyhow::Result { query, locale, country, - } => serde_json::to_value(Registry::open(&generation, &pin)?.resolve_explained( - &query, - locale.as_deref(), - country.as_deref(), - chrono::Utc::now(), - )?)?, + revocations, + revocation_signature, + allowed_publishers, + publisher_identity, + previous_revocations, + previous_revocation_signature, + } => { + let registry = Registry::open(&generation, &pin)?; + let now = chrono::Utc::now(); + if let Some(feed) = revocations { + let signature = revocation_signature + .as_deref() + .context("missing revocation signature")?; + let publishers = allowed_publishers + .as_deref() + .context("missing publisher trust root")?; + let publisher = publisher_identity + .as_deref() + .context("missing publisher identity")?; + let previous = previous_revocations + .as_deref() + .zip(previous_revocation_signature.as_deref()) + .map(|(previous_feed, previous_signature)| { + registry::revocation::verify( + ®istry, + previous_feed, + previous_signature, + publishers, + publisher, + now, + None, + ) + }) + .transpose()?; + let verified = registry::revocation::verify( + ®istry, + &feed, + signature, + publishers, + publisher, + now, + previous.as_ref(), + )?; + serde_json::to_value(registry.resolve_explained_with_revocations( + &query, + locale.as_deref(), + country.as_deref(), + now, + &verified, + )?)? + } else { + serde_json::to_value(registry.resolve_explained( + &query, + locale.as_deref(), + country.as_deref(), + now, + )?)? + } + } _ => anyhow::bail!("expected a registry inspection command"), }) } +#[allow(clippy::too_many_lines)] // One exhaustive acquisition command dispatch table is easier to audit. async fn acquire(command: Command) -> anyhow::Result { Ok(match command { Command::Equivalence { @@ -554,9 +1322,64 @@ async fn acquire(command: Command) -> anyhow::Result { registry::review::authenticate(&decision, &signature, &allowed, &identity)?; serde_json::json!({"review_sequence":registry::identity::record_authenticated(®istry::store::open(&database)?,®istry,&pair,&review,&authentication)?,"authenticated_reviewer":identity,"rebuild_required":true}) } else { - serde_json::to_value(pair)? + let bundle = registry::bundle::equivalence(®istry, &pair)?; + let mut value = serde_json::to_value(pair)?; + value["evidence_bundle"] = serde_json::to_value(bundle)?; + value } } + Command::EquivalenceVote { + generation, + pin, + left, + right, + database, + decision, + signature, + allowed_reviewers, + identity, + } => { + let registry = Registry::open(&generation, &pin)?; + let pair = registry::identity::propose(®istry, &left, &right)?; + let (vote, authentication) = + registry::vote::authenticate(&decision, &signature, &allowed_reviewers, &identity)?; + serde_json::json!({"vote_id":registry::identity::record_vote_authenticated(®istry::store::open(&database)?,®istry,&pair,&vote,&authentication)?,"authenticated_reviewer":identity,"rebuild_required":true}) + } + Command::Observe { + generation, + pin, + fingerprint, + capture, + output, + manifest_output, + maximum_body_bytes, + maximum_redirects, + timeout_seconds, + maximum_bytes_per_second, + } => { + let registry = Registry::open(&generation, &pin)?; + let coordinate = chrono::Utc::now().format("%Y%m%dT%H%M%S%.9fZ").to_string(); + let expand = |path: PathBuf| -> anyhow::Result { + let value = path + .to_str() + .context("observer output paths must be UTF-8")?; + Ok(PathBuf::from(value.replace("{timestamp}", &coordinate))) + }; + let capture = expand(capture)?; + let output = expand(output)?; + let manifest_output = expand(manifest_output)?; + let options = registry::observer::Options { + maximum_body_bytes, + maximum_redirects, + timeout_seconds, + maximum_bytes_per_second, + }; + let observed = + registry::observer::capture(®istry, &fingerprint, &capture, &options).await?; + let manifest = + registry::observer::replay(®istry, &capture, &output, &manifest_output)?; + serde_json::json!({"capture":observed,"capture_path":capture,"output":output,"manifest_output":manifest_output,"manifest":manifest}) + } Command::Download { cache, source, @@ -566,6 +1389,7 @@ async fn acquire(command: Command) -> anyhow::Result { snapshot, scope, maximum_bytes, + coverage, } => serde_json::to_value( registry::download::download( &cache, @@ -577,6 +1401,9 @@ async fn acquire(command: Command) -> anyhow::Result { snapshot, scope, maximum_bytes, + coverage: coverage + .map(|path| registry::read_json(&path)) + .transpose()?, }, ) .await?, @@ -594,15 +1421,25 @@ async fn acquire(command: Command) -> anyhow::Result { snapshot, scope, retrieved_at, + coverage, } => { + let coverage = coverage + .map(|path| registry::read_json(&path)) + .transpose()?; let manifest = SourceManifest { - schema: "argand.site-source/v1".into(), + schema: if coverage.is_some() { + "argand.site-source/v2" + } else { + "argand.site-source/v1" + } + .into(), source, format, compression, source_url, snapshot, scope, + coverage, retrieved_at, license: source.license().into(), license_url: source.license_url().into(), @@ -613,6 +1450,29 @@ async fn acquire(command: Command) -> anyhow::Result { argand_atomic::create_durable(&output, &serde_json::to_vec_pretty(&manifest)?)?; serde_json::to_value(manifest)? } + Command::ObservationManifest { + input, + output, + source, + source_url, + license, + license_url, + retrieved_at, + } => { + let manifest = registry::observation::BatchManifest { + schema: "argand.site-observation-source/v1".into(), + source, + source_url, + license, + license_url, + retrieved_at, + sha256: registry::file_digest(&input)?, + bytes: input.metadata()?.len(), + }; + manifest.validate()?; + argand_atomic::create_durable(&output, &serde_json::to_vec_pretty(&manifest)?)?; + serde_json::to_value(manifest)? + } _ => anyhow::bail!("expected an acquisition command"), }) } diff --git a/crates/argand-site-registry/src/coverage.rs b/crates/argand-site-registry/src/coverage.rs new file mode 100644 index 0000000..79d9f7a --- /dev/null +++ b/crates/argand-site-registry/src/coverage.rs @@ -0,0 +1,576 @@ +// By Nic Weyand! +//! Typed source coverage selection and active-record masking. + +use crate::model::{CoverageKind, SourceManifest}; +use anyhow::{Context, ensure}; +use rusqlite::{Connection, params}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Clone, Debug)] +struct Snapshot { + id: String, + source: String, + scope: String, + retrieved_at: String, + manifest: SourceManifest, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct Selected { + id: String, + precedence: u64, + coordinate: String, +} + +/// Selects one coherent source coverage graph and materializes active records. +/// +/// # Errors +/// Rejects missing/cross-source supersession, coverage forks, mixed legacy and +/// typed coverage, delta gaps, duplicate native records, and SQLite failures. +pub(crate) fn project(db: &Connection) -> anyhow::Result<()> { + let snapshots = snapshots(db)?; + let selected = select(&snapshots)?; + db.execute_batch("BEGIN IMMEDIATE")?; + let result = project_inner(db, &selected); + match result { + Ok(()) => db.execute_batch("COMMIT")?, + Err(error) => { + db.execute_batch("ROLLBACK")?; + return Err(error); + } + } + Ok(()) +} + +/// Stable digest of the complete selected coverage graph. +pub(crate) fn projection_digest(db: &Connection) -> anyhow::Result { + let mut statement = + db.prepare("SELECT id,precedence,partition FROM selected_sources ORDER BY id")?; + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + )) + })? + .collect::, _>>()?; + Ok(crate::digest(&serde_json::to_vec(&( + "argand.site-coverage-selection/v1", + rows, + ))?)) +} + +fn snapshots(db: &Connection) -> anyhow::Result> { + let mut statement = db.prepare( + "SELECT id,source,scope,retrieved_at,manifest FROM sources WHERE complete=1 ORDER BY id", + )?; + statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + })? + .map(|row| { + let (id, source, scope, retrieved_at, raw) = row?; + let manifest: SourceManifest = serde_json::from_str(&raw)?; + manifest.validate()?; + ensure!( + manifest.id()? == id && manifest.source.key() == source && manifest.scope == scope, + "stored source declaration differs from its identity" + ); + Ok(Snapshot { + id, + source, + scope, + retrieved_at, + manifest, + }) + }) + .collect() +} + +/// Returns the current frontier for one typed source coverage coordinate. +/// +/// Used only by explicitly configured scheduled replacement. Invalid existing +/// coverage still fails closed through the same selector as generation builds. +pub(crate) fn frontier( + db: &Connection, + source: &str, + collection: &str, + coordinate: &str, +) -> anyhow::Result> { + let snapshots = snapshots(db)?; + if snapshots.is_empty() { + return Ok(None); + } + let selected = select(&snapshots)?; + Ok(selected + .into_iter() + .filter_map(|item| { + let snapshot = snapshots.iter().find(|snapshot| snapshot.id == item.id)?; + let coverage = snapshot.manifest.coverage.as_ref()?; + (snapshot.source == source + && coverage.collection == collection + && coverage.coordinate() == coordinate) + .then_some((item.precedence, item.id)) + }) + .max_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))) + .map(|(_, id)| id)) +} + +#[allow(clippy::too_many_lines)] // One pass validates and selects the complete coverage graph. +fn select(snapshots: &[Snapshot]) -> anyhow::Result> { + ensure!( + !snapshots.is_empty(), + "registry has no complete source snapshots" + ); + let by_id = snapshots + .iter() + .map(|snapshot| (snapshot.id.as_str(), snapshot)) + .collect::>(); + let mut superseded = BTreeSet::new(); + for snapshot in snapshots { + let Some(coverage) = &snapshot.manifest.coverage else { + continue; + }; + for replaced in &coverage.supersedes { + ensure!(replaced != &snapshot.id, "source cannot supersede itself"); + let prior = by_id + .get(replaced.as_str()) + .context("coverage supersedes an absent or incomplete source")?; + ensure!( + prior.source == snapshot.source, + "coverage cannot supersede another provider" + ); + if let Some(prior_coverage) = &prior.manifest.coverage { + ensure!( + prior_coverage.collection == coverage.collection, + "coverage cannot supersede another collection" + ); + } + superseded.insert(replaced.clone()); + } + } + reject_cycles(snapshots, &by_id)?; + + let mut selected = legacy_selection(snapshots, &superseded); + let typed_sources = snapshots + .iter() + .filter(|snapshot| { + snapshot.manifest.coverage.is_some() && !superseded.contains(&snapshot.id) + }) + .map(|snapshot| snapshot.source.as_str()) + .collect::>(); + for source in typed_sources { + ensure!( + !selected.iter().any(|item| { + by_id + .get(item.id.as_str()) + .is_some_and(|snapshot| snapshot.source == source) + }), + "legacy and typed coverage cannot remain active for one source" + ); + } + + let mut collections: BTreeMap<(&str, &str), Vec<&Snapshot>> = BTreeMap::new(); + for snapshot in snapshots { + if let Some(coverage) = &snapshot.manifest.coverage { + collections + .entry((&snapshot.source, &coverage.collection)) + .or_default() + .push(snapshot); + } + } + for ((source, collection), members) in collections { + let frontiers = members + .iter() + .copied() + .filter(|snapshot| !superseded.contains(&snapshot.id)) + .collect::>(); + ensure!(!frontiers.is_empty(), "coverage collection has no frontier"); + let mut chains = Vec::new(); + for frontier in frontiers { + chains.push(delta_chain(frontier, &by_id)?); + } + let mut coordinates = BTreeSet::new(); + let mut full = 0_u8; + for chain in &chains { + let root = chain.first().context("empty coverage chain")?; + let root_coverage = root + .manifest + .coverage + .as_ref() + .context("typed chain has a legacy root")?; + if root_coverage.kind == CoverageKind::Full { + full = full.saturating_add(1); + } + ensure!( + coordinates.insert(root_coverage.coordinate()), + "coverage collection has conflicting active branches" + ); + } + ensure!( + full == 0 || (full == 1 && chains.len() == 1), + "full coverage cannot compose with another active branch" + ); + for chain in chains { + let coordinate = chain + .first() + .and_then(|snapshot| snapshot.manifest.coverage.as_ref()) + .map(|coverage| format!("{source}:{collection}:{}", coverage.coordinate())) + .context("missing coverage coordinate")?; + for (precedence, snapshot) in chain.into_iter().enumerate() { + selected.push(Selected { + id: snapshot.id.clone(), + precedence: u64::try_from(precedence)?, + coordinate: coordinate.clone(), + }); + } + } + } + selected.sort_by(|left, right| left.id.cmp(&right.id)); + selected.dedup_by(|left, right| { + if left.id != right.id { + return false; + } + left.precedence = left.precedence.max(right.precedence); + true + }); + Ok(selected) +} + +fn legacy_selection(snapshots: &[Snapshot], superseded: &BTreeSet) -> Vec { + let mut latest: BTreeMap<(&str, &str), &Snapshot> = BTreeMap::new(); + for snapshot in snapshots.iter().filter(|snapshot| { + snapshot.manifest.coverage.is_none() && !superseded.contains(&snapshot.id) + }) { + let slot = latest.entry((&snapshot.source, &snapshot.scope)); + slot.and_modify(|current| { + if (&snapshot.retrieved_at, &snapshot.id) > (¤t.retrieved_at, ¤t.id) { + *current = snapshot; + } + }) + .or_insert(snapshot); + } + latest + .into_values() + .map(|snapshot| Selected { + id: snapshot.id.clone(), + precedence: 0, + coordinate: format!("{}:legacy:{}", snapshot.source, snapshot.scope), + }) + .collect() +} + +fn delta_chain<'a>( + frontier: &'a Snapshot, + by_id: &BTreeMap<&str, &'a Snapshot>, +) -> anyhow::Result> { + let mut reverse = Vec::new(); + let mut current = frontier; + let mut seen = BTreeSet::new(); + loop { + ensure!(seen.insert(current.id.as_str()), "coverage base cycle"); + reverse.push(current); + let coverage = current + .manifest + .coverage + .as_ref() + .context("delta chain reached legacy source")?; + if coverage.kind != CoverageKind::Delta { + break; + } + let base_id = coverage.base.as_deref().context("delta has no base")?; + let base = by_id + .get(base_id) + .context("delta base is absent or incomplete")?; + let base_coverage = base + .manifest + .coverage + .as_ref() + .context("delta base uses legacy coverage")?; + ensure!( + base.source == current.source + && base_coverage.collection == coverage.collection + && base_coverage.coordinate() == coverage.coordinate(), + "delta base has different source coverage" + ); + let expected = if base_coverage.kind == CoverageKind::Delta { + base_coverage + .sequence + .context("delta base has no sequence")? + .checked_add(1) + .context("delta sequence overflow")? + } else { + 1 + }; + ensure!( + coverage.sequence == Some(expected), + "delta sequence is not consecutive" + ); + current = base; + } + reverse.reverse(); + Ok(reverse) +} + +fn reject_cycles(snapshots: &[Snapshot], by_id: &BTreeMap<&str, &Snapshot>) -> anyhow::Result<()> { + for snapshot in snapshots { + let mut pending = vec![snapshot.id.as_str()]; + let mut seen = BTreeSet::new(); + while let Some(id) = pending.pop() { + ensure!(seen.insert(id), "coverage supersession cycle"); + let current = by_id.get(id).context("coverage source disappeared")?; + if let Some(coverage) = ¤t.manifest.coverage { + pending.extend(coverage.supersedes.iter().map(String::as_str)); + } + } + } + Ok(()) +} + +fn project_inner(db: &Connection, selected: &[Selected]) -> anyhow::Result<()> { + { + let mut insert = + db.prepare("INSERT INTO selected_sources(id,precedence,partition) VALUES(?1,?2,?3)")?; + for item in selected { + insert.execute(params![ + item.id, + i64::try_from(item.precedence)?, + item.coordinate + ])?; + } + } + let mut query = db.prepare( + "SELECT s.source,r.native_id,a.precedence,r.source_id,r.ordinal \ + FROM records r JOIN selected_sources a ON a.id=r.source_id \ + JOIN sources s ON s.id=r.source_id \ + ORDER BY s.source,r.native_id,a.precedence DESC,r.source_id,r.ordinal", + )?; + let mut rows = query.query([])?; + let mut insert = db.prepare("INSERT INTO active_records VALUES(?1,?2)")?; + let mut previous: Option<(String, String, u64)> = None; + while let Some(row) = rows.next()? { + let source: String = row.get(0)?; + let native: String = row.get(1)?; + let precedence = crate::store::unsigned(row, 2)?; + let key = (source, native); + if let Some((prior_source, prior_native, prior_precedence)) = &previous + && (&key.0, &key.1) == (prior_source, prior_native) + { + ensure!( + precedence < *prior_precedence, + "active source partitions contain duplicate native records" + ); + continue; + } + insert.execute(params![row.get::<_, String>(3)?, row.get::<_, i64>(4)?])?; + previous = Some((key.0, key.1, precedence)); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{Compression, Format, Source, SourceCoverage}; + use chrono::{TimeZone, Utc}; + + fn snapshot( + name: &str, + kind: CoverageKind, + partition: Option<&str>, + base: Option<&str>, + sequence: Option, + supersedes: Vec, + ) -> anyhow::Result { + let id = crate::digest(name.as_bytes()); + let coverage = SourceCoverage { + collection: "entities".into(), + kind, + partition: partition.map(str::to_owned), + base: base.map(str::to_owned), + sequence, + supersedes, + }; + coverage.validate()?; + let manifest = SourceManifest { + schema: "argand.site-source/v2".into(), + source: Source::Wikidata, + format: Format::WikidataEntities, + compression: Compression::None, + snapshot: name.into(), + scope: name.into(), + coverage: Some(coverage), + source_url: "https://www.wikidata.org/wiki/Special:EntityData/Q355.json".into(), + license: Source::Wikidata.license().into(), + license_url: Source::Wikidata.license_url().into(), + retrieved_at: Utc + .with_ymd_and_hms(2026, 9, 13, 0, 0, 0) + .single() + .context("test timestamp")?, + sha256: crate::digest(b"input"), + bytes: 5, + }; + Ok(Snapshot { + id, + source: "wikidata".into(), + scope: name.into(), + retrieved_at: manifest.retrieved_at.to_rfc3339(), + manifest, + }) + } + + #[test] + fn full_supersedes_partitions_and_delta_keeps_its_base() -> anyhow::Result<()> { + let left = snapshot( + "left", + CoverageKind::Partition, + Some("left"), + None, + None, + Vec::new(), + )?; + let right = snapshot( + "right", + CoverageKind::Partition, + Some("right"), + None, + None, + Vec::new(), + )?; + let full = snapshot( + "full", + CoverageKind::Full, + None, + None, + None, + vec![left.id.clone(), right.id.clone()], + )?; + let delta = snapshot( + "delta", + CoverageKind::Delta, + None, + Some(&full.id), + Some(1), + vec![full.id.clone()], + )?; + let selected = select(&[left, right, full.clone(), delta.clone()])?; + assert_eq!(selected.len(), 2); + assert!(selected.iter().any(|item| item.id == full.id)); + assert!(selected.iter().any(|item| item.id == delta.id)); + Ok(()) + } + + #[test] + fn conflicting_full_and_partition_fail_closed() -> anyhow::Result<()> { + let full = snapshot("full", CoverageKind::Full, None, None, None, Vec::new())?; + let partition = snapshot( + "partition", + CoverageKind::Partition, + Some("part"), + None, + None, + Vec::new(), + )?; + assert!(select(&[full, partition]).is_err()); + Ok(()) + } + + #[test] + fn disjoint_partitions_compose_but_overlap_fails() -> anyhow::Result<()> { + let left = snapshot( + "left", + CoverageKind::Partition, + Some("left"), + None, + None, + Vec::new(), + )?; + let right = snapshot( + "right", + CoverageKind::Partition, + Some("right"), + None, + None, + Vec::new(), + )?; + assert_eq!(select(&[left.clone(), right])?.len(), 2); + let overlap = snapshot( + "overlap", + CoverageKind::Partition, + Some("left"), + None, + None, + Vec::new(), + )?; + assert!(select(&[left, overlap]).is_err()); + Ok(()) + } + + #[test] + fn missing_skipped_and_forked_delta_history_fails() -> anyhow::Result<()> { + let missing_id = crate::digest(b"absent"); + let missing = snapshot( + "missing", + CoverageKind::Delta, + None, + Some(&missing_id), + Some(1), + vec![missing_id.clone()], + )?; + assert!(select(&[missing]).is_err()); + + let full = snapshot("full", CoverageKind::Full, None, None, None, Vec::new())?; + let skipped = snapshot( + "skipped", + CoverageKind::Delta, + None, + Some(&full.id), + Some(2), + vec![full.id.clone()], + )?; + assert!(select(&[full.clone(), skipped]).is_err()); + + let first = snapshot( + "first", + CoverageKind::Delta, + None, + Some(&full.id), + Some(1), + vec![full.id.clone()], + )?; + let fork = snapshot( + "fork", + CoverageKind::Delta, + None, + Some(&full.id), + Some(1), + vec![full.id.clone()], + )?; + assert!(select(&[full, first, fork]).is_err()); + Ok(()) + } + + #[test] + fn legacy_and_typed_source_cannot_mix_without_explicit_migration() -> anyhow::Result<()> { + let mut legacy = snapshot("legacy", CoverageKind::Full, None, None, None, Vec::new())?; + legacy.manifest.schema = "argand.site-source/v1".into(); + legacy.manifest.coverage = None; + let typed = snapshot( + "typed", + CoverageKind::Partition, + Some("p0"), + None, + None, + Vec::new(), + )?; + assert!(select(&[legacy, typed]).is_err()); + Ok(()) + } +} diff --git a/crates/argand-site-registry/src/crux.rs b/crates/argand-site-registry/src/crux.rs index 5990633..eeacfc2 100644 --- a/crates/argand-site-registry/src/crux.rs +++ b/crates/argand-site-registry/src/crux.rs @@ -30,6 +30,9 @@ pub struct CruxDownload { pub maximum_bytes_billed: u64, /// Maximum exported CSV bytes. pub maximum_output_bytes: u64, + /// Explicit monthly partition/delta coverage declaration. + #[serde(default)] + pub coverage: Option, } /// Acquires a complete `CrUX` projection using `GOOGLE_OAUTH_ACCESS_TOKEN`. @@ -79,7 +82,12 @@ pub async fn download(cache: &Path, request: &CruxDownload) -> anyhow::Result anyhow::Result anyhow::Result { &request.month, &request.country, request.maximum_bytes_billed, + &request.coverage, ))?)) } @@ -346,6 +356,7 @@ mod tests { country: Some("GB".into()), maximum_bytes_billed: 1_000_000, maximum_output_bytes: 1_000_000, + coverage: None, }; let sql = query(&request)?; let original_job = job_key(&request)?; diff --git a/crates/argand-site-registry/src/diff.rs b/crates/argand-site-registry/src/diff.rs index 0a8a1dc..0839abb 100644 --- a/crates/argand-site-registry/src/diff.rs +++ b/crates/argand-site-registry/src/diff.rs @@ -44,8 +44,8 @@ struct Table { const TABLES: &[Table] = &[ Table { subject: "source_selection", - scan: "SELECT s.id,json_object('source_id',s.id,'manifest',json(s.manifest)) FROM sources s JOIN selected_sources a USING(id) ORDER BY s.id", - find: "SELECT json_object('source_id',s.id,'manifest',json(s.manifest)) FROM sources s JOIN selected_sources a USING(id) WHERE s.id=?1", + scan: "SELECT s.id,json_object('source_id',s.id,'precedence',a.precedence,'partition',a.partition,'manifest',json(s.manifest)) FROM sources s JOIN selected_sources a USING(id) ORDER BY s.id", + find: "SELECT json_object('source_id',s.id,'precedence',a.precedence,'partition',a.partition,'manifest',json(s.manifest)) FROM sources s JOIN selected_sources a USING(id) WHERE s.id=?1", }, Table { subject: "entity", @@ -54,13 +54,13 @@ const TABLES: &[Table] = &[ }, Table { subject: "name", - scan: "SELECT fact,json_object('entity',entity,'fact',fact,'key',key,'text',text,'language',language,'kind',kind) FROM names ORDER BY fact", - find: "SELECT json_object('entity',entity,'fact',fact,'key',key,'text',text,'language',language,'kind',kind) FROM names WHERE fact=?1", + scan: "SELECT fingerprint,json_object('fingerprint',fingerprint,'entity',entity,'fact',fact,'key',key,'text',text,'language',language,'kind',kind) FROM names ORDER BY fingerprint", + find: "SELECT json_object('fingerprint',fingerprint,'entity',entity,'fact',fact,'key',key,'text',text,'language',language,'kind',kind) FROM names WHERE fingerprint=?1", }, Table { subject: "fact", - scan: "SELECT f.id,json_object('id',f.id,'source_id',f.source_id,'subject',f.subject,'predicate',f.predicate,'value',json(f.value),'selector',f.selector,'confidence',f.confidence) FROM facts f JOIN selected_sources s ON s.id=f.source_id ORDER BY f.id", - find: "SELECT json_object('id',f.id,'source_id',f.source_id,'subject',f.subject,'predicate',f.predicate,'value',json(f.value),'selector',f.selector,'confidence',f.confidence) FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.id=?1", + scan: "SELECT f.id,json_object('id',f.id,'source_id',f.source_id,'subject',f.subject,'predicate',f.predicate,'value',json(f.value),'selector',f.selector,'confidence',f.confidence) FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal ORDER BY f.id", + find: "SELECT json_object('id',f.id,'source_id',f.source_id,'subject',f.subject,'predicate',f.predicate,'value',json(f.value),'selector',f.selector,'confidence',f.confidence) FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.id=?1", }, Table { subject: "property", @@ -92,6 +92,26 @@ const TABLES: &[Table] = &[ scan: "SELECT fingerprint,json_object('fingerprint',fingerprint,'left_entity',left_entity,'right_entity',right_entity,'left_signature',left_signature,'right_signature',right_signature) FROM equivalences ORDER BY fingerprint", find: "SELECT json_object('fingerprint',fingerprint,'left_entity',left_entity,'right_entity',right_entity,'left_signature',left_signature,'right_signature',right_signature) FROM equivalences WHERE fingerprint=?1", }, + Table { + subject: "vote", + scan: "SELECT v.id,json_object('id',v.id,'fingerprint',v.fingerprint,'subject_kind',v.subject_kind,'decision',v.decision,'reviewer',v.reviewer,'reason',v.reason,'evidence_bundle',v.evidence_bundle,'policy',v.policy_sha256,'reviewed_at',v.reviewed_at,'expires_at',v.expires_at,'role',v.role,'locale',v.locale,'country',v.country,'supersedes',json(v.supersedes_json),'accepted_at',v.accepted_at,'authentication',json_object('signer',a.signer,'signature_sha256',a.signature_sha256,'namespace',a.namespace,'decision_sha256',a.decision_sha256,'key_sha256',a.key_sha256)) FROM votes v JOIN vote_auth a USING(sequence) ORDER BY v.id", + find: "SELECT json_object('id',v.id,'fingerprint',v.fingerprint,'subject_kind',v.subject_kind,'decision',v.decision,'reviewer',v.reviewer,'reason',v.reason,'evidence_bundle',v.evidence_bundle,'policy',v.policy_sha256,'reviewed_at',v.reviewed_at,'expires_at',v.expires_at,'role',v.role,'locale',v.locale,'country',v.country,'supersedes',json(v.supersedes_json),'accepted_at',v.accepted_at,'authentication',json_object('signer',a.signer,'signature_sha256',a.signature_sha256,'namespace',a.namespace,'decision_sha256',a.decision_sha256,'key_sha256',a.key_sha256)) FROM votes v JOIN vote_auth a USING(sequence) WHERE v.id=?1", + }, + Table { + subject: "observation_batch", + scan: "SELECT id,json_object('id',id,'source',source,'retrieved_at',retrieved_at,'manifest',json(manifest_json),'records',records,'complete',json(complete)) FROM observation_batches WHERE complete=1 ORDER BY id", + find: "SELECT json_object('id',id,'source',source,'retrieved_at',retrieved_at,'manifest',json(manifest_json),'records',records,'complete',json(complete)) FROM observation_batches WHERE complete=1 AND id=?1", + }, + Table { + subject: "observation", + scan: "SELECT fingerprint,json_object('fingerprint',fingerprint,'batch_id',batch_id,'subject_kind',subject_kind,'subject_fingerprint',subject_fingerprint,'document',json(document_json)) FROM observations ORDER BY fingerprint", + find: "SELECT json_object('fingerprint',fingerprint,'batch_id',batch_id,'subject_kind',subject_kind,'subject_fingerprint',subject_fingerprint,'document',json(document_json)) FROM observations WHERE fingerprint=?1", + }, + Table { + subject: "review_policy", + scan: "SELECT CAST(singleton AS TEXT),json_object('id',id,'document',json(document)) FROM review_policy ORDER BY singleton", + find: "SELECT json_object('id',id,'document',json(document)) FROM review_policy WHERE singleton=CAST(?1 AS INTEGER)", + }, ]; /// Streams every material change between two authenticated generations. @@ -103,13 +123,13 @@ pub fn write(old: &Registry, new: &Registry, output: &mut dyn Write) -> anyhow:: inner: output, written: 0, }; - output.line(&json!({"schema":"argand.site-diff/v3","type":"header","old":old.identity,"new":new.identity,"attribution":crate::release::attribution(),"descriptions_included":false}))?; + output.line(&json!({"schema":"argand.site-diff/v4","type":"header","old":old.identity,"new":new.identity,"old_coverage":old.receipt.coverage_sha256,"new_coverage":new.receipt.coverage_sha256,"old_policy":old.receipt.review_policy_sha256,"new_policy":new.receipt.review_policy_sha256,"attribution":crate::release::attribution(),"descriptions_included":false}))?; let mut changes = 0_u64; for table in TABLES { changes += removed_or_changed(table, &old.db, &new.db, &mut output)?; changes += added(table, &new.db, &old.db, &mut output)?; } - output.line(&json!({"schema":"argand.site-diff/v3","type":"summary","changes":changes}))?; + output.line(&json!({"schema":"argand.site-diff/v4","type":"summary","changes":changes}))?; Ok(()) } @@ -193,7 +213,7 @@ fn event( }) .transpose() }; - output.line(&json!({"schema":"argand.site-diff/v3","type":"change","subject":subject,"change":change,"key":key,"before":parse(before)?,"after":parse(after)?}))?; + output.line(&json!({"schema":"argand.site-diff/v4","type":"change","subject":subject,"change":change,"key":key,"before":parse(before)?,"after":parse(after)?}))?; Ok(()) } diff --git a/crates/argand-site-registry/src/download.rs b/crates/argand-site-registry/src/download.rs index e0f0083..da644e3 100644 --- a/crates/argand-site-registry/src/download.rs +++ b/crates/argand-site-registry/src/download.rs @@ -32,6 +32,9 @@ pub struct Download { pub scope: String, /// Maximum downloaded object bytes. pub maximum_bytes: u64, + /// Explicit full/partition/delta coverage; absent preserves legacy scope semantics. + #[serde(default)] + pub coverage: Option, } /// Result points to immutable cached bytes and their manifest. @@ -137,6 +140,7 @@ pub async fn download(cache: &Path, request: &Download) -> anyhow::Result anyhow::Result anyhow::Result<(u64, Ve "f.predicate IN('P17','P159','P407','P1001','P297','P218','P219','P220')" }; let from = format!( - "FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.subject=?1 AND {predicate}" + "FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.subject=?1 AND {predicate}" ); let total = db.query_row(&format!("SELECT count(*) {from}"), [entity], |r| { crate::store::unsigned(r, 0) diff --git a/crates/argand-site-registry/src/generation.rs b/crates/argand-site-registry/src/generation.rs index ee6682f..5fe1b06 100644 --- a/crates/argand-site-registry/src/generation.rs +++ b/crates/argand-site-registry/src/generation.rs @@ -51,15 +51,27 @@ impl Registry { && crate::digest(&attribution) == receipt.attribution_sha256, "registry license or attribution digest mismatch" ); - let rules_supported = crate::store::supported_rule_version(&receipt.rules) - || (allow_legacy_rules && crate::store::legacy_rule_version(&receipt.rules)); - ensure!( - receipt.schema == "argand.site-registry/v1" && rules_supported, - "unsupported registry contract" - ); + let current = receipt.schema == "argand.site-registry/v2" + && crate::store::supported_rule_version(&receipt.rules) + && crate::model::valid_digest(&receipt.review_policy_sha256) + && receipt.review_policy.id()? == receipt.review_policy_sha256 + && crate::model::valid_digest(&receipt.coverage_sha256) + && receipt.decision_time_policy == "argand.site-decision-time/v1" + && (receipt.review_policy.allow_legacy_reviews + || crate::model::valid_digest(&receipt.reviewer_trust_sha256)); + let legacy = allow_legacy_rules + && receipt.schema == "argand.site-registry/v1" + && crate::store::legacy_rule_version(&receipt.rules); + ensure!(current || legacy, "unsupported registry contract"); let database = path.join("registry.sqlite"); let (db, database) = open_authenticated_database(&database, &receipt.database_sha256)?; crate::store::configure(&db)?; + if current { + ensure!( + crate::coverage::projection_digest(&db)? == receipt.coverage_sha256, + "selected coverage graph differs from receipt" + ); + } Ok(Self { db, _database: database, diff --git a/crates/argand-site-registry/src/identity.rs b/crates/argand-site-registry/src/identity.rs index 542442d..69b358b 100644 --- a/crates/argand-site-registry/src/identity.rs +++ b/crates/argand-site-registry/src/identity.rs @@ -49,10 +49,8 @@ pub fn propose(registry: &Registry, left: &str, right: &str) -> anyhow::Result anyhow::Result { + let expected = propose(registry, &pair.entities[0], &pair.entities[1])?; + ensure!( + pair.fingerprint == expected.fingerprint && vote.fingerprint == expected.fingerprint, + "identity vote fingerprint differs from current proposal" + ); + for entity in &expected.entities { + let mut statement = registry.db.prepare("SELECT DISTINCT f.source_id FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.subject=?1")?; + for id in statement.query_map([entity], |row| row.get::<_, String>(0))? { + crate::store::source(db, &id?)?; + } + } + crate::vote::record_equivalence_authenticated_at( + db, + registry, + &expected, + vote, + authentication, + Utc::now(), + ) +} + fn record_inner( db: &Connection, registry: &Registry, @@ -108,7 +139,7 @@ fn record_inner( "identity reviews cannot assert a destination role" ); for entity in &expected.entities { - let mut statement = registry.db.prepare("SELECT DISTINCT f.source_id FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.subject=?1")?; + let mut statement = registry.db.prepare("SELECT DISTINCT f.source_id FROM facts f JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal WHERE f.subject=?1")?; for id in statement.query_map([entity], |r| r.get::<_, String>(0))? { crate::store::source(db, &id?)?; } @@ -152,6 +183,26 @@ pub(crate) fn expand( registry: &Registry, initial: &str, now: DateTime, +) -> anyhow::Result, Vec)>> { + expand_with_revocations(registry, initial, now, None) +} + +pub(crate) fn expand_with_revocations( + registry: &Registry, + initial: &str, + now: DateTime, + revocations: Option<&crate::revocation::VerifiedRevocations>, +) -> anyhow::Result, Vec)>> { + if !registry.receipt.review_policy.allow_legacy_reviews { + return expand_policy(registry, initial, now, revocations); + } + expand_legacy(registry, initial, now) +} + +fn expand_legacy( + registry: &Registry, + initial: &str, + now: DateTime, ) -> anyhow::Result, Vec)>> { let mut entities = BTreeSet::from([initial.to_owned()]); let mut pending = vec![initial.to_owned()]; @@ -211,6 +262,68 @@ pub(crate) fn expand( Ok(Some((entities, evidence.into_values().collect()))) } +fn expand_policy( + registry: &Registry, + initial: &str, + now: DateTime, + revocations: Option<&crate::revocation::VerifiedRevocations>, +) -> anyhow::Result, Vec)>> { + let mut entities = BTreeSet::from([initial.to_owned()]); + let mut pending = vec![initial.to_owned()]; + let mut evidence = BTreeMap::new(); + while let Some(entity) = pending.pop() { + let mut statement = registry.db.prepare( + "SELECT fingerprint,left_entity,right_entity FROM equivalences WHERE left_entity=?1 OR right_entity=?1 ORDER BY fingerprint", + )?; + let pairs = statement + .query_map([&entity], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + })? + .collect::, _>>()?; + for (fingerprint, left, right) in pairs { + if revocations.is_some_and(|feed| { + feed.blocks(crate::policy::SubjectKind::Equivalence, &fingerprint) + }) { + continue; + } + let pair = propose(registry, &left, &right)?; + ensure!( + pair.fingerprint == fingerprint, + "stored equivalence fingerprint is invalid" + ); + let bundle = crate::bundle::equivalence(registry, &pair)?; + let decision = crate::vote::decision_for_bundle( + registry, + crate::policy::SubjectKind::Equivalence, + &fingerprint, + &bundle.id, + now, + )?; + if decision.status != crate::vote::DecisionStatus::Approved { + continue; + } + let pair_ids = [left, right]; + evidence.insert( + fingerprint.clone(), + json!({"fingerprint":fingerprint,"entities":pair_ids,"evidence_bundle":bundle.id,"policy_decision":decision,"source":"argand_reviewer_votes","source_identifier":fingerprint,"license":"CC0-1.0","confidence":9000}), + ); + for id in pair_ids { + if entities.insert(id.clone()) { + pending.push(id); + } + } + if entities.len() > 64 || evidence.len() > 256 { + return Ok(None); + } + } + } + Ok(Some((entities, evidence.into_values().collect()))) +} + pub(crate) enum CandidateSearch { Ready { matched_entities: u64, @@ -220,32 +333,73 @@ pub(crate) enum CandidateSearch { AmbiguousIdentity { matched_entities: u64, }, + NoActiveNameReview { + matched_entities: u64, + }, SafetyLimitExceeded { matched_entities: u64, }, } -pub(crate) fn candidate_search( +pub(crate) fn candidate_search_with_revocations( registry: &Registry, query: &str, now: DateTime, + revocations: Option<&crate::revocation::VerifiedRevocations>, ) -> anyhow::Result { let key = crate::normalize::name_key(query)?; - let mut statement = registry - .db - .prepare("SELECT DISTINCT entity FROM names WHERE key=?1 ORDER BY entity LIMIT 65")?; - let matched = statement - .query_map([key], |r| r.get::<_, String>(0))? + let mut statement = registry.db.prepare( + "SELECT entity,fingerprint FROM names WHERE key=?1 ORDER BY entity,fingerprint LIMIT 257", + )?; + let raw = statement + .query_map([key], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? .collect::, _>>()?; - if matched.is_empty() { + if raw.is_empty() { return Ok(CandidateSearch::NoMatch); } - if matched.len() > 64 { + let raw_entities = raw + .iter() + .map(|(entity, _)| entity.clone()) + .collect::>(); + if raw_entities.len() > 64 || raw.len() > 256 { return Ok(CandidateSearch::SafetyLimitExceeded { - matched_entities: u64::try_from(matched.len())?, + matched_entities: u64::try_from(raw_entities.len())?, }); } - let Some((entities, evidence)) = expand(registry, &matched[0], now)? else { + let matched = if registry.receipt.review_policy.require_name_votes { + let mut approved = BTreeSet::new(); + for (entity, fingerprint) in raw { + if revocations + .is_some_and(|feed| feed.blocks(crate::policy::SubjectKind::Name, &fingerprint)) + { + continue; + } + if crate::vote::decision( + registry, + crate::policy::SubjectKind::Name, + &fingerprint, + now, + )? + .status + == crate::vote::DecisionStatus::Approved + { + approved.insert(entity); + } + } + if approved.is_empty() { + return Ok(CandidateSearch::NoActiveNameReview { + matched_entities: u64::try_from(raw_entities.len())?, + }); + } + approved.into_iter().collect::>() + } else { + raw_entities.into_iter().collect::>() + }; + let Some((entities, evidence)) = + expand_with_revocations(registry, &matched[0], now, revocations)? + else { return Ok(CandidateSearch::SafetyLimitExceeded { matched_entities: u64::try_from(matched.len())?, }); diff --git a/crates/argand-site-registry/src/lib.rs b/crates/argand-site-registry/src/lib.rs index a186dc5..8b1f7d8 100644 --- a/crates/argand-site-registry/src/lib.rs +++ b/crates/argand-site-registry/src/lib.rs @@ -3,7 +3,9 @@ pub mod adapters; pub mod build; +pub mod bundle; pub mod catalog; +mod coverage; pub mod crux; pub mod diff; pub mod download; @@ -15,15 +17,20 @@ mod json; pub mod model; pub mod normalize; pub mod observation; +pub mod observer; +pub mod policy; pub mod query; +pub mod queue; pub mod release; mod resolution; pub mod review; +pub mod revocation; mod ssh; pub mod store; pub use resolution::{Resolution, ResolutionCounts, ResolutionStatus}; pub mod update; +pub mod vote; use sha2::{Digest, Sha256}; use std::{fs::File, io::Read, path::Path}; diff --git a/crates/argand-site-registry/src/model.rs b/crates/argand-site-registry/src/model.rs index 62b34cc..744a54b 100644 --- a/crates/argand-site-registry/src/model.rs +++ b/crates/argand-site-registry/src/model.rs @@ -88,11 +88,108 @@ pub enum Compression { Bzip2, } +/// How one source object participates in an explicitly declared coverage set. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum CoverageKind { + /// One complete snapshot for the declared collection. + Full, + /// One publisher-declared disjoint partition of a collection. + Partition, + /// An ordered change set applied to an authenticated base object. + Delta, +} + +/// Typed source replacement and composition semantics. +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SourceCoverage { + /// Stable provider collection, such as `wikidata-entities`. + pub collection: String, + /// Coverage behavior of this object. + pub kind: CoverageKind, + /// Stable disjoint partition name; absent for a collection-wide object. + #[serde(default)] + pub partition: Option, + /// Exact prior source ID for a delta. + #[serde(default)] + pub base: Option, + /// Consecutive sequence within a delta chain. + #[serde(default)] + pub sequence: Option, + /// Exact older source IDs replaced by this object. + #[serde(default)] + pub supersedes: Vec, +} + +impl SourceCoverage { + /// Validates local coverage syntax. Cross-snapshot relationships are checked at build time. + /// + /// # Errors + /// Rejects ambiguous kinds, unsafe identifiers, and malformed source references. + pub fn validate(&self) -> anyhow::Result<()> { + fn identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b':' | b'.') + }) + } + ensure!(identifier(&self.collection), "invalid coverage collection"); + ensure!( + self.partition.as_deref().is_none_or(identifier), + "invalid coverage partition" + ); + ensure!( + self.supersedes.len() <= 4096 && self.supersedes.iter().all(|id| valid_digest(id)), + "invalid coverage supersession" + ); + let unique = self + .supersedes + .iter() + .collect::>(); + ensure!( + unique.len() == self.supersedes.len(), + "duplicate coverage supersession" + ); + match self.kind { + CoverageKind::Full => ensure!( + self.partition.is_none() && self.base.is_none() && self.sequence.is_none(), + "full coverage cannot declare partition or delta coordinates" + ), + CoverageKind::Partition => ensure!( + self.partition.is_some() && self.base.is_none() && self.sequence.is_none(), + "partition coverage needs only a partition" + ), + CoverageKind::Delta => { + ensure!( + self.base.as_deref().is_some_and(valid_digest) + && self.sequence.is_some_and(|sequence| sequence > 0), + "delta coverage needs a base digest and positive sequence" + ); + ensure!( + self.base + .as_ref() + .is_some_and(|base| self.supersedes.contains(base)), + "delta must explicitly supersede its base" + ); + } + } + Ok(()) + } + + /// Stable partition coordinate used to detect conflicting active branches. + #[must_use] + pub fn coordinate(&self) -> &str { + self.partition.as_deref().unwrap_or("__full__") + } +} + /// An immutable, externally auditable input declaration; local paths are separate. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct SourceManifest { - /// Must be `argand.site-source/v1`. + /// `argand.site-source/v1` for legacy scope semantics or v2 for typed coverage. pub schema: String, /// Approved provider. pub source: Source, @@ -105,6 +202,9 @@ pub struct SourceManifest { pub snapshot: String, /// Replacement scope, e.g. `full` or `selection:facebook`. pub scope: String, + /// Typed replacement semantics for schema v2; absent on legacy v1 declarations. + #[serde(default)] + pub coverage: Option, /// Original distribution URL, not an arbitrary mirror. pub source_url: String, /// Exact data license identifier. @@ -126,8 +226,11 @@ impl SourceManifest { /// Returns a descriptive error for unsupported source declarations. pub fn validate(&self) -> anyhow::Result<()> { ensure!( - self.schema == "argand.site-source/v1", - "unsupported source schema" + matches!( + (self.schema.as_str(), self.coverage.as_ref()), + ("argand.site-source/v1", None) | ("argand.site-source/v2", Some(_)) + ), + "unsupported source schema or coverage declaration" ); ensure!( self.license == self.source.license() && self.license_url == self.source.license_url(), @@ -155,6 +258,9 @@ impl SourceManifest { | (Source::Psl, Format::PslText) ); ensure!(valid, "source/format mismatch"); + if let Some(coverage) = &self.coverage { + coverage.validate()?; + } crate::download::validate_source_url(self.source, &self.source_url)?; Ok(()) } diff --git a/crates/argand-site-registry/src/observation.rs b/crates/argand-site-registry/src/observation.rs index f345f2c..49ecf05 100644 --- a/crates/argand-site-registry/src/observation.rs +++ b/crates/argand-site-registry/src/observation.rs @@ -1,11 +1,22 @@ // By Nic Weyand! //! Extension contract for later crawler evidence. No network or ownership inference. +use anyhow::{Context, ensure}; use chrono::{DateTime, Utc}; +use rusqlite::{Connection, OptionalExtension, params}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + io::{BufRead, BufReader, Read, Seek}, + path::Path, +}; + +const MAXIMUM_BATCH_BYTES: u64 = 1024 * 1024 * 1024; +const MAXIMUM_RECORD_BYTES: usize = 1024 * 1024; +const MAXIMUM_RECORDS: u64 = 10_000_000; /// Observed relationship, distinct from an entity-ownership claim. -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum ObservationKind { /// An actual HTTP redirect with its status code. @@ -29,6 +40,40 @@ pub enum ObservationKind { /// Country as declared by the site. country: String, }, + /// HTTP response status observed without an asserted relationship. + HttpStatus { + /// Valid three-digit HTTP status. + status: u16, + }, + /// Bounded observer failure retained instead of pretending no fetch occurred. + FetchFailure { + /// Stable allowlisted failure class. + class: String, + }, + /// Hash of the complete public address set pinned for one request. + DnsResolution { + /// SHA-256 over sorted socket addresses. + addresses_sha256: String, + }, + /// Hash of the verified leaf certificate returned for an HTTPS request. + TlsCertificate { + /// SHA-256 over the leaf certificate DER bytes. + certificate_sha256: String, + }, + /// Rights-reviewed domain-registration state from an external adapter. + DomainRegistration { + /// `active`, `expiry_risk`, `expired`, `redemption`, or `unknown`. + state: String, + /// Registry-reported expiry when the source supplies one. + expires_at: Option>, + }, + /// Result from an explicitly configured, rights-reviewed malware policy. + MalwarePolicy { + /// Stable publisher policy identifier, never an inferred vendor. + policy: String, + /// `clean`, `suspicious`, `malicious`, or `unknown`. + result: String, + }, } /// Immutable evidence coordinates for a future crawler-source adapter. @@ -60,7 +105,7 @@ pub struct Observation { } /// Deterministic crawler evidence prepared for a later rights-reviewed adapter. -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] pub struct NormalizedObservation { /// Versioned extension contract. pub schema: String, @@ -99,7 +144,6 @@ impl Observation { &self, normalizer: &crate::normalize::Normalizer, ) -> anyhow::Result { - use anyhow::ensure; ensure!( !self.source.trim().is_empty() && self.source.len() <= 128 @@ -163,7 +207,6 @@ impl Observation { } fn validate_relation(relation: &ObservationKind) -> anyhow::Result<()> { - use anyhow::ensure; match relation { ObservationKind::Redirect { status } => ensure!( matches!(status, 301 | 302 | 303 | 307 | 308), @@ -181,7 +224,430 @@ fn validate_relation(relation: &ObservationKind) -> anyhow::Result<()> { country.len() == 2 && country.bytes().all(|byte| byte.is_ascii_uppercase()), "invalid country selector scope" ), + ObservationKind::HttpStatus { status } => { + ensure!((100..=599).contains(status), "invalid observed HTTP status"); + } + ObservationKind::FetchFailure { class } => ensure!( + matches!( + class.as_str(), + "dns" + | "timeout" + | "tls" + | "connection" + | "http_status" + | "content_type" + | "content_encoding" + | "size_limit" + | "policy_block" + ), + "invalid observer failure class" + ), + ObservationKind::DnsResolution { addresses_sha256 } => ensure!( + crate::model::valid_digest(addresses_sha256), + "invalid DNS address-set digest" + ), + ObservationKind::TlsCertificate { certificate_sha256 } => ensure!( + crate::model::valid_digest(certificate_sha256), + "invalid TLS certificate digest" + ), + ObservationKind::DomainRegistration { state, expires_at } => { + ensure!( + matches!( + state.as_str(), + "active" | "expiry_risk" | "expired" | "redemption" | "unknown" + ), + "invalid domain-registration state" + ); + ensure!( + expires_at.is_none_or(|value| value.timestamp() >= 0), + "invalid domain expiry time" + ); + } + ObservationKind::MalwarePolicy { policy, result } => ensure!( + !policy.trim().is_empty() + && policy.len() <= 256 + && policy + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + && matches!( + result.as_str(), + "clean" | "suspicious" | "malicious" | "unknown" + ), + "invalid malware-policy observation" + ), ObservationKind::Canonical | ObservationKind::JsonLdSameAs | ObservationKind::Sitemap => {} } Ok(()) } + +/// Immutable declaration for one JSONL observation batch. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct BatchManifest { + /// `argand.site-observation-source/v1`. + pub schema: String, + /// Producer or capture collection. + pub source: String, + /// HTTPS documentation for the observation source. + pub source_url: String, + /// Rights declaration for the emitted observation metadata. + pub license: String, + /// HTTPS evidence for the rights declaration. + pub license_url: String, + /// Retrieval time of the batch. + pub retrieved_at: DateTime, + /// SHA-256 over exact JSONL bytes. + pub sha256: String, + /// Exact JSONL byte length. + pub bytes: u64, +} + +impl BatchManifest { + /// Validates bounded source, rights, and object identity fields. + /// + /// # Errors + /// Rejects unsupported, malformed, non-HTTPS, empty, or oversized declarations. + pub fn validate(&self) -> anyhow::Result<()> { + ensure!( + self.schema == "argand.site-observation-source/v1", + "unsupported observation batch schema" + ); + ensure!( + !self.source.trim().is_empty() && self.source.len() <= 128, + "observation batch source is required and bounded" + ); + for value in [&self.source_url, &self.license_url] { + let url = url::Url::parse(value)?; + ensure!( + url.scheme() == "https" && url.host_str().is_some(), + "observation documentation URLs must use HTTPS" + ); + } + ensure!( + !self.license.trim().is_empty() + && self.license.len() <= 128 + && self.bytes > 0 + && self.bytes <= MAXIMUM_BATCH_BYTES + && crate::model::valid_digest(&self.sha256), + "invalid observation rights or object identity" + ); + Ok(()) + } + + /// Stable identity over exact manifest fields. + /// + /// # Errors + /// Returns validation or serialization errors. + pub fn id(&self) -> anyhow::Result { + self.validate()?; + Ok(crate::digest(&serde_json::to_vec(self)?)) + } +} + +/// One JSONL record binding an observation to a granular review subject. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Assertion { + /// Name, edge, or equivalence fingerprint being observed. + pub subject_kind: crate::policy::SubjectKind, + /// Exact material subject fingerprint. + pub subject_fingerprint: String, + /// Source-native observation evidence. + pub observation: Observation, +} + +/// Imports a complete pinned JSONL batch against one verified candidate generation. +/// +/// # Errors +/// Rejects altered/oversized batches, stale subjects, rights mismatches, malformed +/// records, duplicate conflicts, and SQLite/filesystem failures. +#[allow(clippy::too_many_lines)] // Streaming validation and append share one transaction boundary. +pub fn import( + db: &Connection, + registry: &crate::query::Registry, + manifest: &BatchManifest, + path: &Path, +) -> anyhow::Result { + manifest.validate()?; + let id = manifest.id()?; + let mut file = crate::generation::open_no_follow(path)?; + ensure!( + file.metadata()?.is_file() && file.metadata()?.len() == manifest.bytes, + "observation batch length/type mismatch" + ); + let mut hash = Sha256::new(); + let observed = std::io::copy(&mut file, &mut HashWriter(&mut hash))?; + ensure!( + observed == manifest.bytes && format!("{:x}", hash.finalize()) == manifest.sha256, + "observation batch digest mismatch" + ); + let existing: Option = db + .query_row( + "SELECT complete FROM observation_batches WHERE id=?1", + [&id], + |row| row.get(0), + ) + .optional()?; + if existing == Some(true) { + return Ok(id); + } + ensure!( + existing.is_none(), + "incomplete observation batch is present" + ); + file.rewind()?; + let normalizer = registry.normalizer()?; + db.execute_batch("BEGIN IMMEDIATE")?; + let result = (|| { + db.execute( + "INSERT INTO observation_batches(id,source,retrieved_at,manifest_json) VALUES(?1,?2,?3,?4)", + params![id, manifest.source, manifest.retrieved_at.to_rfc3339(), serde_json::to_string(manifest)?], + )?; + let mut reader = BufReader::new(file); + let mut line = Vec::new(); + let mut records = 0_u64; + loop { + line.clear(); + let read = reader + .by_ref() + .take(u64::try_from(MAXIMUM_RECORD_BYTES)? + 1) + .read_until(b'\n', &mut line)?; + if read == 0 { + break; + } + ensure!( + line.len() <= MAXIMUM_RECORD_BYTES, + "observation record exceeds 1 MiB" + ); + if line.iter().all(u8::is_ascii_whitespace) { + continue; + } + records = records + .checked_add(1) + .context("observation record overflow")?; + ensure!( + records <= MAXIMUM_RECORDS, + "observation record cap exceeded" + ); + let assertion: Assertion = serde_json::from_value(crate::json::parse(&line)?)?; + validate_subject(registry, &assertion)?; + ensure!( + assertion.observation.source == manifest.source + && assertion.observation.license == manifest.license + && assertion.observation.license_url == manifest.license_url, + "observation record disagrees with batch rights/source" + ); + let normalized_observation = assertion.observation.normalize(&normalizer)?; + let document = serde_json::to_string(&normalized_observation)?; + db.execute( + "INSERT INTO observations VALUES(?1,?2,?3,?4,?5)", + params![ + normalized_observation.fingerprint, + id, + assertion.subject_kind.key(), + assertion.subject_fingerprint, + document + ], + ) + .context("duplicate observation fingerprint")?; + } + ensure!(records > 0, "observation batch is empty"); + db.execute( + "UPDATE observation_batches SET records=?2,complete=1 WHERE id=?1", + params![id, i64::try_from(records)?], + )?; + Ok(records) + })(); + match result { + Ok(_) => db.execute_batch("COMMIT")?, + Err(error) => { + db.execute_batch("ROLLBACK")?; + return Err(error); + } + } + Ok(id) +} + +fn validate_subject( + registry: &crate::query::Registry, + assertion: &Assertion, +) -> anyhow::Result<()> { + ensure!( + crate::model::valid_digest(&assertion.subject_fingerprint), + "invalid observation subject fingerprint" + ); + let (table, column) = match assertion.subject_kind { + crate::policy::SubjectKind::Name => ("names", "fingerprint"), + crate::policy::SubjectKind::Edge => ("edges", "fingerprint"), + crate::policy::SubjectKind::Equivalence => ("equivalences", "fingerprint"), + }; + let exists: bool = registry.db.query_row( + &format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE {column}=?1)"), + [&assertion.subject_fingerprint], + |row| row.get(0), + )?; + ensure!( + exists, + "observation subject is absent from candidate generation" + ); + Ok(()) +} + +struct HashWriter<'a>(&'a mut Sha256); + +impl std::io::Write for HashWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Bounded observation evidence attached to one review subject. +#[derive(Debug, Serialize)] +pub struct Lookup { + /// Exact subject fingerprint. + pub subject_fingerprint: String, + /// Complete matching count before the output limit. + pub total: u64, + /// True when records were omitted by the output limit. + pub truncated: bool, + /// Normalized observations with their batch manifests. + pub observations: Vec, +} + +/// Reverse observation lookup for an exact URL, hostname, or registrable domain. +#[derive(Debug, Serialize)] +pub struct ReverseLookup { + /// Original lookup target. + pub input: String, + /// Strict normalized URL or domain. + pub normalized: serde_json::Value, + /// Complete matching observation count. + pub total: u64, + /// True when results were omitted by the output limit. + pub truncated: bool, + /// Subject-bound observations with batch declarations. + pub observations: Vec, +} + +/// Returns observations for one exact review subject. +/// +/// # Errors +/// Rejects malformed fingerprints/limits or corrupt registry evidence. +pub fn lookup( + registry: &crate::query::Registry, + subject_fingerprint: &str, + limit: u32, +) -> anyhow::Result { + ensure!( + crate::model::valid_digest(subject_fingerprint), + "invalid observation subject fingerprint" + ); + ensure!( + (1..=1000).contains(&limit), + "observation limit must be 1..1000" + ); + let total = registry.db.query_row( + "SELECT count(*) FROM observations WHERE subject_fingerprint=?1", + [subject_fingerprint], + |row| crate::store::unsigned(row, 0), + )?; + let mut statement = registry.db.prepare( + "SELECT o.document_json,b.manifest_json FROM observations o JOIN observation_batches b ON b.id=o.batch_id WHERE o.subject_fingerprint=?1 ORDER BY o.fingerprint LIMIT ?2", + )?; + let rows = statement + .query_map(params![subject_fingerprint, limit], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .map(|row| { + let (observation, manifest) = row?; + Ok(serde_json::json!({ + "observation": serde_json::from_str::(&observation)?, + "batch": serde_json::from_str::(&manifest)?, + })) + }) + .collect::>>()?; + Ok(Lookup { + subject_fingerprint: subject_fingerprint.into(), + total, + truncated: total > u64::from(limit), + observations: rows, + }) +} + +/// Finds observations whose source or target matches an exact web target. +/// +/// # Errors +/// Rejects malformed inputs/limits or corrupt observation evidence. +pub fn reverse_lookup( + registry: &crate::query::Registry, + input: &str, + limit: u32, +) -> anyhow::Result { + ensure!( + (1..=1000).contains(&limit), + "observation limit must be 1..1000" + ); + let parser = registry.normalizer()?; + let (normalized, predicate, arguments) = if input.contains("://") { + let property = parser.url(input)?; + let arguments = vec![property.url.clone(), String::new()]; + ( + serde_json::to_value(property)?, + "json_extract(o.document_json,'$.from.url')=?1 OR json_extract(o.document_json,'$.to.url')=?1", + arguments, + ) + } else { + let domain = parser.domain(input)?; + let arguments = vec![domain.hostname.clone(), domain.registrable_domain.clone()]; + ( + serde_json::to_value(domain)?, + "json_extract(o.document_json,'$.from.domain.hostname')=?1 OR json_extract(o.document_json,'$.to.domain.hostname')=?1 OR json_extract(o.document_json,'$.from.domain.registrable_domain')=?2 OR json_extract(o.document_json,'$.to.domain.registrable_domain')=?2", + arguments, + ) + }; + let from = format!( + "FROM observations o JOIN observation_batches b ON b.id=o.batch_id WHERE {predicate}" + ); + let total = registry.db.query_row( + &format!("SELECT count(*) {from}"), + params![arguments[0], arguments[1]], + |row| crate::store::unsigned(row, 0), + )?; + let mut statement = registry.db.prepare(&format!( + "SELECT o.subject_kind,o.subject_fingerprint,o.document_json,b.manifest_json {from} ORDER BY o.subject_kind,o.subject_fingerprint,o.fingerprint LIMIT ?3" + ))?; + let rows = statement + .query_map(params![arguments[0], arguments[1], limit], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .map(|row| { + let (subject_kind, subject_fingerprint, observation, manifest) = row?; + Ok(serde_json::json!({ + "subject_kind":subject_kind, + "subject_fingerprint":subject_fingerprint, + "observation":serde_json::from_str::(&observation)?, + "batch":serde_json::from_str::(&manifest)?, + })) + }) + .collect::>>()?; + let result = ReverseLookup { + input: input.into(), + normalized, + total, + truncated: total > u64::from(limit), + observations: rows, + }; + let mut output = 0; + crate::query::account_output(&mut output, &result)?; + Ok(result) +} diff --git a/crates/argand-site-registry/src/observer.rs b/crates/argand-site-registry/src/observer.rs new file mode 100644 index 0000000..ac1244c --- /dev/null +++ b/crates/argand-site-registry/src/observer.rs @@ -0,0 +1,1243 @@ +// By Nic Weyand! +//! Bounded, DNS-pinned observation of already-imported website candidates. + +use crate::{ + observation::{Assertion, BatchManifest, Observation, ObservationKind}, + policy::SubjectKind, + query::Registry, +}; +use anyhow::{Context, ensure}; +use chrono::{DateTime, Utc}; +use reqwest::header; +use scraper::{Html, Selector}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeSet, + io::{Read, Write}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, + path::Path, + time::Duration, +}; +use url::Url; + +const MAXIMUM_HEADER_BYTES: usize = 64 * 1024; +const MAXIMUM_BODY_BYTES: u64 = 8 * 1024 * 1024; +const MAXIMUM_REDIRECTS: u8 = 5; +const MAXIMUM_EXTRACTED_LINKS: usize = 512; +const SOURCE: &str = "argand_candidate_observer"; +const SOURCE_URL: &str = "https://git.argand.org/nicweyand/argand-site-registry"; +const LICENSE: &str = "CC0-1.0"; +const LICENSE_URL: &str = "https://creativecommons.org/publicdomain/zero/1.0/"; + +/// Network and storage bounds for one candidate observation. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Options { + /// Maximum accepted response body bytes. + pub maximum_body_bytes: u64, + /// Maximum followed redirect hops. + pub maximum_redirects: u8, + /// Whole-request timeout for each hop. + pub timeout_seconds: u64, + /// Maximum final-response bytes accepted per second. + pub maximum_bytes_per_second: u64, +} + +impl Default for Options { + fn default() -> Self { + Self { + maximum_body_bytes: 2 * 1024 * 1024, + maximum_redirects: MAXIMUM_REDIRECTS, + timeout_seconds: 20, + maximum_bytes_per_second: 1024 * 1024, + } + } +} + +impl Options { + fn validate(&self) -> anyhow::Result<()> { + ensure!( + (1..=MAXIMUM_BODY_BYTES).contains(&self.maximum_body_bytes), + "observer body limit must be 1..8 MiB" + ); + ensure!( + self.maximum_redirects <= MAXIMUM_REDIRECTS, + "observer redirect limit exceeds five" + ); + ensure!( + (1..=120).contains(&self.timeout_seconds), + "observer timeout must be 1..120 seconds" + ); + ensure!( + (1024..=64 * 1024 * 1024).contains(&self.maximum_bytes_per_second), + "observer bandwidth limit must be 1 KiB/s..64 MiB/s" + ); + Ok(()) + } +} + +/// One safe, replayable candidate response capture. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Capture { + /// `argand.site-observer-capture/v1`. + pub schema: String, + /// Exact edge fingerprint that authorized the observation. + pub subject_fingerprint: String, + /// Start URL from the pinned candidate generation. + pub start_url: String, + /// Accepted capture time. + pub retrieved_at: DateTime, + /// Ordered HTTP hops actually requested. + pub hops: Vec, + /// Final bounded response body digest. + pub body_sha256: String, + /// Exact final body length. + pub body_bytes: u64, + /// Optional bounded failure class. + pub failure: Option, +} + +/// One requested HTTP hop with only the response metadata needed for replay. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Hop { + /// Strict normalized requested URL. + pub url: String, + /// HTTP response status. + pub status: u16, + /// Strict normalized followed redirect destination, if any. + pub redirect_to: Option, + /// Response content type without parameters. + pub content_type: Option, + /// Bounded HTTP Link header values. + pub link_headers: Vec, + /// Hash of the complete public address set pinned for this request. + pub addresses_sha256: Option, + /// Hash of the verified leaf certificate DER for HTTPS. + pub certificate_sha256: Option, +} + +impl Capture { + #[allow(clippy::too_many_lines)] // Replay checks every security-relevant capture field together. + fn validate(&self, registry: &Registry, body: &[u8]) -> anyhow::Result<()> { + ensure!( + self.schema == "argand.site-observer-capture/v1" + && crate::model::valid_digest(&self.subject_fingerprint) + && crate::model::valid_digest(&self.body_sha256), + "invalid observer capture identity" + ); + ensure!( + u64::try_from(body.len())? == self.body_bytes + && crate::digest(body) == self.body_sha256 + && self.body_bytes <= MAXIMUM_BODY_BYTES, + "observer body differs from capture" + ); + ensure!( + !self.hops.is_empty() && self.hops.len() <= usize::from(MAXIMUM_REDIRECTS) + 1, + "invalid observer hop count" + ); + ensure!( + registry.candidate(&self.subject_fingerprint)?.url == self.start_url, + "capture is not bound to the current candidate URL" + ); + let normalizer = registry.normalizer()?; + normalizer.url(&self.start_url)?; + let mut requested = BTreeSet::new(); + for (index, hop) in self.hops.iter().enumerate() { + let normalized_hop = normalizer.url(&hop.url)?; + let hop_url = Url::parse(&normalized_hop.url)?; + validate_outbound_url(&hop_url)?; + let expected = if index == 0 { + &self.start_url + } else { + self.hops[index - 1] + .redirect_to + .as_ref() + .context("capture hop is not connected to its predecessor")? + }; + ensure!( + &hop.url == expected, + "capture redirect chain is discontinuous" + ); + ensure!( + requested.insert(&hop.url), + "capture repeats an already requested URL" + ); + ensure!((100..=599).contains(&hop.status), "invalid capture status"); + ensure!( + hop.addresses_sha256 + .as_ref() + .is_none_or(|digest| crate::model::valid_digest(digest)), + "invalid capture DNS digest" + ); + ensure!( + hop.certificate_sha256 + .as_ref() + .is_none_or(|digest| crate::model::valid_digest(digest)), + "invalid capture certificate digest" + ); + ensure!( + hop.link_headers.len() <= 64 + && hop.link_headers.iter().all(|value| value.len() <= 8192), + "capture Link headers are oversized" + ); + if let Some(target) = &hop.redirect_to { + ensure!( + matches!(hop.status, 301 | 302 | 303 | 307 | 308), + "capture redirect has a non-redirect status" + ); + let normalized_target = normalizer.url(target)?; + let target_url = Url::parse(&normalized_target.url)?; + validate_outbound_url(&target_url)?; + ensure!( + hop_url.scheme() != "https" || target_url.scheme() == "https", + "capture contains an HTTPS downgrade" + ); + if let Some(next) = self.hops.get(index + 1) { + ensure!( + &next.url == target, + "capture redirect target was not followed" + ); + } else { + ensure!( + self.failure.as_deref() == Some("policy_block"), + "terminal redirect needs a policy failure" + ); + } + } else { + ensure!( + index + 1 == self.hops.len(), + "non-final capture hop lacks a redirect" + ); + } + } + if let Some(class) = &self.failure { + validate_failure_class(class)?; + } else { + let final_hop = self.hops.last().context("capture has no final hop")?; + ensure!( + (200..=299).contains(&final_hop.status) + && final_hop.content_type.as_deref().is_some_and(|value| { + matches!(value, "text/html" | "application/xhtml+xml") + }), + "successful capture needs a final HTML response" + ); + } + Ok(()) + } +} + +/// Fetches one existing edge candidate and writes an immutable replay cache. +/// +/// # Errors +/// Rejects unapproved inputs, unsafe DNS/ports/redirects, oversized responses, +/// malformed HTTP metadata, existing cache paths, and filesystem failures. +#[allow(clippy::too_many_lines)] // The bounded redirect state machine is linear for auditability. +pub async fn capture( + registry: &Registry, + subject_fingerprint: &str, + directory: &Path, + options: &Options, +) -> anyhow::Result { + options.validate()?; + ensure!( + crate::model::valid_digest(subject_fingerprint), + "invalid edge fingerprint" + ); + ensure!(!directory.exists(), "observer capture path already exists"); + let candidate = registry.candidate(subject_fingerprint)?; + ensure!(candidate.eligible, "observer candidate is ineligible"); + let normalizer = registry.normalizer()?; + let mut current = Url::parse(&normalizer.url(&candidate.url)?.url)?; + let start_url = current.to_string(); + let retrieved_at = Utc::now(); + let mut hops = Vec::new(); + let mut body = Vec::new(); + let mut failure = None; + let mut requested = BTreeSet::new(); + + for redirect_count in 0..=options.maximum_redirects { + validate_outbound_url(¤t)?; + ensure!( + requested.insert(current.to_string()), + "observer redirect loop reached an already requested URL" + ); + let host = current.host_str().context("observer URL has no hostname")?; + let port = current + .port_or_known_default() + .context("observer URL has no supported port")?; + let addresses = match resolve_public(host, port).await { + Ok(addresses) => addresses, + Err(error) => { + failure = Some( + match error { + ResolveError::Forbidden => "policy_block", + ResolveError::Lookup | ResolveError::Empty => "dns", + } + .into(), + ); + hops.push(Hop { + url: current.to_string(), + status: 599, + redirect_to: None, + content_type: None, + link_headers: Vec::new(), + addresses_sha256: None, + certificate_sha256: None, + }); + break; + } + }; + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(options.timeout_seconds.min(30))) + .timeout(Duration::from_secs(options.timeout_seconds)) + .user_agent(concat!("argand-site-registry/", env!("CARGO_PKG_VERSION"))) + .tls_info(true) + .resolve_to_addrs(host, &addresses) + .build()?; + let response = match client + .get(current.clone()) + .header(header::ACCEPT, "text/html,application/xhtml+xml;q=0.9") + .header(header::ACCEPT_ENCODING, "identity") + .send() + .await + { + Ok(response) => response, + Err(error) => { + failure = Some(classify_request_error(&error).into()); + hops.push(Hop { + url: current.to_string(), + status: 599, + redirect_to: None, + content_type: None, + link_headers: Vec::new(), + addresses_sha256: Some(address_digest(&addresses)?), + certificate_sha256: None, + }); + break; + } + }; + validate_headers(response.headers())?; + let status = response.status(); + let certificate_sha256 = response + .extensions() + .get::() + .and_then(reqwest::tls::TlsInfo::peer_certificate) + .map(crate::digest); + let addresses_sha256 = address_digest(&addresses)?; + let content_type = content_type(response.headers()); + let content_encoding = response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .unwrap_or("identity") + .trim(); + let link_headers = response + .headers() + .get_all(header::LINK) + .iter() + .map(|value| value.to_str().map(str::to_owned)) + .collect::, _>>()?; + if status.is_redirection() { + let location = response.headers().get(header::LOCATION); + let Some(location) = location else { + hops.push(Hop { + url: current.to_string(), + status: status.as_u16(), + redirect_to: None, + content_type, + link_headers, + addresses_sha256: Some(addresses_sha256), + certificate_sha256, + }); + failure = Some("http_status".into()); + break; + }; + let target = location + .to_str() + .ok() + .and_then(|value| current.join(value).ok()) + .and_then(|value| normalizer.url(value.as_str()).ok()) + .and_then(|value| Url::parse(&value.url).ok()); + let Some(target) = target else { + hops.push(Hop { + url: current.to_string(), + status: status.as_u16(), + redirect_to: None, + content_type, + link_headers, + addresses_sha256: Some(addresses_sha256), + certificate_sha256, + }); + failure = Some("policy_block".into()); + break; + }; + if current.scheme() == "https" && target.scheme() != "https" { + hops.push(Hop { + url: current.to_string(), + status: status.as_u16(), + redirect_to: None, + content_type, + link_headers, + addresses_sha256: Some(addresses_sha256), + certificate_sha256, + }); + failure = Some("policy_block".into()); + break; + } + validate_outbound_url(&target)?; + hops.push(Hop { + url: current.to_string(), + status: status.as_u16(), + redirect_to: Some(target.to_string()), + content_type, + link_headers, + addresses_sha256: Some(addresses_sha256), + certificate_sha256, + }); + if requested.contains(target.as_str()) { + failure = Some("policy_block".into()); + break; + } + if redirect_count == options.maximum_redirects { + failure = Some("policy_block".into()); + break; + } + current = target; + continue; + } + let html = content_type + .as_deref() + .is_some_and(|value| matches!(value, "text/html" | "application/xhtml+xml")); + hops.push(Hop { + url: current.to_string(), + status: status.as_u16(), + redirect_to: None, + content_type, + link_headers, + addresses_sha256: Some(addresses_sha256), + certificate_sha256, + }); + if !status.is_success() { + failure = Some("http_status".into()); + break; + } + if !content_encoding.eq_ignore_ascii_case("identity") { + failure = Some("content_encoding".into()); + break; + } + if !html { + failure = Some("content_type".into()); + break; + } + if let Some(length) = response.content_length() + && length > options.maximum_body_bytes + { + failure = Some("size_limit".into()); + break; + } + let mut response = response; + let body_started = tokio::time::Instant::now(); + loop { + match response.chunk().await { + Ok(Some(chunk)) => { + let next = body + .len() + .checked_add(chunk.len()) + .context("body size overflow")?; + if u64::try_from(next)? > options.maximum_body_bytes { + body.clear(); + failure = Some("size_limit".into()); + break; + } + body.extend_from_slice(&chunk); + let minimum_nanos = u128::try_from(body.len())? + .checked_mul(1_000_000_000) + .context("observer bandwidth duration overflow")? + / u128::from(options.maximum_bytes_per_second); + let minimum_elapsed = Duration::from_nanos(u64::try_from(minimum_nanos)?); + if let Some(wait) = minimum_elapsed.checked_sub(body_started.elapsed()) { + tokio::time::sleep(wait).await; + } + } + Ok(None) => break, + Err(error) => { + body.clear(); + failure = Some(classify_request_error(&error).into()); + break; + } + } + } + break; + } + + let capture = Capture { + schema: "argand.site-observer-capture/v1".into(), + subject_fingerprint: subject_fingerprint.into(), + start_url, + retrieved_at, + hops, + body_sha256: crate::digest(&body), + body_bytes: u64::try_from(body.len())?, + failure, + }; + capture.validate(registry, &body)?; + std::fs::create_dir(directory)?; + let write_result = (|| { + argand_atomic::create_durable(&directory.join("BODY.bin"), &body)?; + argand_atomic::create_durable( + &directory.join("CAPTURE.json"), + &serde_json::to_vec_pretty(&capture)?, + )?; + Ok::<_, anyhow::Error>(()) + })(); + if let Err(error) = write_result { + let _ = std::fs::remove_dir_all(directory); + return Err(error); + } + Ok(capture) +} + +/// Replays a cache without network access and writes importable JSONL plus its manifest. +/// +/// # Errors +/// Rejects altered/stale caches, malformed HTML/metadata, existing output paths, or +/// filesystem failures. +pub fn replay( + registry: &Registry, + directory: &Path, + output: &Path, + manifest_output: &Path, +) -> anyhow::Result { + ensure!( + !output.exists() && !manifest_output.exists(), + "observer output already exists" + ); + let capture: Capture = crate::read_json(&directory.join("CAPTURE.json"))?; + let mut body = Vec::new(); + crate::generation::open_no_follow(&directory.join("BODY.bin"))? + .take(MAXIMUM_BODY_BYTES + 1) + .read_to_end(&mut body)?; + ensure!( + u64::try_from(body.len())? <= MAXIMUM_BODY_BYTES, + "observer cache body exceeds 8 MiB" + ); + capture.validate(registry, &body)?; + let assertions = assertions(registry, &capture, &body)?; + let mut jsonl = Vec::new(); + for assertion in assertions { + serde_json::to_writer(&mut jsonl, &assertion)?; // atomic-writes: allow in-memory buffer + jsonl.write_all(b"\n")?; + } + ensure!(!jsonl.is_empty(), "observer emitted no evidence"); + let manifest = BatchManifest { + schema: "argand.site-observation-source/v1".into(), + source: SOURCE.into(), + source_url: SOURCE_URL.into(), + license: LICENSE.into(), + license_url: LICENSE_URL.into(), + retrieved_at: capture.retrieved_at, + sha256: crate::digest(&jsonl), + bytes: u64::try_from(jsonl.len())?, + }; + manifest.validate()?; + argand_atomic::create_durable(output, &jsonl)?; + if let Err(error) = + argand_atomic::create_durable(manifest_output, &serde_json::to_vec_pretty(&manifest)?) + { + let _ = std::fs::remove_file(output); + return Err(error.into()); + } + Ok(manifest) +} + +fn assertions( + registry: &Registry, + capture: &Capture, + body: &[u8], +) -> anyhow::Result> { + let mut result = Vec::new(); + for (index, hop) in capture.hops.iter().enumerate() { + let source_identifier = source_identifier(capture, &format!("hop:{index}"))?; + result.push(assertion( + capture, + ObservationKind::HttpStatus { status: hop.status }, + &hop.url, + &hop.url, + &source_identifier, + "http:status", + 10_000, + )); + if let Some(addresses_sha256) = &hop.addresses_sha256 { + result.push(assertion( + capture, + ObservationKind::DnsResolution { + addresses_sha256: addresses_sha256.clone(), + }, + &hop.url, + &hop.url, + &source_identifier, + "dns:addresses", + 10_000, + )); + } + if let Some(certificate_sha256) = &hop.certificate_sha256 { + result.push(assertion( + capture, + ObservationKind::TlsCertificate { + certificate_sha256: certificate_sha256.clone(), + }, + &hop.url, + &hop.url, + &source_identifier, + "tls:leaf-certificate", + 10_000, + )); + } + if let Some(target) = &hop.redirect_to { + result.push(assertion( + capture, + ObservationKind::Redirect { status: hop.status }, + &hop.url, + target, + &source_identifier, + "http:location", + 10_000, + )); + } + for (header_index, target) in sitemap_links(&hop.url, &hop.link_headers)? { + result.push(assertion( + capture, + ObservationKind::Sitemap, + &hop.url, + &target, + &source_identifier, + &format!("http:link:{header_index}"), + 9000, + )); + } + } + if let Some(class) = &capture.failure { + let last = capture.hops.last().context("capture has no final hop")?; + result.push(assertion( + capture, + ObservationKind::FetchFailure { + class: class.clone(), + }, + &last.url, + &last.url, + &source_identifier(capture, "failure")?, + "observer:failure", + 10_000, + )); + } else { + let final_url = &capture.hops.last().context("capture has no final hop")?.url; + result.extend(parse_html(capture, final_url, body)?); + } + let normalizer = registry.normalizer()?; + let mut seen = BTreeSet::new(); + result.retain(|assertion| { + let normalized_observation = assertion.observation.normalize(&normalizer); + normalized_observation.is_ok_and(|observation| seen.insert(observation.fingerprint)) + }); + ensure!( + result.len() <= MAXIMUM_EXTRACTED_LINKS, + "observer evidence cap exceeded" + ); + Ok(result) +} + +#[allow(clippy::too_many_lines)] // Supported HTML evidence classes stay visible in one bounded parser. +fn parse_html(capture: &Capture, final_url: &str, body: &[u8]) -> anyhow::Result> { + let base = Url::parse(final_url)?; + let html = String::from_utf8_lossy(body); + let document = Html::parse_document(&html); + let link_selector = Selector::parse("link[href]") + .map_err(|error| anyhow::anyhow!("invalid built-in link selector: {error}"))?; + let hreflang_selector = Selector::parse("a[href][hreflang]") + .map_err(|error| anyhow::anyhow!("invalid built-in hreflang selector: {error}"))?; + let country_selector = Selector::parse("a[href][data-country]") + .map_err(|error| anyhow::anyhow!("invalid built-in country selector: {error}"))?; + let json_ld_selector = Selector::parse("script[type='application/ld+json']") + .map_err(|error| anyhow::anyhow!("invalid built-in JSON-LD selector: {error}"))?; + let mut result = Vec::new(); + for (index, element) in document.select(&link_selector).enumerate() { + let Some(href) = element.value().attr("href") else { + continue; + }; + let Some(target) = join_http(&base, href) else { + continue; + }; + let relations = element + .value() + .attr("rel") + .unwrap_or_default() + .split_ascii_whitespace() + .map(str::to_ascii_lowercase) + .collect::>(); + let (relation, selector, confidence) = if relations.contains("canonical") { + (ObservationKind::Canonical, "link[rel=canonical]", 9000) + } else if relations.contains("sitemap") { + (ObservationKind::Sitemap, "link[rel=sitemap]", 9000) + } else if let Some(locale) = element.value().attr("hreflang") { + let Some(locale) = normalized_locale(locale) else { + continue; + }; + ( + ObservationKind::Hreflang { locale }, + "link[rel=alternate][hreflang]", + 8500, + ) + } else { + continue; + }; + result.push(assertion( + capture, + relation, + final_url, + &target, + &source_identifier(capture, &format!("html:link:{index}"))?, + selector, + confidence, + )); + ensure!( + result.len() <= MAXIMUM_EXTRACTED_LINKS, + "observer evidence cap exceeded" + ); + } + for (index, element) in document.select(&hreflang_selector).enumerate() { + let Some(locale) = element.value().attr("hreflang").and_then(normalized_locale) else { + continue; + }; + let Some(target) = element + .value() + .attr("href") + .and_then(|href| join_http(&base, href)) + else { + continue; + }; + result.push(assertion( + capture, + ObservationKind::Hreflang { locale }, + final_url, + &target, + &source_identifier(capture, &format!("html:hreflang:{index}"))?, + "a[hreflang]", + 8000, + )); + ensure!( + result.len() <= MAXIMUM_EXTRACTED_LINKS, + "observer evidence cap exceeded" + ); + } + for (index, element) in document.select(&country_selector).enumerate() { + let Some(country) = element + .value() + .attr("data-country") + .map(str::to_ascii_uppercase) + .filter(|value| { + value.len() == 2 && value.bytes().all(|byte| byte.is_ascii_uppercase()) + }) + else { + continue; + }; + let Some(target) = element + .value() + .attr("href") + .and_then(|href| join_http(&base, href)) + else { + continue; + }; + result.push(assertion( + capture, + ObservationKind::CountrySelector { country }, + final_url, + &target, + &source_identifier(capture, &format!("html:country:{index}"))?, + "a[data-country]", + 7500, + )); + ensure!( + result.len() <= MAXIMUM_EXTRACTED_LINKS, + "observer evidence cap exceeded" + ); + } + let mut same_as = Vec::new(); + for element in document.select(&json_ld_selector) { + let text = element.text().collect::(); + if text.len() > 1024 * 1024 { + continue; + } + if let Ok(value) = serde_json::from_str::(&text) { + collect_same_as(&value, &mut same_as); + } + } + for (index, value) in same_as.into_iter().take(128).enumerate() { + let Some(target) = join_http(&base, &value) else { + continue; + }; + result.push(assertion( + capture, + ObservationKind::JsonLdSameAs, + final_url, + &target, + &source_identifier(capture, &format!("html:same-as:{index}"))?, + "json-ld:sameAs", + 8000, + )); + ensure!( + result.len() <= MAXIMUM_EXTRACTED_LINKS, + "observer evidence cap exceeded" + ); + } + Ok(result) +} + +fn assertion( + capture: &Capture, + relation: ObservationKind, + from_url: &str, + to_url: &str, + source_identifier: &str, + selector: &str, + confidence: u16, +) -> Assertion { + Assertion { + subject_kind: SubjectKind::Edge, + subject_fingerprint: capture.subject_fingerprint.clone(), + observation: Observation { + relation, + from_url: from_url.into(), + to_url: to_url.into(), + source: SOURCE.into(), + source_identifier: source_identifier.into(), + license: LICENSE.into(), + license_url: LICENSE_URL.into(), + retrieved_at: capture.retrieved_at, + content_sha256: capture.body_sha256.clone(), + selector: selector.into(), + confidence, + }, + } +} + +fn source_identifier(capture: &Capture, selector: &str) -> anyhow::Result { + Ok(crate::digest(&serde_json::to_vec(&( + "argand.site-observer-source/v1", + &capture.subject_fingerprint, + &capture.start_url, + capture.retrieved_at, + &capture.body_sha256, + selector, + ))?)) +} + +fn collect_same_as(value: &serde_json::Value, output: &mut Vec) { + if output.len() >= 128 { + return; + } + match value { + serde_json::Value::Array(values) => { + for value in values { + collect_same_as(value, output); + } + } + serde_json::Value::Object(values) => { + if let Some(value) = values.get("sameAs") { + match value { + serde_json::Value::String(value) => output.push(value.clone()), + serde_json::Value::Array(values) => output.extend( + values + .iter() + .filter_map(|value| value.as_str().map(str::to_owned)), + ), + _ => {} + } + } + for (key, value) in values { + if key != "sameAs" { + collect_same_as(value, output); + } + } + } + _ => {} + } +} + +fn sitemap_links(base: &str, headers: &[String]) -> anyhow::Result> { + let base = Url::parse(base)?; + let mut result = Vec::new(); + for (index, header) in headers.iter().enumerate() { + for value in header.split(',') { + let Some((target, parameters)) = value.trim().split_once(';') else { + continue; + }; + let relation = parameters.to_ascii_lowercase(); + if !relation.contains("rel=\"sitemap\"") && !relation.contains("rel=sitemap") { + continue; + } + let target = target.trim(); + if let Some(target) = target + .strip_prefix('<') + .and_then(|value| value.strip_suffix('>')) + && let Some(target) = join_http(&base, target) + { + result.push((index, target)); + } + } + } + Ok(result) +} + +fn normalized_locale(value: &str) -> Option { + let value = value.trim(); + (!value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')) + .then(|| value.to_owned()) +} + +fn join_http(base: &Url, value: &str) -> Option { + base.join(value) + .ok() + .filter(|target| matches!(target.scheme(), "http" | "https")) + .map(|mut target| { + target.set_fragment(None); + target.to_string() + }) +} + +fn validate_headers(headers: &header::HeaderMap) -> anyhow::Result<()> { + let bytes = headers.iter().try_fold(0_usize, |used, (name, value)| { + used.checked_add(name.as_str().len()) + .and_then(|used| used.checked_add(value.as_bytes().len())) + .context("response header size overflow") + })?; + ensure!( + bytes <= MAXIMUM_HEADER_BYTES, + "observer response headers exceed 64 KiB" + ); + Ok(()) +} + +fn content_type(headers: &header::HeaderMap) -> Option { + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .map(str::trim) + .map(str::to_ascii_lowercase) +} + +fn validate_outbound_url(url: &Url) -> anyhow::Result<()> { + ensure!( + matches!(url.scheme(), "http" | "https") + && url.username().is_empty() + && url.password().is_none(), + "observer URL scheme or credentials are forbidden" + ); + let port = url + .port_or_known_default() + .context("unsupported observer port")?; + ensure!( + (url.scheme() == "http" && port == 80) || (url.scheme() == "https" && port == 443), + "observer only permits default HTTP(S) ports" + ); + ensure!(url.host_str().is_some(), "observer URL has no hostname"); + Ok(()) +} + +#[derive(Debug)] +enum ResolveError { + Lookup, + Empty, + Forbidden, +} + +async fn resolve_public(host: &str, port: u16) -> Result, ResolveError> { + let addresses = tokio::net::lookup_host((host, port)) + .await + .map_err(|_| ResolveError::Lookup)? + .collect::>() + .into_iter() + .collect::>(); + if addresses.is_empty() { + return Err(ResolveError::Empty); + } + if !addresses.iter().all(|address| is_public_ip(address.ip())) { + return Err(ResolveError::Forbidden); + } + Ok(addresses) +} + +fn address_digest(addresses: &[SocketAddr]) -> anyhow::Result { + Ok(crate::digest(&serde_json::to_vec(&( + "argand.site-observer-addresses/v1", + addresses, + ))?)) +} + +fn is_public_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(ip) => is_public_v4(ip), + IpAddr::V6(ip) => is_public_v6(ip), + } +} + +fn is_public_v4(ip: Ipv4Addr) -> bool { + let octets = ip.octets(); + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_broadcast() + || ip.is_documentation() + || ip.is_unspecified() + || ip.is_multicast() + || octets[0] == 0 + || octets[0] >= 224 + || (octets[0] == 100 && (64..=127).contains(&octets[1])) + || (octets[0] == 192 && octets[1] == 0 && octets[2] == 0) + || (octets[0] == 192 && octets[1] == 88 && octets[2] == 99) + || (octets[0] == 198 && matches!(octets[1], 18 | 19))) +} + +fn is_public_v6(ip: Ipv6Addr) -> bool { + let segments = ip.segments(); + if let Some(ip) = ip.to_ipv4_mapped() { + return is_public_v4(ip); + } + !(ip.is_loopback() + || ip.is_unspecified() + || ip.is_multicast() + || (segments[0] & 0xfe00) == 0xfc00 + || (segments[0] & 0xffc0) == 0xfe80 + || (segments[0] & 0xffc0) == 0xfec0 + || (segments[..6] == [0, 0, 0, 0, 0, 0]) + || (segments[0] == 0x0064 && segments[1] == 0xff9b) + || (segments[0] == 0x0100 && segments[1..4] == [0, 0, 0]) + || (segments[0] == 0x2001 + && (matches!(segments[1], 0 | 2 | 3 | 0x0db8) + || (segments[1] == 4 && segments[2] == 0x0112) + || (0x10..=0x2f).contains(&segments[1]))) + || segments[0] == 0x2002) +} + +fn classify_request_error(error: &reqwest::Error) -> &'static str { + if error.is_timeout() { + "timeout" + } else if error.is_connect() { + let lower = error.to_string().to_ascii_lowercase(); + if lower.contains("tls") || lower.contains("certificate") { + "tls" + } else { + "connection" + } + } else if error + .status() + .is_some_and(|status| status.is_client_error()) + { + "http_status" + } else { + "connection" + } +} + +fn validate_failure_class(class: &str) -> anyhow::Result<()> { + ensure!( + matches!( + class, + "dns" + | "timeout" + | "tls" + | "connection" + | "http_status" + | "content_type" + | "content_encoding" + | "size_limit" + | "policy_block" + ), + "invalid observer failure class" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn blocks_non_public_networks() { + for ip in [ + "127.0.0.1", + "10.0.0.1", + "100.64.0.1", + "169.254.1.1", + "192.0.2.1", + "198.18.0.1", + "192.88.99.1", + "224.0.0.1", + "::1", + "::127.0.0.1", + "fc00::1", + "fe80::1", + "fec0::1", + "64:ff9b::7f00:1", + "64:ff9b:1::1", + "100::1", + "2001::1", + "2001:2::1", + "2001:10::1", + "2002:7f00:1::", + "2001:db8::1", + "::ffff:127.0.0.1", + ] { + let parsed: IpAddr = ip + .parse() + .unwrap_or_else(|error| unreachable!("fixture: {error}")); + assert!(!is_public_ip(parsed), "{ip} was accepted"); + } + assert!(is_public_ip(IpAddr::V4(Ipv4Addr::new(93, 184, 216, 34)))); + assert!(is_public_ip( + "2606:2800:220:1:248:1893:25c8:1946" + .parse() + .unwrap_or_else(|error| unreachable!("fixture: {error}")) + )); + } + + #[test] + fn collects_only_explicit_json_ld_same_as() { + let mut output = Vec::new(); + collect_same_as( + &serde_json::json!({"name":"Example","sameAs":["https://example.com/a"],"nested":{"sameAs":"https://example.org/"}}), + &mut output, + ); + assert_eq!(output, ["https://example.com/a", "https://example.org/"]); + } + + #[test] + fn parses_bounded_sitemap_link_headers() -> anyhow::Result<()> { + assert_eq!( + sitemap_links( + "https://example.com/", + &["; rel=\"sitemap\", ; rel=next".into()] + )?, + [(0, "https://example.com/sitemap.xml".into())] + ); + Ok(()) + } + + #[test] + fn rejects_credentials_protocols_and_nondefault_ports() { + for value in [ + "file:///etc/passwd", + "https://user@example.com/", + "https://example.com:8443/", + "http://example.com:443/", + ] { + assert!( + validate_outbound_url( + &Url::parse(value) + .unwrap_or_else(|error| { unreachable!("fixed URL fixture: {error}") }) + ) + .is_err() + ); + } + } + + #[test] + fn oversized_extracted_link_sets_fail_closed() { + use std::fmt::Write as _; + let links = (0..=MAXIMUM_EXTRACTED_LINKS).fold(String::new(), |mut links, index| { + write!(links, "") + .unwrap_or_else(|error| unreachable!("writing to a string failed: {error}")); + links + }); + let body = format!("{links}"); + let capture = Capture { + schema: "argand.site-observer-capture/v1".into(), + subject_fingerprint: crate::digest(b"subject"), + start_url: "https://example.com/".into(), + retrieved_at: Utc::now(), + hops: Vec::new(), + body_sha256: crate::digest(body.as_bytes()), + body_bytes: u64::try_from(body.len()) + .unwrap_or_else(|error| unreachable!("bounded fixture length: {error}")), + failure: None, + }; + assert!(parse_html(&capture, "https://example.com/", body.as_bytes()).is_err()); + } + + #[test] + fn malformed_and_mixed_encoding_markup_is_bounded_and_deterministic() -> anyhow::Result<()> { + let body = b"\xff"; + let capture = Capture { + schema: "argand.site-observer-capture/v1".into(), + subject_fingerprint: crate::digest(b"subject"), + start_url: "https://example.com/".into(), + retrieved_at: Utc::now(), + hops: Vec::new(), + body_sha256: crate::digest(body), + body_bytes: u64::try_from(body.len())?, + failure: None, + }; + let first = parse_html(&capture, "https://example.com/", body)?; + let second = parse_html(&capture, "https://example.com/", body)?; + assert_eq!(serde_json::to_vec(&first)?, serde_json::to_vec(&second)?); + assert!(first.iter().any(|assertion| { + assertion.observation.relation == ObservationKind::Canonical + && assertion.observation.to_url == "https://example.com/ok" + })); + Ok(()) + } + + #[test] + fn parser_mutation_corpus_is_deterministic_and_bounded() -> anyhow::Result<()> { + let seed = b""; + let mut state = 0x6d_5a_56_da_u64; + for case in 0..512_u64 { + let mut body = seed.to_vec(); + for _ in 0..=case % 17 { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1); + let index = usize::try_from(state % u64::try_from(body.len())?)?; + body[index] = u8::try_from((state >> 24) & 0xff)?; + } + if case % 11 == 0 { + body.truncate(usize::try_from(case % u64::try_from(body.len())?)?); + } + let capture = Capture { + schema: "argand.site-observer-capture/v1".into(), + subject_fingerprint: crate::digest(b"subject"), + start_url: "https://example.com/".into(), + retrieved_at: Utc::now(), + hops: Vec::new(), + body_sha256: crate::digest(&body), + body_bytes: u64::try_from(body.len())?, + failure: None, + }; + let first = parse_html(&capture, "https://example.com/", &body); + let second = parse_html(&capture, "https://example.com/", &body); + match (first, second) { + (Ok(first), Ok(second)) => { + ensure!( + first.len() <= MAXIMUM_EXTRACTED_LINKS + && serde_json::to_vec(&first)? == serde_json::to_vec(&second)?, + "mutated parser result was oversized or nondeterministic" + ); + } + (Err(first), Err(second)) => ensure!( + first.to_string() == second.to_string(), + "mutated parser error was nondeterministic" + ), + _ => anyhow::bail!("mutated parser changed success state"), + } + } + Ok(()) + } +} diff --git a/crates/argand-site-registry/src/policy.rs b/crates/argand-site-registry/src/policy.rs new file mode 100644 index 0000000..cac336a --- /dev/null +++ b/crates/argand-site-registry/src/policy.rs @@ -0,0 +1,287 @@ +// By Nic Weyand! +//! Versioned reviewer thresholds compiled by every registry consumer. + +use anyhow::ensure; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// Kind of immutable assertion receiving reviewer votes. +#[derive( + Clone, Copy, Debug, Deserialize, Serialize, clap::ValueEnum, Eq, Ord, PartialEq, PartialOrd, +)] +#[serde(rename_all = "snake_case")] +pub enum SubjectKind { + /// One normalized name or alias attached to one stable source entity. + Name, + /// One source-asserted entity-to-web-property relationship. + Edge, + /// One explicit equivalence between stable source entity IDs. + Equivalence, +} + +impl SubjectKind { + /// Stable database and JSON spelling. + #[must_use] + pub const fn key(self) -> &'static str { + match self { + Self::Name => "name", + Self::Edge => "edge", + Self::Equivalence => "equivalence", + } + } +} + +/// Independent reviewer and reviewer-group threshold for one subject kind. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct Threshold { + /// Distinct authenticated reviewer identities required. + pub approvals: u16, + /// Distinct configured groups required; unmapped identities form their own group. + pub groups: u16, +} + +impl Threshold { + fn validate(self) -> anyhow::Result<()> { + ensure!( + (1..=32).contains(&self.approvals) && (1..=self.approvals).contains(&self.groups), + "invalid review threshold" + ); + Ok(()) + } +} + +/// Exact publisher policy authenticated by the generation receipt. +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +#[allow(clippy::struct_excessive_bools)] // Independent switches are authenticated public contract fields. +pub struct ReviewPolicy { + /// `argand.site-policy/v1`. + pub schema: String, + /// Human-readable bounded policy name. + pub name: String, + /// Name-to-entity decision threshold. + pub names: Threshold, + /// Entity-to-property decision threshold. + pub edges: Threshold, + /// Entity-equivalence decision threshold. + pub equivalences: Threshold, + /// A single authenticated revocation blocks use until explicitly superseded. + pub sticky_revocations: bool, + /// Release publisher identity may not supply a counted approval. + pub publisher_reviewer_separation: bool, + /// Whether name facts need votes before `resolve`; audit lookup is unaffected. + pub require_name_votes: bool, + /// Whether v0.3 legacy reviews may be used by the compatibility policy. + pub allow_legacy_reviews: bool, + /// Maximum effective approval age from trusted writer acceptance, at most 90 days. + pub maximum_approval_days: u16, + /// Whether an edge can qualify before any observation batch exists. + pub require_edge_observation: bool, + /// Optional maximum age of the latest edge observation. + pub maximum_observation_age_days: Option, + /// Whether a domain asserted for another entity places an approved edge on probation. + pub block_source_conflicts: bool, + /// Whether dangerous observer drift places an otherwise approved edge on probation. + pub block_dangerous_drift: bool, + /// Optional reviewer-to-independent-group mapping. + #[serde(default)] + pub reviewer_groups: BTreeMap, + /// Optional stricter thresholds for documented material risk classes. + #[serde(default)] + pub risk_thresholds: BTreeMap, +} + +impl ReviewPolicy { + /// Strict default used by the CLI and public [`crate::build::build`]. + #[must_use] + pub fn reference() -> Self { + let threshold = Threshold { + approvals: 2, + groups: 2, + }; + Self { + schema: "argand.site-policy/v1".into(), + name: "argand-reference-v1".into(), + names: threshold, + edges: threshold, + equivalences: threshold, + sticky_revocations: true, + publisher_reviewer_separation: true, + require_name_votes: true, + allow_legacy_reviews: false, + maximum_approval_days: 90, + require_edge_observation: false, + maximum_observation_age_days: Some(30), + block_source_conflicts: true, + block_dangerous_drift: true, + reviewer_groups: BTreeMap::new(), + risk_thresholds: BTreeMap::new(), + } + } + + /// Explicit compatibility policy for migration/testing; never the CLI default. + #[must_use] + pub fn legacy_compatible() -> Self { + let threshold = Threshold { + approvals: 1, + groups: 1, + }; + Self { + schema: "argand.site-policy/v1".into(), + name: "legacy-v0.3-compatibility".into(), + names: threshold, + edges: threshold, + equivalences: threshold, + sticky_revocations: true, + publisher_reviewer_separation: false, + require_name_votes: false, + allow_legacy_reviews: true, + maximum_approval_days: 90, + require_edge_observation: false, + maximum_observation_age_days: None, + block_source_conflicts: false, + block_dangerous_drift: false, + reviewer_groups: BTreeMap::new(), + risk_thresholds: BTreeMap::new(), + } + } + + /// Validates bounded thresholds and identity/group declarations. + /// + /// # Errors + /// Rejects unsupported schemas, unsafe identifiers, and inconsistent policy. + pub fn validate(&self) -> anyhow::Result<()> { + ensure!( + self.schema == "argand.site-policy/v1", + "unsupported review policy" + ); + ensure!( + !self.name.trim().is_empty() && self.name.len() <= 256, + "review policy name is required and bounded" + ); + self.names.validate()?; + self.edges.validate()?; + self.equivalences.validate()?; + ensure!( + (1..=90).contains(&self.maximum_approval_days) + && self + .maximum_observation_age_days + .is_none_or(|days| (1..=365).contains(&days)), + "invalid review or observation age policy" + ); + ensure!( + self.sticky_revocations || self.allow_legacy_reviews, + "new policies must preserve sticky revocations" + ); + ensure!( + self.reviewer_groups.len() <= 4096 + && self.reviewer_groups.iter().all(|(reviewer, group)| { + !reviewer.trim().is_empty() + && reviewer.len() <= 256 + && !group.trim().is_empty() + && group.len() <= 256 + }), + "invalid reviewer-group mapping" + ); + ensure!( + self.risk_thresholds.len() <= 16 + && self.risk_thresholds.iter().all(|(risk, threshold)| { + matches!(risk.as_str(), "source_conflict" | "dangerous_drift") + && threshold.validate().is_ok() + }), + "invalid risk-threshold mapping" + ); + Ok(()) + } + + /// Content identity authenticated by a generation receipt. + /// + /// # Errors + /// Returns validation or serialization errors. + pub fn id(&self) -> anyhow::Result { + self.validate()?; + Ok(crate::digest(&serde_json::to_vec(self)?)) + } + + /// Threshold for an assertion kind. + #[must_use] + pub const fn threshold(&self, kind: SubjectKind) -> Threshold { + match kind { + SubjectKind::Name => self.names, + SubjectKind::Edge => self.edges, + SubjectKind::Equivalence => self.equivalences, + } + } + + /// Raises the subject threshold for every current material risk class. + #[must_use] + pub fn threshold_for<'a>( + &self, + kind: SubjectKind, + risks: impl Iterator, + ) -> Threshold { + risks.fold(self.threshold(kind), |current, risk| { + self.risk_thresholds + .get(risk) + .map_or(current, |extra| Threshold { + approvals: current.approvals.max(extra.approvals), + groups: current.groups.max(extra.groups), + }) + }) + } + + /// Counts distinct policy groups for reviewer identities. + #[must_use] + pub fn group_count<'a>(&self, reviewers: impl Iterator) -> usize { + reviewers + .map(|reviewer| { + self.reviewer_groups + .get(reviewer) + .map_or(reviewer, String::as_str) + }) + .collect::>() + .len() + } +} + +impl Default for ReviewPolicy { + fn default() -> Self { + Self::legacy_compatible() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reference_policy_requires_two_independent_identities() -> anyhow::Result<()> { + let policy = ReviewPolicy::reference(); + policy.validate()?; + let mut custom = policy.clone(); + custom.risk_thresholds.insert( + "dangerous_drift".into(), + Threshold { + approvals: 3, + groups: 2, + }, + ); + assert_eq!( + custom.threshold_for(SubjectKind::Edge, ["dangerous_drift"].into_iter()), + Threshold { + approvals: 3, + groups: 2 + } + ); + custom.validate()?; + assert_eq!(policy.group_count(["one", "two"].into_iter()), 2); + let mut same_group = policy; + same_group.reviewer_groups = BTreeMap::from([ + ("one".into(), "organization".into()), + ("two".into(), "organization".into()), + ]); + assert_eq!(same_group.group_count(["one", "two"].into_iter()), 1); + Ok(()) + } +} diff --git a/crates/argand-site-registry/src/query.rs b/crates/argand-site-registry/src/query.rs index 09e76aa..f59fdb9 100644 --- a/crates/argand-site-registry/src/query.rs +++ b/crates/argand-site-registry/src/query.rs @@ -63,6 +63,8 @@ pub struct Candidate { pub provenance: Vec, /// Latest review, including expiry and its own provenance declaration. pub review: Option, + /// Current policy result when this candidate was selected by `resolve`. + pub policy_decision: Option, /// Whether this assertion can be considered for approval. pub eligible: bool, } @@ -232,6 +234,7 @@ impl Registry { evidence: serde_json::from_str(&evidence)?, provenance, review, + policy_decision: None, eligible, }; let mut output_bytes = 0; diff --git a/crates/argand-site-registry/src/queue.rs b/crates/argand-site-registry/src/queue.rs new file mode 100644 index 0000000..cca3fd5 --- /dev/null +++ b/crates/argand-site-registry/src/queue.rs @@ -0,0 +1,639 @@ +// By Nic Weyand! +//! Deterministic review queues and route-drift classification. + +use crate::{ + bundle::EvidenceBundle, + observation::{NormalizedObservation, ObservationKind}, + policy::SubjectKind, + query::Registry, + vote::{DecisionStatus, PolicyDecision}, +}; +use anyhow::{Context, ensure}; +use chrono::{DateTime, Utc}; +use rusqlite::params; +use serde::Serialize; +use std::collections::{BTreeMap, BTreeSet}; + +const MAXIMUM_QUEUE_LIMIT: u32 = 1000; +const MAXIMUM_SCANNED_SUBJECTS: u32 = 100_000; + +/// Why a subject needs operator attention. +#[derive(Clone, Debug, Serialize, Eq, Ord, PartialEq, PartialOrd)] +#[serde(rename_all = "snake_case")] +pub enum QueueReason { + /// No current approval votes exist. + NewSubject, + /// The configured independent-review quorum is not met. + InsufficientQuorum, + /// A sticky revocation blocks the subject. + Revoked, + /// Current approvals expired. + Expired, + /// Votes refer to an earlier evidence bundle. + StaleEvidence, + /// Votes were made under a different review-policy epoch. + StalePolicy, + /// Current approvals conflict across scopes. + Disputed, + /// Approval quorum exists but policy has placed the subject on probation. + Probationary, + /// Approval expires within seven days. + ExpiringSoon, + /// Current sources disagree about destination scope. + SourceConflict, + /// An eligible website has not been observed. + MissingObservation, + /// Current observation evidence indicates a material route change. + MaterialDrift, +} + +/// One stable, risk-ordered unit of review work. +#[derive(Debug, Serialize)] +pub struct QueueItem { + /// Higher values appear first. + pub risk: u16, + /// Granular assertion type. + pub subject_kind: SubjectKind, + /// Material assertion identity. + pub fingerprint: String, + /// Stable owning entity where applicable. + pub entity_id: Option, + /// Human-readable name or URL. + pub display: String, + /// Deterministic reasons for inclusion. + pub reasons: Vec, + /// Current policy compilation. + pub policy_decision: PolicyDecision, + /// Exact current evidence a new vote must sign. + pub evidence_bundle: EvidenceBundle, + /// Website observation state for edge subjects. + pub drift: Option, +} + +/// Bounded deterministic queue result. +#[derive(Debug, Serialize)] +pub struct ReviewQueue { + /// Verified generation pin. + pub registry: String, + /// Exact evaluation time supplied by the caller. + pub at: DateTime, + /// Complete number of queueable subjects within the scan bound. + pub total: u64, + /// Whether queue items were omitted by the output limit. + pub truncated: bool, + /// Risk-ordered review work. + pub items: Vec, +} + +/// A material drift category derived only from retained observations. +#[derive(Clone, Debug, Serialize, Eq, Ord, PartialEq, PartialOrd)] +#[serde(rename_all = "snake_case")] +pub enum DriftClass { + /// No observations are attached to the subject. + Unobserved, + /// Latest observation completed without a classified change. + Healthy, + /// Latest capture recorded a transport, policy, or server failure. + Unreachable, + /// Latest redirect crosses the candidate's registrable domain. + CrossDomainRedirect, + /// Latest canonical crosses the candidate's registrable domain. + CrossDomainCanonical, + /// Redirect destinations differ from the preceding capture. + RedirectTargetChanged, + /// Canonical destinations differ from the preceding capture. + CanonicalTargetChanged, + /// Public DNS address-set hashes differ from the preceding capture. + DnsChanged, + /// Verified leaf-certificate hashes differ from the preceding capture. + TlsCertificateChanged, + /// Retrieved content hashes differ from the preceding capture. + ContentChanged, + /// A rights-reviewed domain source reports expiry risk or inactive state. + DomainExpiryIndicator, + /// An explicitly configured malware policy reports suspicious or malicious. + MalwarePolicyBlocked, +} + +/// Observation comparison for one exact website assertion. +#[derive(Clone, Debug, Serialize)] +pub struct DriftReport { + /// Exact edge fingerprint. + pub fingerprint: String, + /// Current candidate URL. + pub url: String, + /// Deterministic drift classes. + pub classes: Vec, + /// Latest complete observation batch identity. + pub latest_batch: Option, + /// Latest batch capture time. + pub latest_at: Option>, + /// Immediately preceding batch identity. + pub previous_batch: Option, + /// Material change should receive fresh review. + pub review_required: bool, + /// Conservative candidate for a signed emergency revocation. + pub revocation_candidate: bool, +} + +#[derive(Default)] +struct Snapshot { + id: String, + at: Option>, + failures: BTreeSet, + redirect_targets: BTreeSet, + canonical_targets: BTreeSet, + dns: BTreeSet, + certificates: BTreeSet, + content: BTreeSet, + domain_expiry_risk: bool, + malware_policy_blocked: bool, +} + +/// Builds a read-only review queue under the generation's authenticated policy. +/// +/// # Errors +/// Rejects unsafe limits and corrupt subject, vote, or observation evidence. +pub fn review_queue( + registry: &Registry, + at: DateTime, + limit: u32, + maximum_subjects: u32, +) -> anyhow::Result { + ensure!( + (1..=MAXIMUM_QUEUE_LIMIT).contains(&limit), + "review queue limit must be 1..1000" + ); + ensure!( + (1..=MAXIMUM_SCANNED_SUBJECTS).contains(&maximum_subjects), + "review queue scan bound must be 1..100000" + ); + let mut items = Vec::new(); + let mut scanned = 0_u32; + scan_names(registry, at, maximum_subjects, &mut scanned, &mut items)?; + scan_edges(registry, at, maximum_subjects, &mut scanned, &mut items)?; + scan_equivalences(registry, at, maximum_subjects, &mut scanned, &mut items)?; + items.sort_by(|left, right| { + right + .risk + .cmp(&left.risk) + .then_with(|| left.subject_kind.cmp(&right.subject_kind)) + .then_with(|| left.fingerprint.cmp(&right.fingerprint)) + }); + let total = u64::try_from(items.len())?; + items.truncate(usize::try_from(limit)?); + let queue = ReviewQueue { + registry: registry.identity.clone(), + at, + total, + truncated: total > u64::from(limit), + items, + }; + let mut output = 0; + crate::query::account_output(&mut output, &queue)?; + Ok(queue) +} + +fn scan_names( + registry: &Registry, + at: DateTime, + maximum: u32, + scanned: &mut u32, + items: &mut Vec, +) -> anyhow::Result<()> { + if !registry.receipt.review_policy.require_name_votes || *scanned >= maximum { + return Ok(()); + } + let remaining = maximum - *scanned; + let mut statement = registry + .db + .prepare("SELECT fingerprint,entity,text,kind FROM names ORDER BY fingerprint LIMIT ?1")?; + let rows = statement + .query_map([remaining], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .collect::, _>>()?; + *scanned = scanned.saturating_add(u32::try_from(rows.len())?); + for (fingerprint, entity, text, kind) in rows { + let bundle = crate::bundle::build(registry, SubjectKind::Name, &fingerprint)?; + let decision = crate::vote::decision_for_bundle( + registry, + SubjectKind::Name, + &fingerprint, + &bundle.id, + at, + )?; + let (mut reasons, mut risk) = policy_reasons(&decision, at); + if reasons.is_empty() { + continue; + } + if kind == "alias" { + risk = risk.saturating_add(25); + } + reasons.sort(); + items.push(QueueItem { + risk, + subject_kind: SubjectKind::Name, + fingerprint, + entity_id: Some(entity), + display: text, + reasons, + policy_decision: decision, + evidence_bundle: bundle, + drift: None, + }); + } + Ok(()) +} + +fn scan_edges( + registry: &Registry, + at: DateTime, + maximum: u32, + scanned: &mut u32, + items: &mut Vec, +) -> anyhow::Result<()> { + if *scanned >= maximum { + return Ok(()); + } + let remaining = maximum - *scanned; + let mut statement = registry + .db + .prepare("SELECT fingerprint FROM edges WHERE eligible=1 ORDER BY fingerprint LIMIT ?1")?; + let rows = statement + .query_map([remaining], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + *scanned = scanned.saturating_add(u32::try_from(rows.len())?); + for fingerprint in rows { + let candidate = registry.candidate(&fingerprint)?; + let bundle = crate::bundle::build(registry, SubjectKind::Edge, &fingerprint)?; + let decision = crate::vote::decision_for_bundle( + registry, + SubjectKind::Edge, + &fingerprint, + &bundle.id, + at, + )?; + let drift = drift(registry, &fingerprint)?; + let (mut reasons, mut risk) = policy_reasons(&decision, at); + if domain_entity_conflict(registry, &candidate, at)? { + reasons.push(QueueReason::SourceConflict); + risk = risk.max(775); + } + if drift.classes == [DriftClass::Unobserved] { + reasons.push(QueueReason::MissingObservation); + risk = risk.max(300); + } else if drift.review_required { + reasons.push(QueueReason::MaterialDrift); + risk = risk.max(if drift.revocation_candidate { 975 } else { 850 }); + } + if reasons.is_empty() { + continue; + } + reasons.sort(); + reasons.dedup(); + items.push(QueueItem { + risk, + subject_kind: SubjectKind::Edge, + fingerprint, + entity_id: Some(candidate.entity_id), + display: candidate.url, + reasons, + policy_decision: decision, + evidence_bundle: bundle, + drift: Some(drift), + }); + } + Ok(()) +} + +pub(crate) fn domain_entity_conflict( + registry: &Registry, + candidate: &crate::query::Candidate, + at: DateTime, +) -> anyhow::Result { + let property: crate::normalize::WebProperty = + serde_json::from_value(candidate.web_property.clone())?; + let equivalent = crate::identity::expand(registry, &candidate.entity_id, at)?.map_or_else( + || std::collections::BTreeSet::from([candidate.entity_id.clone()]), + |(entities, _)| entities, + ); + let mut statement = registry.db.prepare( + "SELECT DISTINCT other.entity FROM edges other JOIN properties p ON p.id=other.property WHERE p.domain=?1 ORDER BY other.entity", + )?; + for entity in statement.query_map([property.domain.registrable_domain], |row| { + row.get::<_, String>(0) + })? { + if !equivalent.contains(&entity?) { + return Ok(true); + } + } + Ok(false) +} + +fn scan_equivalences( + registry: &Registry, + at: DateTime, + maximum: u32, + scanned: &mut u32, + items: &mut Vec, +) -> anyhow::Result<()> { + if *scanned >= maximum { + return Ok(()); + } + let remaining = maximum - *scanned; + let mut statement = registry.db.prepare( + "SELECT fingerprint,left_entity,right_entity FROM equivalences ORDER BY fingerprint LIMIT ?1", + )?; + let rows = statement + .query_map([remaining], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + })? + .collect::, _>>()?; + *scanned = scanned.saturating_add(u32::try_from(rows.len())?); + for (fingerprint, left, right) in rows { + let pair = crate::identity::propose(registry, &left, &right)?; + ensure!( + pair.fingerprint == fingerprint, + "stored equivalence identity is stale" + ); + let bundle = crate::bundle::equivalence(registry, &pair)?; + let decision = crate::vote::decision_for_bundle( + registry, + SubjectKind::Equivalence, + &fingerprint, + &bundle.id, + at, + )?; + let (mut reasons, risk) = policy_reasons(&decision, at); + if reasons.is_empty() { + continue; + } + reasons.sort(); + items.push(QueueItem { + risk, + subject_kind: SubjectKind::Equivalence, + fingerprint, + entity_id: None, + display: format!("{left} = {right}"), + reasons, + policy_decision: decision, + evidence_bundle: bundle, + drift: None, + }); + } + Ok(()) +} + +fn policy_reasons(decision: &PolicyDecision, at: DateTime) -> (Vec, u16) { + let mut reasons = Vec::new(); + let mut risk = 0; + match decision.status { + DecisionStatus::Approved => { + if decision + .scopes + .iter() + .any(|scope| scope.expires_at <= at + chrono::Duration::days(7)) + { + reasons.push(QueueReason::ExpiringSoon); + risk = 500; + } + } + DecisionStatus::Revoked => { + reasons.push(QueueReason::Revoked); + risk = 1000; + } + DecisionStatus::Expired => { + reasons.push(QueueReason::Expired); + risk = 825; + } + DecisionStatus::StaleEvidence => { + reasons.push(QueueReason::StaleEvidence); + risk = 900; + } + DecisionStatus::StalePolicy => { + reasons.push(QueueReason::StalePolicy); + risk = 925; + } + DecisionStatus::Disputed => { + reasons.push(QueueReason::Disputed); + risk = 950; + } + DecisionStatus::Probationary => { + reasons.push(QueueReason::Probationary); + risk = 925; + } + DecisionStatus::InsufficientReview => { + reasons.push(if decision.approvals == 0 { + QueueReason::NewSubject + } else { + QueueReason::InsufficientQuorum + }); + risk = 700; + } + } + (reasons, risk) +} + +/// Classifies observation changes for one exact edge without changing route state. +/// +/// # Errors +/// Rejects absent/non-edge fingerprints and corrupt observation evidence. +#[allow(clippy::too_many_lines)] // Drift classes share one explicit transition precedence. +pub fn drift(registry: &Registry, fingerprint: &str) -> anyhow::Result { + ensure!( + crate::model::valid_digest(fingerprint), + "invalid edge fingerprint" + ); + let candidate = registry.candidate(fingerprint)?; + let property: crate::normalize::WebProperty = + serde_json::from_value(candidate.web_property.clone())?; + let mut statement = registry.db.prepare( + "SELECT o.batch_id,b.retrieved_at,o.document_json FROM observations o JOIN observation_batches b ON b.id=o.batch_id WHERE o.subject_kind='edge' AND o.subject_fingerprint=?1 AND b.complete=1 ORDER BY b.retrieved_at,o.batch_id,o.fingerprint", + )?; + let mut snapshots: BTreeMap<(DateTime, String), Snapshot> = BTreeMap::new(); + let mut rows = statement.query([fingerprint])?; + while let Some(row) = rows.next()? { + let id: String = row.get(0)?; + let at = DateTime::parse_from_rfc3339(&row.get::<_, String>(1)?)?.with_timezone(&Utc); + let observation: NormalizedObservation = serde_json::from_str(&row.get::<_, String>(2)?)?; + let snapshot = snapshots.entry((at, id.clone())).or_default(); + snapshot.id = id; + snapshot.at = Some(at); + snapshot.content.insert(observation.content_sha256.clone()); + match observation.relation { + ObservationKind::FetchFailure { class } => { + snapshot.failures.insert(class); + } + ObservationKind::HttpStatus { status } if status >= 500 => { + snapshot.failures.insert(format!("http_{status}")); + } + ObservationKind::Redirect { .. } => { + snapshot.redirect_targets.insert(observation.to.url.clone()); + } + ObservationKind::Canonical => { + snapshot + .canonical_targets + .insert(observation.to.url.clone()); + } + ObservationKind::DnsResolution { addresses_sha256 } => { + snapshot.dns.insert(addresses_sha256); + } + ObservationKind::TlsCertificate { certificate_sha256 } => { + snapshot.certificates.insert(certificate_sha256); + } + ObservationKind::DomainRegistration { state, expires_at } => { + snapshot.domain_expiry_risk |= + matches!(state.as_str(), "expiry_risk" | "expired" | "redemption") + || expires_at + .is_some_and(|expiry| expiry <= at + chrono::Duration::days(30)); + } + ObservationKind::MalwarePolicy { result, .. } => { + snapshot.malware_policy_blocked |= + matches!(result.as_str(), "suspicious" | "malicious"); + } + _ => {} + } + } + let mut snapshots = snapshots.into_values().collect::>(); + let Some(latest) = snapshots.pop() else { + return Ok(DriftReport { + fingerprint: fingerprint.into(), + url: candidate.url, + classes: vec![DriftClass::Unobserved], + latest_batch: None, + latest_at: None, + previous_batch: None, + review_required: false, + revocation_candidate: false, + }); + }; + let previous = snapshots.last(); + let mut classes = BTreeSet::new(); + if !latest.failures.is_empty() { + classes.insert(DriftClass::Unreachable); + } + if contains_cross_domain( + registry, + &property.domain.registrable_domain, + &latest.redirect_targets, + )? { + classes.insert(DriftClass::CrossDomainRedirect); + } + if contains_cross_domain( + registry, + &property.domain.registrable_domain, + &latest.canonical_targets, + )? { + classes.insert(DriftClass::CrossDomainCanonical); + } + if let Some(previous) = previous { + if previous.redirect_targets != latest.redirect_targets { + classes.insert(DriftClass::RedirectTargetChanged); + } + if previous.canonical_targets != latest.canonical_targets { + classes.insert(DriftClass::CanonicalTargetChanged); + } + if !previous.dns.is_empty() && !latest.dns.is_empty() && previous.dns != latest.dns { + classes.insert(DriftClass::DnsChanged); + } + if !previous.certificates.is_empty() + && !latest.certificates.is_empty() + && previous.certificates != latest.certificates + { + classes.insert(DriftClass::TlsCertificateChanged); + } + if !previous.content.is_empty() + && !latest.content.is_empty() + && previous.content != latest.content + { + classes.insert(DriftClass::ContentChanged); + } + } + if latest.domain_expiry_risk { + classes.insert(DriftClass::DomainExpiryIndicator); + } + if latest.malware_policy_blocked { + classes.insert(DriftClass::MalwarePolicyBlocked); + } + if classes.is_empty() { + classes.insert(DriftClass::Healthy); + } + let revocation_candidate = classes.iter().any(|class| { + matches!( + class, + DriftClass::Unreachable + | DriftClass::CrossDomainRedirect + | DriftClass::CrossDomainCanonical + | DriftClass::DomainExpiryIndicator + | DriftClass::MalwarePolicyBlocked + ) + }); + let review_required = classes + .iter() + .any(|class| !matches!(class, DriftClass::Healthy | DriftClass::Unobserved)); + Ok(DriftReport { + fingerprint: fingerprint.into(), + url: candidate.url, + classes: classes.into_iter().collect(), + latest_batch: Some(latest.id), + latest_at: latest.at, + previous_batch: previous.map(|snapshot| snapshot.id.clone()), + review_required, + revocation_candidate, + }) +} + +fn contains_cross_domain( + registry: &Registry, + expected: &str, + targets: &BTreeSet, +) -> anyhow::Result { + let normalizer = registry.normalizer()?; + for target in targets { + let target = normalizer + .url(target) + .context("invalid stored observation URL")?; + if target.domain.registrable_domain != expected { + return Ok(true); + } + } + Ok(false) +} + +/// Returns all material emergency-revocation candidates within a bounded scan. +/// +/// # Errors +/// Rejects unsafe bounds and corrupt candidate/observation evidence. +pub fn revocation_candidates( + registry: &Registry, + maximum_subjects: u32, +) -> anyhow::Result> { + ensure!( + (1..=MAXIMUM_SCANNED_SUBJECTS).contains(&maximum_subjects), + "revocation scan bound must be 1..100000" + ); + let mut statement = registry + .db + .prepare("SELECT fingerprint FROM edges WHERE eligible=1 ORDER BY fingerprint LIMIT ?1")?; + let fingerprints = statement + .query_map(params![maximum_subjects], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + let mut reports = Vec::new(); + for fingerprint in fingerprints { + let report = drift(registry, &fingerprint)?; + if report.revocation_candidate { + reports.push(report); + } + } + Ok(reports) +} diff --git a/crates/argand-site-registry/src/release.rs b/crates/argand-site-registry/src/release.rs index 73120ef..9124d28 100644 --- a/crates/argand-site-registry/src/release.rs +++ b/crates/argand-site-registry/src/release.rs @@ -21,6 +21,7 @@ pub fn attribution() -> Value { "crux":{"license":"CC-BY-4.0","credit":"Chrome UX Report, Google","url":"https://developer.chrome.com/docs/crux/","license_url":"https://creativecommons.org/licenses/by/4.0/"}, "curlie":{"license":"CC-BY-3.0","credit":"With content from Curlie.org - the largest human-edited directory of the web. Contribute by submitting a website or becoming an editor.","url":"https://curlie.org/","license_url":"https://creativecommons.org/licenses/by/3.0/","public_display":"Use the prescribed HTML attribution on every page using Curlie content: https://curlie.org/docs/en/license.html"}, "psl":{"license":"MPL-2.0","url":"https://publicsuffix.org/list/","license_url":"https://mozilla.org/MPL/2.0/"}, + "argand_candidate_observer":{"license":"CC0-1.0","url":"https://git.argand.org/nicweyand/argand-site-registry","license_url":"https://creativecommons.org/publicdomain/zero/1.0/","scope":"locally authored observation metadata; captured page content is not redistributed"}, "changes":"Argand normalizes and combines assertions; provider endorsement is not implied."}) } @@ -35,23 +36,55 @@ pub fn export( include_descriptions: bool, ) -> anyhow::Result<()> { argand_atomic::create_durable_with(output, |file| { - export_inner(registry, file, include_descriptions).map_err(std::io::Error::other) + export_inner(registry, file, include_descriptions, ExportMode::Active) + .map_err(std::io::Error::other) })?; Ok(()) } +/// Exports retained active and superseded assertions for audit, never admission. +/// +/// # Errors +/// Returns existing-output, query, serialization, or filesystem errors. +pub fn export_audit( + registry: &Registry, + output: &Path, + include_descriptions: bool, +) -> anyhow::Result<()> { + argand_atomic::create_durable_with(output, |file| { + export_inner(registry, file, include_descriptions, ExportMode::Audit) + .map_err(std::io::Error::other) + })?; + Ok(()) +} + +#[derive(Clone, Copy)] +enum ExportMode { + Active, + Audit, +} + fn export_inner( registry: &Registry, file: &mut File, include_descriptions: bool, + mode: ExportMode, ) -> anyhow::Result<()> { let mut writer = BufWriter::new(file); writeln!( writer, "{}", - json!({"schema":"argand.site-export/v1","registry":registry.identity,"attribution":attribution(),"descriptions_included":include_descriptions}) + json!({"schema":"argand.site-export/v2","registry":registry.identity,"mode":match mode { ExportMode::Active => "active", ExportMode::Audit => "audit" },"rules":registry.receipt.rules,"coverage":{"schema":"argand.site-coverage-selection/v1","sha256":registry.receipt.coverage_sha256},"selected_sources":registry.receipt.sources.iter().map(crate::model::SourceManifest::id).collect::>>()?,"attribution_sha256":registry.receipt.attribution_sha256,"attribution":attribution(),"descriptions_included":include_descriptions}) )?; - let mut stmt=registry.db.prepare("SELECT f.id,f.subject,f.predicate,f.value,f.selector,f.confidence,s.manifest,r.native_id,f.source_id FROM facts f JOIN sources s ON s.id=f.source_id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE s.complete=1 ORDER BY f.id")?; + let query = match mode { + ExportMode::Active => { + "SELECT f.id,f.subject,f.predicate,f.value,f.selector,f.confidence,s.manifest,r.native_id,f.source_id,'active',NULL FROM facts f JOIN sources s ON s.id=f.source_id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal LEFT JOIN rejected j ON j.fact=f.id WHERE j.fact IS NULL ORDER BY f.id" + } + ExportMode::Audit => { + "SELECT f.id,f.subject,f.predicate,f.value,f.selector,f.confidence,s.manifest,r.native_id,f.source_id,CASE WHEN j.fact IS NOT NULL THEN 'rejected' WHEN a.source_id IS NULL THEN 'superseded' ELSE 'active' END,j.reason FROM facts f JOIN sources s ON s.id=f.source_id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal LEFT JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal LEFT JOIN rejected j ON j.fact=f.id WHERE s.complete=1 ORDER BY f.id" + } + }; + let mut stmt = registry.db.prepare(query)?; let mut rows = stmt.query([])?; while let Some(row) = rows.next()? { let predicate: String = row.get(2)?; @@ -65,12 +98,61 @@ fn export_inner( { object.remove("description"); } + let selection_state: String = row.get(9)?; + let source_manifest: Value = serde_json::from_str(&row.get::<_, String>(6)?)?; + let source_name = source_manifest["source"] + .as_str() + .context("source manifest has no source")?; + let native_id: String = row.get(7)?; + let source_id: String = row.get(8)?; + let replacement_source_ids = if matches!(mode, ExportMode::Audit) + && selection_state == "superseded" + { + let mut replacements = registry.db.prepare( + "SELECT DISTINCT active.source_id FROM active_records active JOIN records r ON r.source_id=active.source_id AND r.ordinal=active.ordinal JOIN sources s ON s.id=active.source_id WHERE s.source=?1 AND r.native_id=?2 AND active.source_id<>?3 ORDER BY active.source_id", + )?; + replacements + .query_map( + rusqlite::params![source_name, native_id, source_id], + |item| item.get::<_, String>(0), + )? + .collect::, _>>()? + } else { + Vec::new() + }; writeln!( writer, "{}", - json!({"type":"assertion","id":row.get::<_,String>(0)?,"subject":row.get::<_,String>(1)?,"predicate":predicate,"value":value,"selector":row.get::<_,String>(4)?,"confidence":row.get::<_,u16>(5)?,"source":serde_json::from_str::(&row.get::<_,String>(6)?)?,"source_identifier":row.get::<_,String>(7)?,"source_snapshot_id":row.get::<_,String>(8)?,"description_redacted":!include_descriptions && predicate=="category"}) + json!({"type":"assertion","selection_state":selection_state,"rejection_reason":row.get::<_,Option>(10)?,"replacement_source_ids":replacement_source_ids,"id":row.get::<_,String>(0)?,"subject":row.get::<_,String>(1)?,"predicate":predicate,"value":value,"selector":row.get::<_,String>(4)?,"confidence":row.get::<_,u16>(5)?,"source":source_manifest,"source_identifier":native_id,"source_snapshot_id":source_id,"description_redacted":!include_descriptions && predicate=="category"}) )?; } + drop(rows); + drop(stmt); + if matches!(mode, ExportMode::Audit) { + let mut tombstones = registry.db.prepare( + "SELECT s.source,r.native_id,r.source_id,a.precedence,s.manifest FROM active_records current JOIN records r ON r.source_id=current.source_id AND r.ordinal=current.ordinal JOIN sources s ON s.id=r.source_id JOIN selected_sources a ON a.id=r.source_id WHERE NOT EXISTS(SELECT 1 FROM facts f WHERE f.source_id=r.source_id AND f.ordinal=r.ordinal) ORDER BY s.source,r.native_id,r.source_id", + )?; + let mut rows = tombstones.query([])?; + while let Some(row) = rows.next()? { + let source: String = row.get(0)?; + let native_id: String = row.get(1)?; + let source_id: String = row.get(2)?; + let precedence: i64 = row.get(3)?; + let mut replaced = registry.db.prepare( + "SELECT f.id FROM records prior JOIN sources s ON s.id=prior.source_id JOIN selected_sources a ON a.id=prior.source_id JOIN facts f ON f.source_id=prior.source_id AND f.ordinal=prior.ordinal WHERE s.source=?1 AND prior.native_id=?2 AND a.precedence(0) + })? + .collect::, _>>()?; + writeln!( + writer, + "{}", + json!({"type":"tombstone","selection_state":"tombstoned","source":serde_json::from_str::(&row.get::<_,String>(4)?)?,"source_identifier":native_id,"source_snapshot_id":source_id,"replaces":replaced}) + )?; + } + } writer.flush()?; Ok(()) } @@ -84,17 +166,66 @@ pub fn sign( key: &Path, pin: &str, allowed_reviewers: &Path, +) -> anyhow::Result<()> { + sign_inner(generation, key, pin, allowed_reviewers, None) +} + +/// Signs a generation while enforcing publisher-reviewer separation for its identity. +/// +/// # Errors +/// Returns trust, policy, key, existing signature, or signing process failures. +pub fn sign_as( + generation: &Path, + key: &Path, + pin: &str, + allowed_reviewers: &Path, + publisher_identity: &str, +) -> anyhow::Result<()> { + ensure!( + !publisher_identity.trim().is_empty() && publisher_identity.len() <= 256, + "publisher identity is required and bounded" + ); + sign_inner( + generation, + key, + pin, + allowed_reviewers, + Some(publisher_identity), + ) +} + +fn sign_inner( + generation: &Path, + key: &Path, + pin: &str, + allowed_reviewers: &Path, + publisher_identity: Option<&str>, ) -> anyhow::Result<()> { let receipt = crate::ssh::sealed_input(&generation.join("COMPLETE.json"), 1024 * 1024)?; ensure!(crate::digest(&receipt.bytes) == pin, "receipt pin mismatch"); let registry = Registry::open(generation, pin)?; + verify_reviewer_trust(®istry, allowed_reviewers)?; crate::review::verify_all(®istry.db, allowed_reviewers)?; + crate::vote::verify_all(®istry.db, allowed_reviewers)?; + if registry.receipt.review_policy.publisher_reviewer_separation { + let identity = publisher_identity + .context("strict release policy requires the publisher identity before signing")?; + crate::vote::verify_publisher_separation(®istry, identity)?; + } crate::ssh::sign( "argand-site-registry", &receipt.bytes, key, &generation.join("COMPLETE.json.sig"), )?; + let signature_path = generation.join("COMPLETE.json.sig"); + let separation = crate::ssh::sealed_input(&signature_path, 64 * 1024).and_then(|signature| { + crate::vote::verify_publisher_key_separation(®istry, &signature.bytes) + }); + if let Err(error) = separation { + let _ = fs::remove_file(signature_path); + return Err(error); + } File::open(generation)?.sync_all()?; Ok(()) } @@ -121,10 +252,30 @@ pub fn verify_signed( identity, )?; let registry = Registry::open(generation, &pin)?; + verify_reviewer_trust(®istry, allowed_reviewers)?; crate::review::verify_all(®istry.db, allowed_reviewers)?; + crate::vote::verify_all(®istry.db, allowed_reviewers)?; + crate::vote::verify_publisher_separation(®istry, identity)?; + crate::vote::verify_publisher_key_separation(®istry, &signature.bytes)?; Ok(registry) } +fn verify_reviewer_trust(registry: &Registry, allowed_reviewers: &Path) -> anyhow::Result<()> { + if registry.receipt.reviewer_trust_sha256.is_empty() { + ensure!( + registry.receipt.review_policy.allow_legacy_reviews, + "strict release is missing a reviewer trust-root digest" + ); + return Ok(()); + } + let trust = crate::ssh::sealed_input(allowed_reviewers, 1024 * 1024)?; + ensure!( + crate::digest(&trust.bytes) == registry.receipt.reviewer_trust_sha256, + "reviewer trust root differs from generation receipt" + ); + Ok(()) +} + /// Activates a verified generation using one durable pointer. Refuses rollback /// past distributed revocations; rebuild old inputs with the current review log. /// @@ -149,7 +300,7 @@ pub fn activate( .truncate(false) .open(parent.join("activation.lock"))?; lock.try_lock().context("another activation is running")?; - let revocation: u64 = registry.db.query_row( + let legacy_revocation: u64 = registry.db.query_row( "SELECT coalesce(max(sequence),0) FROM reviews WHERE decision='revoke'", [], |r| crate::store::unsigned(r, 0), @@ -157,7 +308,7 @@ pub fn activate( if current.exists() { let previous: Value = crate::read_json(current)?; ensure!( - revocation + legacy_revocation >= previous["revocation_sequence"] .as_u64() .context("invalid current pointer")?, @@ -178,7 +329,7 @@ pub fn activate( argand_atomic::replace_durable( current, &serde_json::to_vec_pretty( - &json!({"schema":"argand.site-current/v1","generation":generation.canonicalize()?,"receipt_sha256":registry.identity,"signer":identity,"revocation_sequence":revocation}), + &json!({"schema":"argand.site-current/v2","generation":generation.canonicalize()?,"receipt_sha256":registry.identity,"signer":identity,"revocation_sequence":legacy_revocation,"vote_revocations_sha256":vote_revocations_sha256(®istry.db)?}), )?, )?; Ok(()) @@ -209,9 +360,46 @@ fn preserve_revocations(old: &Registry, new: &Registry) -> anyhow::Result<()> { "rollback would replace revocation history" ); } + let old_has_votes: bool = old.db.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='votes')", + [], + |row| row.get(0), + )?; + if old_has_votes { + let mut statement = old.db.prepare( + "SELECT id,document_json,accepted_at FROM votes WHERE decision='revoke' ORDER BY id", + )?; + let mut rows = statement.query([])?; + while let Some(row) = rows.next()? { + let id: String = row.get(0)?; + let expected: (Vec, String) = (row.get(1)?, row.get(2)?); + let found = new + .db + .query_row( + "SELECT document_json,accepted_at FROM votes WHERE id=?1 AND decision='revoke'", + [&id], + |current| Ok((current.get(0)?, current.get(1)?)), + ) + .context("rollback would discard an authenticated vote revocation")?; + ensure!( + found == expected, + "rollback would alter an authenticated vote revocation" + ); + } + } Ok(()) } +fn vote_revocations_sha256(db: &rusqlite::Connection) -> anyhow::Result { + use sha2::{Digest, Sha256}; + let mut hash = Sha256::new(); + let mut statement = db.prepare("SELECT id FROM votes WHERE decision='revoke' ORDER BY id")?; + for id in statement.query_map([], |row| row.get::<_, String>(0))? { + hash.update(id?.as_bytes()); + } + Ok(format!("{:x}", hash.finalize())) +} + /// Streams added/removed assertion fingerprints between two pinned generations. /// /// # Errors diff --git a/crates/argand-site-registry/src/resolution.rs b/crates/argand-site-registry/src/resolution.rs index f54ae09..d4a6333 100644 --- a/crates/argand-site-registry/src/resolution.rs +++ b/crates/argand-site-registry/src/resolution.rs @@ -19,6 +19,8 @@ pub enum ResolutionStatus { NoNameMatch, /// Matching source entities lack an active reviewed equivalence chain. AmbiguousIdentity, + /// Matching source names exist, but none has a current approval quorum. + NoActiveNameReview, /// Identity or edge expansion exceeded defensive bounds. SafetyLimitExceeded, /// Matching entities have no currently eligible website assertion. @@ -36,12 +38,24 @@ pub enum ResolutionStatus { pub struct ResolutionCounts { /// Source entities matching the normalized name or alias. pub matched_entities: u64, + /// Matching name assertions without a current approval quorum. + pub unapproved_names: u64, /// Assertion candidates examined after identity review. pub considered: u64, /// Candidates eligible for destination review. pub eligible: u64, /// Eligible candidates without any review entry. pub missing_review: u64, + /// Candidates whose votes do not satisfy the configured quorum. + pub insufficient_quorum: u64, + /// Candidates whose votes reference superseded evidence. + pub stale_evidence: u64, + /// Candidates whose votes were made under another policy epoch. + pub stale_policy: u64, + /// Candidates with conflicting current approval scopes. + pub disputed: u64, + /// Candidates held by observation or source-risk policy. + pub probationary: u64, /// Candidates whose latest decision is a revocation. pub revoked: u64, /// Approvals whose validity interval has ended. @@ -97,10 +111,60 @@ impl Registry { locale: Option<&str>, country: Option<&str>, now: DateTime, + ) -> anyhow::Result { + self.resolve_explained_inner(query, locale, country, now, None) + } + + /// Resolves with a verified emergency revocation overlay before a full + /// replacement generation has reached this consumer. + /// + /// # Errors + /// Returns incompatible feed, malformed request/data, or database failures. + pub fn resolve_explained_with_revocations( + &self, + query: &str, + locale: Option<&str>, + country: Option<&str>, + now: DateTime, + revocations: &crate::revocation::VerifiedRevocations, + ) -> anyhow::Result { + revocations.ensure_applicable(self, now)?; + self.resolve_explained_inner(query, locale, country, now, Some(revocations)) + } + + /// Returns only the destination after applying a verified emergency feed. + /// + /// # Errors + /// Returns incompatible feed, malformed request/data, or database failures. + pub fn resolve_with_revocations( + &self, + query: &str, + locale: Option<&str>, + country: Option<&str>, + now: DateTime, + revocations: &crate::revocation::VerifiedRevocations, + ) -> anyhow::Result> { + Ok(self + .resolve_explained_with_revocations(query, locale, country, now, revocations)? + .destination) + } + + fn resolve_explained_inner( + &self, + query: &str, + locale: Option<&str>, + country: Option<&str>, + now: DateTime, + revocations: Option<&crate::revocation::VerifiedRevocations>, ) -> anyhow::Result { let key = name_key(query)?; let mut counts = ResolutionCounts::default(); - let candidates = match crate::identity::candidate_search(self, query, now)? { + let candidates = match crate::identity::candidate_search_with_revocations( + self, + query, + now, + revocations, + )? { crate::identity::CandidateSearch::Ready { matched_entities, candidates, @@ -120,6 +184,16 @@ impl Registry { counts, )); } + crate::identity::CandidateSearch::NoActiveNameReview { matched_entities } => { + counts.matched_entities = matched_entities; + counts.unapproved_names = matched_entities; + return Ok(result( + key, + ResolutionStatus::NoActiveNameReview, + None, + counts, + )); + } crate::identity::CandidateSearch::SafetyLimitExceeded { matched_entities } => { counts.matched_entities = matched_entities; return Ok(result( @@ -130,12 +204,24 @@ impl Registry { )); } }; - let (status, destination) = choose(candidates, locale, country, now, &mut counts)?; + let (status, destination) = if self.receipt.review_policy.allow_legacy_reviews { + choose_legacy(candidates, locale, country, now, &mut counts)? + } else { + choose_policy( + self, + candidates, + locale, + country, + now, + revocations, + &mut counts, + )? + }; Ok(result(key, status, destination, counts)) } } -fn choose( +fn choose_legacy( candidates: Vec, locale: Option<&str>, country: Option<&str>, @@ -222,6 +308,119 @@ fn choose( }) } +#[allow(clippy::too_many_lines)] // Abstention counts and scope selection are one ordered decision pass. +fn choose_policy( + registry: &Registry, + candidates: Vec, + locale: Option<&str>, + country: Option<&str>, + now: DateTime, + revocations: Option<&crate::revocation::VerifiedRevocations>, + counts: &mut ResolutionCounts, +) -> anyhow::Result<(ResolutionStatus, Option)> { + let mut best = None; + let mut score = 0; + let mut ambiguous = false; + for mut candidate in candidates { + counts.considered += 1; + if !candidate.eligible { + continue; + } + counts.eligible += 1; + if revocations.is_some_and(|feed| { + feed.blocks(crate::policy::SubjectKind::Edge, &candidate.fingerprint) + }) { + counts.revoked += 1; + continue; + } + let decision = crate::vote::decision( + registry, + crate::policy::SubjectKind::Edge, + &candidate.fingerprint, + now, + )?; + match decision.status { + crate::vote::DecisionStatus::Revoked => { + counts.revoked += 1; + continue; + } + crate::vote::DecisionStatus::Expired => { + counts.expired += 1; + continue; + } + crate::vote::DecisionStatus::StaleEvidence => { + counts.stale_evidence += 1; + continue; + } + crate::vote::DecisionStatus::StalePolicy => { + counts.stale_policy += 1; + continue; + } + crate::vote::DecisionStatus::Disputed => { + counts.disputed += 1; + continue; + } + crate::vote::DecisionStatus::Probationary => { + counts.probationary += 1; + continue; + } + crate::vote::DecisionStatus::InsufficientReview => { + counts.insufficient_quorum += 1; + continue; + } + crate::vote::DecisionStatus::Approved => {} + } + let mut selected_score = 0; + for scope in &decision.scopes { + let matches_country = !scope.country.is_empty() + && country.is_some_and(|value| value.eq_ignore_ascii_case(&scope.country)); + let matches_locale = !scope.locale.is_empty() + && locale.is_some_and(|value| value.eq_ignore_ascii_case(&scope.locale)); + let current = if scope.role == "regional" { + if (!scope.country.is_empty() && !matches_country) + || (!scope.locale.is_empty() && !matches_locale) + { + continue; + } + 2 + u8::from(matches_country) + u8::from(matches_locale) + } else if scope.role == "primary" { + 1 + } else { + continue; + }; + selected_score = selected_score.max(current); + } + if selected_score == 0 { + counts.region_mismatch += 1; + continue; + } + counts.active_approvals += 1; + candidate.policy_decision = Some(decision); + if selected_score > score { + best = Some(candidate); + score = selected_score; + ambiguous = false; + } else if selected_score == score + && best + .as_ref() + .is_some_and(|prior: &Candidate| prior.url != candidate.url) + { + ambiguous = true; + } + } + Ok(if ambiguous { + (ResolutionStatus::AmbiguousDestination, None) + } else if let Some(candidate) = best { + (ResolutionStatus::Resolved, Some(candidate)) + } else if counts.eligible == 0 { + (ResolutionStatus::NoEligibleDestination, None) + } else if counts.region_mismatch > 0 { + (ResolutionStatus::RegionMismatch, None) + } else { + (ResolutionStatus::NoActiveReview, None) + }) +} + fn result( query: String, status: ResolutionStatus, diff --git a/crates/argand-site-registry/src/revocation.rs b/crates/argand-site-registry/src/revocation.rs new file mode 100644 index 0000000..39ec8b2 --- /dev/null +++ b/crates/argand-site-registry/src/revocation.rs @@ -0,0 +1,459 @@ +// By Nic Weyand! +//! Small publisher-signed revocation overlays for pinned offline consumers. + +use crate::{policy::SubjectKind, query::Registry, vote::DecisionStatus}; +use anyhow::{Context, ensure}; +use chrono::{DateTime, Utc}; +use rusqlite::OptionalExtension; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, fs, path::Path}; + +/// SSH signature namespace for exact revocation-feed JSON bytes. +pub const SIGNATURE_NAMESPACE: &str = "argand-site-registry-revocations"; +const MAXIMUM_FEED_LIFETIME_DAYS: i64 = 7; + +/// One granular subject with retained revocation identities across policy epochs. +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RevocationEntry { + /// Name, website edge, or explicit entity equivalence. + pub subject_kind: SubjectKind, + /// Stable material assertion fingerprint blocked by this entry. + pub fingerprint: String, + /// Every accepted revocation vote ID across retained policy epochs, sorted. + pub revocations: Vec, + /// Whether this subject remains blocked at the feed's effective time. + pub active: bool, + /// Fresh approval vote IDs that explicitly supersede every revocation. + pub superseding_votes: Vec, +} + +/// Complete cumulative emergency revocation state from one registry generation. +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RevocationFeed { + /// `argand.site-revocations/v1`. + pub schema: String, + /// Receipt digest of the generation that compiled this feed. + pub registry: String, + /// Registry derivation rules required by cached consumers. + pub rules: String, + /// Exact review-policy epoch under which entries were compiled. + pub policy_sha256: String, + /// Exact reviewer trust-root digest bound into the source generation. + pub reviewer_trust_sha256: String, + /// Explicit time used to evaluate acceptance and supersession. + pub effective_at: DateTime, + /// Artifact refresh deadline; revocation votes themselves never expire. + pub expires_at: DateTime, + /// Cumulative revocation subjects across policy epochs, including superseded history. + pub entries: Vec, +} + +impl RevocationFeed { + fn validate(&self) -> anyhow::Result<()> { + ensure!( + self.schema == "argand.site-revocations/v1", + "unsupported revocation feed" + ); + ensure!( + crate::model::valid_digest(&self.registry) + && crate::model::valid_digest(&self.policy_sha256) + && crate::model::valid_digest(&self.reviewer_trust_sha256), + "revocation feed needs registry, policy, and trust-root digests" + ); + ensure!( + self.rules == crate::store::RULE_VERSION, + "revocation feed rules are unsupported" + ); + ensure!( + self.expires_at > self.effective_at + && self.expires_at - self.effective_at + <= chrono::Duration::days(MAXIMUM_FEED_LIFETIME_DAYS), + "revocation feed lifetime must be at most seven days" + ); + ensure!( + self.entries.len() <= 100_000, + "revocation feed exceeds 100000 subjects" + ); + let mut previous = None; + let mut vote_ids = std::collections::BTreeSet::new(); + for entry in &self.entries { + ensure!( + crate::model::valid_digest(&entry.fingerprint) + && !entry.revocations.is_empty() + && entry.revocations.len() <= 256 + && entry.superseding_votes.len() <= 256 + && entry + .revocations + .iter() + .chain(&entry.superseding_votes) + .all(|id| crate::model::valid_digest(id)), + "invalid revocation entry" + ); + ensure!( + entry.revocations.windows(2).all(|pair| pair[0] < pair[1]) + && entry + .superseding_votes + .windows(2) + .all(|pair| pair[0] < pair[1]), + "revocation vote IDs must be sorted and unique" + ); + ensure!( + entry.active || !entry.superseding_votes.is_empty(), + "inactive revocation needs explicit superseding votes" + ); + let coordinate = (entry.subject_kind, entry.fingerprint.as_str()); + ensure!( + previous.is_none_or(|prior| prior < coordinate), + "revocation entries must be sorted and unique" + ); + previous = Some(coordinate); + for id in &entry.revocations { + ensure!( + vote_ids.insert(id), + "revocation vote ID reused across subjects" + ); + } + } + Ok(()) + } +} + +/// A feed whose exact bytes have been authenticated by a trusted publisher. +#[derive(Clone, Debug)] +pub struct VerifiedRevocations { + feed: RevocationFeed, + /// SHA-256 of the exact signed feed bytes. + pub sha256: String, + /// Allowed-signers identity that authenticated the feed. + pub publisher: String, +} + +impl VerifiedRevocations { + /// Returns the authenticated feed declaration. + #[must_use] + pub const fn feed(&self) -> &RevocationFeed { + &self.feed + } + + /// Whether an exact material subject is currently blocked. + #[must_use] + pub fn blocks(&self, subject_kind: SubjectKind, fingerprint: &str) -> bool { + self.feed + .entries + .binary_search_by(|entry| { + (entry.subject_kind, entry.fingerprint.as_str()).cmp(&(subject_kind, fingerprint)) + }) + .is_ok_and(|index| self.feed.entries[index].active) + } + + pub(crate) fn ensure_compatible(&self, registry: &Registry) -> anyhow::Result<()> { + ensure!( + self.feed.rules == registry.receipt.rules + && self.feed.policy_sha256 == registry.receipt.review_policy_sha256 + && self.feed.reviewer_trust_sha256 == registry.receipt.reviewer_trust_sha256, + "revocation feed is incompatible with this cached registry" + ); + Ok(()) + } + + pub(crate) fn ensure_applicable( + &self, + registry: &Registry, + now: DateTime, + ) -> anyhow::Result<()> { + self.ensure_compatible(registry)?; + ensure!( + self.feed.effective_at <= now + chrono::Duration::minutes(5), + "revocation feed effective time is in the future" + ); + ensure!(now < self.feed.expires_at, "revocation feed has expired"); + Ok(()) + } +} + +/// Creates a deterministic cumulative feed at an explicit evaluation time. +/// +/// # Errors +/// Rejects malformed retained votes, corrupt subjects, or existing output paths. +pub fn export( + registry: &Registry, + output: &Path, + effective_at: DateTime, +) -> anyhow::Result<()> { + let feed = compile(registry, effective_at)?; + argand_atomic::create_durable(output, &serde_json::to_vec_pretty(&feed)?)?; + Ok(()) +} + +/// Signs an exact feed after proving it matches the pinned source generation. +/// +/// # Errors +/// Rejects altered feeds, unverified reviewer state, publisher/reviewer conflicts, +/// existing output paths, and signing failures. +pub fn sign( + registry: &Registry, + input: &Path, + output: &Path, + key: &Path, + allowed_reviewers: &Path, + publisher: &str, +) -> anyhow::Result<()> { + ensure!( + !publisher.trim().is_empty() && publisher.len() <= 256, + "publisher identity is required and bounded" + ); + let bytes = crate::ssh::sealed_input(input, 16 * 1024 * 1024)?; + let feed: RevocationFeed = serde_json::from_value(crate::json::parse(&bytes.bytes)?)?; + feed.validate()?; + ensure!( + feed == compile(registry, feed.effective_at)?, + "revocation feed differs from its source generation" + ); + verify_reviewer_trust(registry, allowed_reviewers)?; + crate::review::verify_all(®istry.db, allowed_reviewers)?; + crate::vote::verify_all(®istry.db, allowed_reviewers)?; + crate::vote::verify_publisher_separation(registry, publisher)?; + crate::ssh::sign(SIGNATURE_NAMESPACE, &bytes.bytes, key, output)?; + let signature = crate::ssh::sealed_input(output, 64 * 1024)?; + if let Err(error) = crate::vote::verify_publisher_key_separation(registry, &signature.bytes) { + let _ = fs::remove_file(output); + return Err(error); + } + Ok(()) +} + +/// Verifies a publisher-signed feed for application to a compatible cached registry. +/// +/// # Errors +/// Rejects malformed bytes, untrusted signatures, incompatible registries, future +/// effective times, publisher/reviewer conflicts, or revocation rollback. +pub fn verify( + registry: &Registry, + input: &Path, + signature: &Path, + allowed_publishers: &Path, + publisher: &str, + now: DateTime, + previous: Option<&VerifiedRevocations>, +) -> anyhow::Result { + ensure!( + !publisher.trim().is_empty() && publisher.len() <= 256, + "publisher identity is required and bounded" + ); + let bytes = crate::ssh::sealed_input(input, 16 * 1024 * 1024)?; + let signature = crate::ssh::sealed_input(signature, 64 * 1024)?; + let publishers = crate::ssh::sealed_input(allowed_publishers, 1024 * 1024)?; + crate::ssh::verify( + SIGNATURE_NAMESPACE, + &bytes.bytes, + &signature.bytes, + &publishers.bytes, + publisher, + )?; + let feed: RevocationFeed = serde_json::from_value(crate::json::parse(&bytes.bytes)?)?; + feed.validate()?; + ensure!( + feed.effective_at <= now + chrono::Duration::minutes(5), + "revocation feed effective time is in the future" + ); + ensure!(now < feed.expires_at, "revocation feed has expired"); + crate::vote::verify_publisher_separation(registry, publisher)?; + crate::vote::verify_publisher_key_separation(registry, &signature.bytes)?; + let verified = VerifiedRevocations { + feed, + sha256: crate::digest(&bytes.bytes), + publisher: publisher.into(), + }; + verified.ensure_applicable(registry, now)?; + if verified.feed.registry == registry.identity { + ensure!( + verified.feed == compile(registry, verified.feed.effective_at)?, + "revocation feed differs from its exact source generation" + ); + } else { + ensure!( + verified + .feed + .entries + .iter() + .all(|entry| entry.active && entry.superseding_votes.is_empty()), + "cross-generation revocation feeds may only add blocks" + ); + } + if let Some(previous) = previous { + preserve(previous, &verified)?; + } + Ok(verified) +} + +fn preserve(previous: &VerifiedRevocations, current: &VerifiedRevocations) -> anyhow::Result<()> { + ensure!( + current.feed.effective_at >= previous.feed.effective_at + && current.feed.rules == previous.feed.rules + && current.feed.policy_sha256 == previous.feed.policy_sha256 + && current.feed.reviewer_trust_sha256 == previous.feed.reviewer_trust_sha256, + "revocation feed would roll back its compatibility epoch" + ); + let current_entries = current + .feed + .entries + .iter() + .map(|entry| ((entry.subject_kind, entry.fingerprint.as_str()), entry)) + .collect::>(); + for old in &previous.feed.entries { + let new = current_entries + .get(&(old.subject_kind, old.fingerprint.as_str())) + .context("revocation feed would discard a subject")?; + ensure!( + old.revocations + .iter() + .all(|id| new.revocations.binary_search(id).is_ok()), + "revocation feed would discard a vote" + ); + ensure!( + !old.active || new.active || !new.superseding_votes.is_empty(), + "active revocation disappeared without explicit supersession" + ); + } + Ok(()) +} + +fn compile(registry: &Registry, effective_at: DateTime) -> anyhow::Result { + ensure!( + !registry.receipt.review_policy.allow_legacy_reviews, + "v0.4 revocation feeds require authenticated vote policy" + ); + let mut subjects: BTreeMap<(SubjectKind, String), Vec> = BTreeMap::new(); + let mut statement = registry.db.prepare( + "SELECT subject_kind,fingerprint,id,accepted_at FROM votes WHERE decision='revoke' ORDER BY subject_kind,fingerprint,id", + )?; + let mut rows = statement.query([])?; + while let Some(row) = rows.next()? { + let accepted_at = + DateTime::parse_from_rfc3339(&row.get::<_, String>(3)?)?.with_timezone(&Utc); + if accepted_at > effective_at { + continue; + } + let kind = match row.get::<_, String>(0)?.as_str() { + "name" => SubjectKind::Name, + "edge" => SubjectKind::Edge, + "equivalence" => SubjectKind::Equivalence, + _ => anyhow::bail!("stored vote has unknown subject kind"), + }; + subjects + .entry((kind, row.get(1)?)) + .or_default() + .push(row.get(2)?); + } + let mut entries = Vec::with_capacity(subjects.len()); + for ((subject_kind, fingerprint), mut revocations) in subjects { + revocations.sort(); + revocations.dedup(); + let decision = current_decision(registry, subject_kind, &fingerprint, effective_at)?; + let active = decision + .as_ref() + .is_none_or(|decision| decision.status == DecisionStatus::Revoked); + let mut superseding_votes = if active { + Vec::new() + } else { + decision + .as_ref() + .into_iter() + .flat_map(|decision| &decision.scopes) + .flat_map(|scope| scope.votes.iter().cloned()) + .collect::>() + }; + superseding_votes.sort(); + superseding_votes.dedup(); + ensure!( + active || !superseding_votes.is_empty(), + "revocation compilation lost its superseding quorum" + ); + entries.push(RevocationEntry { + subject_kind, + fingerprint, + revocations, + active, + superseding_votes, + }); + } + let feed = RevocationFeed { + schema: "argand.site-revocations/v1".into(), + registry: registry.identity.clone(), + rules: registry.receipt.rules.clone(), + policy_sha256: registry.receipt.review_policy_sha256.clone(), + reviewer_trust_sha256: registry.receipt.reviewer_trust_sha256.clone(), + effective_at, + expires_at: effective_at + chrono::Duration::days(MAXIMUM_FEED_LIFETIME_DAYS), + entries, + }; + feed.validate()?; + Ok(feed) +} + +fn current_decision( + registry: &Registry, + subject_kind: SubjectKind, + fingerprint: &str, + at: DateTime, +) -> anyhow::Result> { + match subject_kind { + SubjectKind::Name => { + let exists: bool = registry.db.query_row( + "SELECT EXISTS(SELECT 1 FROM names WHERE fingerprint=?1)", + [fingerprint], + |row| row.get(0), + )?; + exists + .then(|| crate::vote::decision(registry, subject_kind, fingerprint, at)) + .transpose() + } + SubjectKind::Edge => { + let exists: bool = registry.db.query_row( + "SELECT EXISTS(SELECT 1 FROM edges WHERE fingerprint=?1)", + [fingerprint], + |row| row.get(0), + )?; + exists + .then(|| crate::vote::decision(registry, subject_kind, fingerprint, at)) + .transpose() + } + SubjectKind::Equivalence => { + let pair = registry + .db + .query_row( + "SELECT left_entity,right_entity FROM equivalences WHERE fingerprint=?1", + [fingerprint], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + pair.map(|(left, right)| { + let pair = crate::identity::propose(registry, &left, &right)?; + ensure!( + pair.fingerprint == fingerprint, + "stored equivalence fingerprint is stale" + ); + let bundle = crate::bundle::equivalence(registry, &pair)?; + crate::vote::decision_for_bundle( + registry, + subject_kind, + fingerprint, + &bundle.id, + at, + ) + }) + .transpose() + } + } +} + +fn verify_reviewer_trust(registry: &Registry, allowed_reviewers: &Path) -> anyhow::Result<()> { + let trust = crate::ssh::sealed_input(allowed_reviewers, 1024 * 1024)?; + ensure!( + crate::digest(&trust.bytes) == registry.receipt.reviewer_trust_sha256, + "reviewer trust root differs from generation receipt" + ); + Ok(()) +} diff --git a/crates/argand-site-registry/src/store.rs b/crates/argand-site-registry/src/store.rs index d3e340f..8a34a6d 100644 --- a/crates/argand-site-registry/src/store.rs +++ b/crates/argand-site-registry/src/store.rs @@ -17,7 +17,7 @@ use std::{ }; /// Adapter/normalization contract recorded in all generation identities. -pub const RULE_VERSION: &str = "argand.site-rules/v3"; +pub const RULE_VERSION: &str = "argand.site-rules/v4"; /// Whether a signed immutable generation uses a reader-compatible rule contract. #[must_use] @@ -26,7 +26,10 @@ pub fn supported_rule_version(version: &str) -> bool { } pub(crate) fn legacy_rule_version(version: &str) -> bool { - matches!(version, "argand.site-rules/v1" | "argand.site-rules/v2") + matches!( + version, + "argand.site-rules/v1" | "argand.site-rules/v2" | "argand.site-rules/v3" + ) } /// Opens or migrates the local assertion store with bounded page cache. @@ -46,20 +49,37 @@ pub fn open(path: &Path) -> anyhow::Result { db.execute_batch(include_str!("../migrations/001.sql"))?; db.execute_batch(include_str!("../migrations/002.sql"))?; db.execute_batch(include_str!("../migrations/003.sql"))?; + db.execute_batch(include_str!("../migrations/004.sql"))?; + db.execute_batch(include_str!("../migrations/005.sql"))?; db.execute_batch("COMMIT")?; } 1 => { db.execute_batch("BEGIN IMMEDIATE")?; db.execute_batch(include_str!("../migrations/002.sql"))?; db.execute_batch(include_str!("../migrations/003.sql"))?; + db.execute_batch(include_str!("../migrations/004.sql"))?; + db.execute_batch(include_str!("../migrations/005.sql"))?; db.execute_batch("COMMIT")?; } 2 => { db.execute_batch("BEGIN IMMEDIATE")?; db.execute_batch(include_str!("../migrations/003.sql"))?; + db.execute_batch(include_str!("../migrations/004.sql"))?; + db.execute_batch(include_str!("../migrations/005.sql"))?; db.execute_batch("COMMIT")?; } - 3 => {} + 3 => { + db.execute_batch("BEGIN IMMEDIATE")?; + db.execute_batch(include_str!("../migrations/004.sql"))?; + db.execute_batch(include_str!("../migrations/005.sql"))?; + db.execute_batch("COMMIT")?; + } + 4 => { + db.execute_batch("BEGIN IMMEDIATE")?; + db.execute_batch(include_str!("../migrations/005.sql"))?; + db.execute_batch("COMMIT")?; + } + 5 => {} _ => anyhow::bail!("unsupported registry schema {version}"), } let rules: String = db.query_row( diff --git a/crates/argand-site-registry/src/update.rs b/crates/argand-site-registry/src/update.rs index f4ce455..c309616 100644 --- a/crates/argand-site-registry/src/update.rs +++ b/crates/argand-site-registry/src/update.rs @@ -32,6 +32,13 @@ pub struct Config { /// Explicit billed `CrUX` jobs, empty by default. #[serde(default)] pub crux: Vec, + /// Optional explicit review policy; strict reference policy when absent. + pub review_policy: Option, + /// Exact reviewer SSH trust root required for strict candidate builds. + pub reviewer_trust: Option, + /// Replace the current typed full/partition frontier on each scheduled download. + #[serde(default)] + pub auto_supersede_typed_snapshots: bool, } /// Runs all declared imports, refusing candidate publication on any failure. @@ -39,7 +46,9 @@ pub struct Config { /// /// # Errors /// Returns configuration, source, lock, import, or build errors. +#[allow(clippy::too_many_lines)] // Scheduler order is explicit: acquire, import, build, then publish. pub async fn run(config: &Config) -> anyhow::Result { + validate_automatic_coordinates(config)?; fs::create_dir_all(&config.generations)?; let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable .read(true) @@ -49,6 +58,7 @@ pub async fn run(config: &Config) -> anyhow::Result { .open(config.generations.join("update.lock"))?; lock.try_lock().context("registry update already running")?; let mut inputs = config.inputs.clone(); + let mut db = crate::store::open(&config.database)?; let now = Utc::now(); for request in &config.downloads { let mut request = request.clone(); @@ -56,6 +66,26 @@ pub async fn run(config: &Config) -> anyhow::Result { .snapshot .replace("{date}", &now.format("%Y-%m-%d").to_string()) .replace("{month}", &now.format("%Y-%m").to_string()); + if config.auto_supersede_typed_snapshots + && let Some(coverage) = &mut request.coverage + && matches!( + coverage.kind, + crate::model::CoverageKind::Full | crate::model::CoverageKind::Partition + ) + { + anyhow::ensure!( + coverage.supersedes.is_empty(), + "automatic typed supersession cannot combine with explicit supersedes" + ); + if let Some(frontier) = crate::coverage::frontier( + &db, + request.source.key(), + &coverage.collection, + coverage.coordinate(), + )? { + coverage.supersedes.push(frontier); + } + } inputs.push(crate::download::download(&config.cache, &request).await?); } for request in &config.crux { @@ -67,10 +97,29 @@ pub async fn run(config: &Config) -> anyhow::Result { .format("%Y%m") .to_string(); } + if config.auto_supersede_typed_snapshots + && let Some(coverage) = &mut request.coverage + && matches!( + coverage.kind, + crate::model::CoverageKind::Full | crate::model::CoverageKind::Partition + ) + { + anyhow::ensure!( + coverage.supersedes.is_empty(), + "automatic typed supersession cannot combine with explicit supersedes" + ); + if let Some(frontier) = crate::coverage::frontier( + &db, + crate::model::Source::Crux.key(), + &coverage.collection, + coverage.coordinate(), + )? { + coverage.supersedes.push(frontier); + } + } inputs.push(crate::crux::download(&config.cache, &request).await?); } anyhow::ensure!(!inputs.is_empty(), "update config contains no sources"); - let mut db = crate::store::open(&config.database)?; for input in inputs { let manifest: SourceManifest = crate::read_json(&input.manifest)?; crate::store::import(&mut db, &manifest, &input.input)?; @@ -80,7 +129,18 @@ pub async fn run(config: &Config) -> anyhow::Result { let pending = config .generations .join(format!("pending-{}", now.format("%Y%m%dT%H%M%S%.9fZ"))); - crate::build::build(&db, &pending)?; + let policy = config + .review_policy + .as_deref() + .map(crate::read_json) + .transpose()? + .unwrap_or_else(crate::policy::ReviewPolicy::reference); + crate::build::build_with_policy_and_trust( + &db, + &pending, + &policy, + config.reviewer_trust.as_deref(), + )?; let pin = crate::file_digest(&pending.join("COMPLETE.json"))?; let output = config.generations.join(format!("candidate-{pin}")); if output.exists() { @@ -94,3 +154,114 @@ pub async fn run(config: &Config) -> anyhow::Result { } Ok(output) } + +fn validate_automatic_coordinates(config: &Config) -> anyhow::Result<()> { + if !config.auto_supersede_typed_snapshots { + return Ok(()); + } + let mut coordinates = std::collections::BTreeSet::new(); + for request in &config.downloads { + if let Some(coverage) = &request.coverage + && matches!( + coverage.kind, + crate::model::CoverageKind::Full | crate::model::CoverageKind::Partition + ) + { + anyhow::ensure!( + coordinates.insert(( + request.source.key(), + coverage.collection.as_str(), + coverage.coordinate() + )), + "automatic update repeats one source coverage coordinate" + ); + } + } + for request in &config.crux { + if let Some(coverage) = &request.coverage + && matches!( + coverage.kind, + crate::model::CoverageKind::Full | crate::model::CoverageKind::Partition + ) + { + anyhow::ensure!( + coordinates.insert(( + crate::model::Source::Crux.key(), + coverage.collection.as_str(), + coverage.coordinate() + )), + "automatic update repeats one source coverage coordinate" + ); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{Compression, CoverageKind, Format, Source, SourceCoverage}; + + fn request(partition: &str) -> Download { + Download { + source: Source::Majestic, + format: Format::MajesticCsv, + compression: Compression::None, + url: "https://downloads.majestic.com/majestic_million.csv".into(), + snapshot: "{date}".into(), + scope: "full".into(), + maximum_bytes: 1, + coverage: Some(SourceCoverage { + collection: "default".into(), + kind: CoverageKind::Partition, + partition: Some(partition.into()), + base: None, + sequence: None, + supersedes: Vec::new(), + }), + } + } + + #[test] + fn scheduled_supersession_rejects_duplicate_coordinates_before_acquisition() + -> anyhow::Result<()> { + let config = Config { + cache: "cache".into(), + database: "writer.sqlite".into(), + generations: "generations".into(), + downloads: vec![request("global"), request("global")], + inputs: Vec::new(), + crux: Vec::new(), + review_policy: None, + reviewer_trust: None, + auto_supersede_typed_snapshots: true, + }; + + let Some(error) = validate_automatic_coordinates(&config).err() else { + anyhow::bail!("duplicate coordinate was accepted"); + }; + assert!( + error + .to_string() + .contains("repeats one source coverage coordinate") + ); + Ok(()) + } + + #[test] + fn separate_partition_coordinates_are_allowed() -> anyhow::Result<()> { + let config = Config { + cache: "cache".into(), + database: "writer.sqlite".into(), + generations: "generations".into(), + downloads: vec![request("GB"), request("US")], + inputs: Vec::new(), + crux: Vec::new(), + review_policy: None, + reviewer_trust: None, + auto_supersede_typed_snapshots: true, + }; + + validate_automatic_coordinates(&config) + } +} diff --git a/crates/argand-site-registry/src/vote.rs b/crates/argand-site-registry/src/vote.rs new file mode 100644 index 0000000..adc8c21 --- /dev/null +++ b/crates/argand-site-registry/src/vote.rs @@ -0,0 +1,881 @@ +// By Nic Weyand! +//! Accepted-time reviewer votes and deterministic quorum compilation. + +use crate::{bundle::EvidenceBundle, policy::SubjectKind, query::Registry}; +use anyhow::{Context, ensure}; +use base64::Engine; +use chrono::{DateTime, Utc}; +use rusqlite::{Connection, params}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::Path}; + +/// SSH signature namespace for exact vote JSON bytes. +pub const SIGNATURE_NAMESPACE: &str = "argand-site-registry-vote"; + +/// Authenticated decision carried by one reviewer. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, clap::ValueEnum, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum VoteDecision { + /// Support admitting this exact subject and evidence bundle. + Approve, + /// Block this exact subject until the revocation is explicitly superseded. + Revoke, +} + +impl VoteDecision { + const fn key(self) -> &'static str { + match self { + Self::Approve => "approve", + Self::Revoke => "revoke", + } + } +} + +/// One signed reviewer vote over a granular assertion and current evidence bundle. +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct Vote { + /// `argand.site-vote/v1`. + pub schema: String, + /// Name, edge, or equivalence assertion. + pub subject_kind: SubjectKind, + /// Stable material assertion fingerprint. + pub fingerprint: String, + /// Approve or revoke. + pub decision: VoteDecision, + /// Identity that must match the SSH allowed-signers identity. + pub reviewer: String, + /// Human explanation of the decision. + pub reason: String, + /// Exact current evidence-bundle digest. + pub evidence_bundle: String, + /// Review-policy digest under which this decision was made. + pub policy: String, + /// Reviewer-asserted decision time, retained for audit. + pub reviewed_at: DateTime, + /// Reviewer-requested expiry; required only for approvals. + pub expires_at: Option>, + /// Edge role; `unspecified` for names, equivalences, and revocations. + pub role: String, + /// Explicit reviewed locale or empty. + pub locale: String, + /// Explicit reviewed uppercase two-letter country or empty. + pub country: String, + /// Sticky revocation vote IDs explicitly superseded by this approval. + #[serde(default)] + pub supersedes: Vec, +} + +impl Vote { + /// Validates the signed decision independently of registry state. + /// + /// # Errors + /// Rejects unsupported, oversized, malformed, or internally inconsistent votes. + pub fn validate(&self) -> anyhow::Result<()> { + ensure!( + self.schema == "argand.site-vote/v1", + "unsupported vote schema" + ); + ensure!( + crate::model::valid_digest(&self.fingerprint) + && crate::model::valid_digest(&self.evidence_bundle) + && crate::model::valid_digest(&self.policy), + "vote needs full assertion, evidence, and policy digests" + ); + ensure!( + !self.reviewer.trim().is_empty() + && self.reviewer.len() <= 256 + && !self.reason.trim().is_empty() + && self.reason.len() <= 8192, + "vote reviewer and reason are required and bounded" + ); + ensure!( + self.locale.len() <= 64 + && self + .locale + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-'), + "invalid vote locale" + ); + ensure!( + self.country.is_empty() + || (self.country.len() == 2 + && self.country.bytes().all(|byte| byte.is_ascii_uppercase())), + "vote country must be uppercase two-letter code" + ); + ensure!( + self.supersedes.len() <= 256 + && self + .supersedes + .iter() + .all(|id| crate::model::valid_digest(id)), + "invalid vote supersession list" + ); + let unique = self + .supersedes + .iter() + .collect::>(); + ensure!( + unique.len() == self.supersedes.len(), + "duplicate superseded vote" + ); + match self.decision { + VoteDecision::Approve => { + let expires = self.expires_at.context("approval needs an expiry")?; + ensure!( + expires > self.reviewed_at + && expires - self.reviewed_at <= chrono::Duration::days(90), + "approval must expire within 90 days" + ); + } + VoteDecision::Revoke => ensure!( + self.expires_at.is_none() && self.supersedes.is_empty(), + "revocations neither expire nor supersede another vote" + ), + } + match (self.subject_kind, self.decision) { + (SubjectKind::Edge, VoteDecision::Approve) => { + ensure!( + matches!(self.role.as_str(), "primary" | "regional"), + "edge approval needs a primary or regional role" + ); + ensure!( + self.role != "regional" || !self.locale.is_empty() || !self.country.is_empty(), + "regional approval needs locale or country evidence" + ); + ensure!( + self.role != "primary" || (self.locale.is_empty() && self.country.is_empty()), + "primary is the unscoped global fallback" + ); + } + _ => ensure!( + self.role == "unspecified" && self.locale.is_empty() && self.country.is_empty(), + "only edge approvals can assert destination scope" + ), + } + Ok(()) + } +} + +/// Exact detached signature evidence retained with a vote. +#[derive(Clone, Debug)] +pub struct Authentication { + /// SSH allowed-signers identity. + pub signer: String, + /// SHA-256 of exact detached signature bytes. + pub signature_sha256: String, + /// Domain-separated SSH signature namespace. + pub namespace: String, + /// SHA-256 of exact signed vote JSON bytes. + pub decision_sha256: String, + /// SHA-256 of the SSH public-key blob embedded in the verified signature. + pub key_sha256: String, + decision_json: Vec, + signature: Vec, +} + +/// Verifies exact vote bytes against an independently supplied reviewer trust file. +/// +/// # Errors +/// Rejects malformed/oversized inputs, identity mismatch, and untrusted signatures. +pub fn authenticate( + decision: &Path, + signature: &Path, + allowed_reviewers: &Path, + identity: &str, +) -> anyhow::Result<(Vote, Authentication)> { + ensure!( + !identity.trim().is_empty() && identity.len() <= 256, + "reviewer identity is required and bounded" + ); + let decision = crate::ssh::sealed_input(decision, 1024 * 1024)?; + let signature = crate::ssh::sealed_input(signature, 64 * 1024)?; + let allowed_reviewers = crate::ssh::sealed_input(allowed_reviewers, 1024 * 1024)?; + let vote: Vote = serde_json::from_value(crate::json::parse(&decision.bytes)?)?; + vote.validate()?; + ensure!( + vote.reviewer == identity, + "reviewer must equal authenticated identity" + ); + crate::ssh::verify( + SIGNATURE_NAMESPACE, + &decision.bytes, + &signature.bytes, + &allowed_reviewers.bytes, + identity, + )?; + let key_sha256 = signature_key_sha256(&signature.bytes)?; + Ok(( + vote, + Authentication { + signer: identity.into(), + signature_sha256: crate::digest(&signature.bytes), + namespace: SIGNATURE_NAMESPACE.into(), + decision_sha256: crate::digest(&decision.bytes), + key_sha256, + decision_json: decision.bytes, + signature: signature.bytes, + }, + )) +} + +/// Records a verified name or edge vote using trusted writer acceptance time. +/// +/// # Errors +/// Rejects stale evidence, invalid subjects, bad authentication, or SQLite failures. +pub fn record_authenticated( + db: &Connection, + registry: &Registry, + vote: &Vote, + authentication: &Authentication, +) -> anyhow::Result { + record_authenticated_at(db, registry, vote, authentication, Utc::now()) +} + +/// Verifies an authenticated name or edge vote against current evidence without writing it. +/// +/// # Errors +/// Rejects malformed authentication, stale evidence, invalid subjects, and unrelated +/// revocation supersession references. +pub fn verify_authenticated( + registry: &Registry, + vote: &Vote, + authentication: &Authentication, +) -> anyhow::Result<()> { + ensure!( + vote.subject_kind != SubjectKind::Equivalence, + "equivalence vote verification needs the exact proposed entity pair" + ); + validate_authentication(vote, authentication)?; + let bundle = crate::bundle::build(registry, vote.subject_kind, &vote.fingerprint)?; + validate_bundle_and_subject(registry, vote, &bundle)?; + validate_supersedes(®istry.db, vote) +} + +/// Verifies an authenticated equivalence vote against the exact current entity pair. +/// +/// # Errors +/// Rejects malformed authentication, stale pair evidence, a policy mismatch, or +/// unrelated revocation supersession references. +pub fn verify_equivalence_authenticated( + registry: &Registry, + pair: &crate::identity::Equivalence, + vote: &Vote, + authentication: &Authentication, +) -> anyhow::Result<()> { + ensure!( + vote.subject_kind == SubjectKind::Equivalence && vote.fingerprint == pair.fingerprint, + "vote does not match equivalence proposal" + ); + validate_authentication(vote, authentication)?; + let bundle = crate::bundle::equivalence(registry, pair)?; + ensure!( + bundle.id == vote.evidence_bundle, + "vote references stale equivalence evidence" + ); + ensure!( + vote.policy == registry.receipt.review_policy_sha256, + "vote was made under a different review policy" + ); + validate_supersedes(®istry.db, vote) +} + +fn record_authenticated_at( + db: &Connection, + registry: &Registry, + vote: &Vote, + authentication: &Authentication, + accepted_at: DateTime, +) -> anyhow::Result { + ensure!( + vote.subject_kind != SubjectKind::Equivalence, + "equivalence vote needs the exact proposed entity pair" + ); + validate_authentication(vote, authentication)?; + let bundle = crate::bundle::build(registry, vote.subject_kind, &vote.fingerprint)?; + validate_bundle_and_subject(registry, vote, &bundle)?; + validate_supersedes(db, vote)?; + let transaction = db.unchecked_transaction()?; + let id = append(db, vote, authentication, accepted_at)?; + transaction.commit()?; + Ok(id) +} + +pub(crate) fn record_equivalence_authenticated_at( + db: &Connection, + registry: &Registry, + pair: &crate::identity::Equivalence, + vote: &Vote, + authentication: &Authentication, + accepted_at: DateTime, +) -> anyhow::Result { + verify_equivalence_authenticated(registry, pair, vote, authentication)?; + validate_supersedes(db, vote)?; + let transaction = db.unchecked_transaction()?; + db.execute( + "INSERT OR IGNORE INTO equivalences VALUES(?1,?2,?3,?4,?5)", + params![ + pair.fingerprint, + pair.entities[0], + pair.entities[1], + pair.signatures[0], + pair.signatures[1] + ], + )?; + let id = append(db, vote, authentication, accepted_at)?; + transaction.commit()?; + Ok(id) +} + +fn validate_bundle_and_subject( + registry: &Registry, + vote: &Vote, + bundle: &EvidenceBundle, +) -> anyhow::Result<()> { + ensure!( + vote.policy == registry.receipt.review_policy_sha256, + "vote was made under a different review policy" + ); + ensure!( + bundle.id == vote.evidence_bundle, + "vote references stale evidence" + ); + if vote.subject_kind == SubjectKind::Edge && vote.decision == VoteDecision::Approve { + ensure!( + registry.candidate(&vote.fingerprint)?.eligible, + "ineligible edge cannot be approved" + ); + } + Ok(()) +} + +fn validate_authentication(vote: &Vote, authentication: &Authentication) -> anyhow::Result<()> { + vote.validate()?; + ensure!( + authentication.signer == vote.reviewer + && authentication.namespace == SIGNATURE_NAMESPACE + && crate::model::valid_digest(&authentication.signature_sha256) + && authentication.signature_sha256 == crate::digest(&authentication.signature) + && authentication.decision_sha256 == crate::digest(&authentication.decision_json) + && crate::model::valid_digest(&authentication.key_sha256) + && authentication.key_sha256 == signature_key_sha256(&authentication.signature)?, + "vote authentication does not match decision" + ); + let signed: Vote = serde_json::from_value(crate::json::parse(&authentication.decision_json)?)?; + ensure!(&signed == vote, "signed decision differs from vote"); + Ok(()) +} + +pub(crate) fn signature_key_sha256(signature: &[u8]) -> anyhow::Result { + let text = std::str::from_utf8(signature)?; + let mut encoded = String::new(); + let mut inside = false; + let mut ended = false; + for line in text.lines() { + match line.trim() { + "-----BEGIN SSH SIGNATURE-----" if !inside && !ended => inside = true, + "-----END SSH SIGNATURE-----" if inside => { + inside = false; + ended = true; + } + value if inside => encoded.push_str(value), + value if !value.is_empty() => anyhow::bail!("malformed SSH signature armor"), + _ => {} + } + } + ensure!(!inside && ended, "incomplete SSH signature armor"); + let decoded = base64::engine::general_purpose::STANDARD.decode(encoded)?; + ensure!( + decoded.len() >= 14 && &decoded[..6] == b"SSHSIG", + "invalid SSH signature envelope" + ); + let version = u32::from_be_bytes(decoded[6..10].try_into()?); + ensure!(version == 1, "unsupported SSH signature version"); + let key_len = usize::try_from(u32::from_be_bytes(decoded[10..14].try_into()?))?; + let end = 14_usize + .checked_add(key_len) + .context("SSH signature key length overflow")?; + ensure!( + key_len > 0 && key_len <= 16 * 1024 && end <= decoded.len(), + "invalid SSH signature public key" + ); + Ok(crate::digest(&decoded[14..end])) +} + +fn validate_supersedes(db: &Connection, vote: &Vote) -> anyhow::Result<()> { + for id in &vote.supersedes { + let valid: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM votes WHERE id=?1 AND fingerprint=?2 AND subject_kind=?3 AND decision='revoke')", + params![id, vote.fingerprint, vote.subject_kind.key()], + |row| row.get(0), + )?; + ensure!(valid, "vote supersedes an absent or unrelated revocation"); + } + Ok(()) +} + +fn append( + db: &Connection, + vote: &Vote, + authentication: &Authentication, + accepted_at: DateTime, +) -> anyhow::Result { + let id = authentication.decision_sha256.clone(); + let changed = db.execute( + "INSERT OR IGNORE INTO votes(id,fingerprint,subject_kind,decision,reviewer,reason,evidence_bundle,policy_sha256,reviewed_at,expires_at,role,locale,country,supersedes_json,accepted_at,document_json) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16)", + params![ + id, + vote.fingerprint, + vote.subject_kind.key(), + vote.decision.key(), + vote.reviewer, + vote.reason, + vote.evidence_bundle, + vote.policy, + vote.reviewed_at.to_rfc3339(), + vote.expires_at.map(|time| time.to_rfc3339()), + vote.role, + vote.locale, + vote.country, + serde_json::to_string(&vote.supersedes)?, + accepted_at.to_rfc3339(), + authentication.decision_json, + ], + )?; + if changed == 0 { + let matches: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM votes v JOIN vote_auth a USING(sequence) WHERE v.id=?1 AND v.document_json=?2 AND a.signer=?3 AND a.signature_sha256=?4 AND a.namespace=?5 AND a.decision_sha256=?6 AND a.key_sha256=?7 AND a.signature=?8)", + params![ + id, + authentication.decision_json, + authentication.signer, + authentication.signature_sha256, + authentication.namespace, + authentication.decision_sha256, + authentication.key_sha256, + authentication.signature + ], + |row| row.get(0), + )?; + ensure!(matches, "vote ID collision or authentication mismatch"); + return Ok(id); + } + let sequence = db.last_insert_rowid(); + db.execute( + "INSERT INTO vote_auth VALUES(?1,?2,?3,?4,?5,?6,?7)", + params![ + sequence, + authentication.signer, + authentication.signature_sha256, + authentication.namespace, + authentication.decision_sha256, + authentication.key_sha256, + authentication.signature, + ], + )?; + Ok(id) +} + +/// Result of compiling current authenticated votes under one generation policy. +#[derive(Clone, Debug, Serialize)] +pub struct PolicyDecision { + /// Compiled state. + pub status: DecisionStatus, + /// Exact current evidence bundle. + pub evidence_bundle: String, + /// Required distinct approvals. + pub approvals_required: u16, + /// Required distinct reviewer groups. + pub groups_required: u16, + /// Current evidence-matching approval votes examined. + pub approvals: u64, + /// Active sticky revocations. + pub revocations: Vec, + /// Qualified edge scopes; one unscoped entry for name/equivalence approval. + pub scopes: Vec, + /// Votes excluded because their evidence bundle is no longer current. + pub stale_evidence: u64, + /// Votes excluded because they were signed under another policy epoch. + pub stale_policy: u64, + /// Latest reviewer approvals that expired. + pub expired: u64, + /// Latest reviewer approvals whose effective start is in the future. + pub not_yet_valid: u64, + /// Signed policy rules currently holding an otherwise qualified subject. + pub policy_holds: Vec, +} + +/// Policy state for one granular subject. +#[derive(Clone, Copy, Debug, Serialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum DecisionStatus { + /// At least one exact scope satisfies the configured quorum. + Approved, + /// A sticky revocation has not been superseded by a complete new quorum. + Revoked, + /// Current evidence lacks a complete approval quorum. + InsufficientReview, + /// All current-evidence approvals have expired. + Expired, + /// Votes exist, but none references the current evidence bundle. + StaleEvidence, + /// Votes exist, but none was signed under the current policy epoch. + StalePolicy, + /// Current approvals conflict across destination scopes and no scope has quorum. + Disputed, + /// Approval quorum exists, but signed risk policy requires fresh review or evidence. + Probationary, +} + +/// One exact role/scope that independently reached quorum. +#[derive(Clone, Debug, Serialize, Eq, Ord, PartialEq, PartialOrd)] +pub struct ApprovedScope { + /// `primary`, `regional`, or `unspecified`. + pub role: String, + /// Reviewed locale or empty. + pub locale: String, + /// Reviewed country or empty. + pub country: String, + /// Counted exact vote IDs. + pub votes: Vec, + /// Earliest effective expiry among counted votes. + pub expires_at: DateTime, +} + +#[derive(Clone)] +struct StoredVote { + id: String, + vote: Vote, + accepted_at: DateTime, + key_sha256: String, +} + +/// Compiles a current name or edge decision under the authenticated generation policy. +/// +/// # Errors +/// Rejects missing subjects and corrupt vote/policy rows. +pub fn decision( + registry: &Registry, + subject_kind: SubjectKind, + fingerprint: &str, + now: DateTime, +) -> anyhow::Result { + let bundle = crate::bundle::build(registry, subject_kind, fingerprint)?; + decision_for_bundle(registry, subject_kind, fingerprint, &bundle.id, now) +} + +#[allow(clippy::too_many_lines)] // Quorum, expiry, supersession, and risk compile in one pass. +pub(crate) fn decision_for_bundle( + registry: &Registry, + subject_kind: SubjectKind, + fingerprint: &str, + bundle: &str, + now: DateTime, +) -> anyhow::Result { + let policy = ®istry.receipt.review_policy; + policy.validate()?; + let mut risk_classes = Vec::new(); + let mut source_conflict = false; + let mut dangerous_drift = false; + if subject_kind == SubjectKind::Edge { + let candidate = registry.candidate(fingerprint)?; + if policy.block_source_conflicts || policy.risk_thresholds.contains_key("source_conflict") { + source_conflict = crate::queue::domain_entity_conflict(registry, &candidate, now)?; + if source_conflict { + risk_classes.push("source_conflict"); + } + } + if policy.block_dangerous_drift || policy.risk_thresholds.contains_key("dangerous_drift") { + dangerous_drift = crate::queue::drift(registry, fingerprint)?.revocation_candidate; + if dangerous_drift { + risk_classes.push("dangerous_drift"); + } + } + } + let threshold = policy.threshold_for(subject_kind, risk_classes.iter().copied()); + let votes = load(®istry.db, subject_kind, fingerprint)?; + let stale_policy = u64::try_from( + votes + .iter() + .filter(|vote| { + vote.vote.decision == VoteDecision::Approve + && vote.vote.policy != registry.receipt.review_policy_sha256 + }) + .map(|vote| vote.vote.reviewer.as_str()) + .collect::>() + .len(), + )?; + let mut latest = BTreeMap::new(); + for vote in votes + .iter() + .filter(|vote| vote.vote.policy == registry.receipt.review_policy_sha256) + { + latest.insert(vote.vote.reviewer.as_str(), vote); + } + let mut stale_evidence = 0; + let mut expired = 0; + let mut not_yet_valid = 0; + let mut active_approvals: BTreeMap<(String, String, String), Vec<&StoredVote>> = + BTreeMap::new(); + for vote in latest.into_values() { + if vote.vote.decision != VoteDecision::Approve { + continue; + } + if vote.vote.evidence_bundle != bundle { + stale_evidence += 1; + continue; + } + let starts = vote.accepted_at.max(vote.vote.reviewed_at); + let requested = vote + .vote + .expires_at + .context("stored approval has no expiry")?; + let effective_expiry = requested.min( + vote.accepted_at + chrono::Duration::days(i64::from(policy.maximum_approval_days)), + ); + if now < starts { + not_yet_valid += 1; + continue; + } + if now >= effective_expiry { + expired += 1; + continue; + } + active_approvals + .entry(( + vote.vote.role.clone(), + vote.vote.locale.clone(), + vote.vote.country.clone(), + )) + .or_default() + .push(vote); + } + let revocations = votes + .iter() + .filter(|vote| vote.vote.decision == VoteDecision::Revoke && vote.accepted_at <= now) + .map(|vote| vote.id.clone()) + .collect::>(); + let active_scope_count = active_approvals.len(); + let mut scopes = Vec::new(); + let mut approval_count = 0_u64; + for ((role, locale, country), approvals) in active_approvals { + let mut unique_keys = BTreeMap::new(); + for approval in approvals { + unique_keys.insert(approval.key_sha256.as_str(), approval); + } + let approvals = unique_keys.into_values().collect::>(); + approval_count = approval_count.saturating_add(u64::try_from(approvals.len())?); + let reviewers = approvals + .iter() + .map(|vote| vote.vote.reviewer.as_str()) + .collect::>(); + let enough = approvals.len() >= usize::from(threshold.approvals) + && policy.group_count(reviewers.into_iter()) >= usize::from(threshold.groups); + let supersedes_revocations = revocations.iter().all(|revocation| { + approvals + .iter() + .all(|approval| approval.vote.supersedes.contains(revocation)) + }); + if enough && (!policy.sticky_revocations || supersedes_revocations) { + let expires_at = approvals + .iter() + .filter_map(|vote| { + vote.vote.expires_at.map(|expires| { + expires.min( + vote.accepted_at + + chrono::Duration::days(i64::from(policy.maximum_approval_days)), + ) + }) + }) + .min() + .context("qualified approval scope has no expiry")?; + scopes.push(ApprovedScope { + role, + locale, + country, + votes: approvals.iter().map(|vote| vote.id.clone()).collect(), + expires_at, + }); + } + } + let unresolved_revocation = !revocations.is_empty() && scopes.is_empty(); + let mut policy_holds = Vec::new(); + if subject_kind == SubjectKind::Edge && !scopes.is_empty() { + if policy.block_source_conflicts && source_conflict { + policy_holds.push("source_conflict".into()); + } + let drift = crate::queue::drift(registry, fingerprint)?; + if policy.require_edge_observation + && drift + .classes + .contains(&crate::queue::DriftClass::Unobserved) + { + policy_holds.push("missing_observation".into()); + } + if let (Some(maximum_days), Some(latest_at)) = + (policy.maximum_observation_age_days, drift.latest_at) + && (latest_at > now + chrono::Duration::minutes(5) + || latest_at + chrono::Duration::days(i64::from(maximum_days)) < now) + { + policy_holds.push("stale_observation".into()); + } + if policy.block_dangerous_drift && dangerous_drift { + policy_holds.push("dangerous_drift".into()); + } + } + let status = if unresolved_revocation { + DecisionStatus::Revoked + } else if policy_holds.iter().any(|hold| hold == "source_conflict") { + DecisionStatus::Disputed + } else if !policy_holds.is_empty() { + DecisionStatus::Probationary + } else if !scopes.is_empty() { + DecisionStatus::Approved + } else if active_scope_count > 1 && approval_count > 0 { + DecisionStatus::Disputed + } else if expired > 0 && approval_count == 0 { + DecisionStatus::Expired + } else if stale_evidence > 0 && approval_count == 0 { + DecisionStatus::StaleEvidence + } else if stale_policy > 0 && approval_count == 0 { + DecisionStatus::StalePolicy + } else { + DecisionStatus::InsufficientReview + }; + Ok(PolicyDecision { + status, + evidence_bundle: bundle.into(), + approvals_required: threshold.approvals, + groups_required: threshold.groups, + approvals: approval_count, + revocations, + scopes, + stale_evidence, + stale_policy, + expired, + not_yet_valid, + policy_holds, + }) +} + +fn load( + db: &Connection, + subject_kind: SubjectKind, + fingerprint: &str, +) -> anyhow::Result> { + let mut statement = db.prepare( + "SELECT v.id,v.document_json,v.accepted_at,a.signer,a.signature_sha256,a.namespace,a.decision_sha256,a.key_sha256,a.signature FROM votes v JOIN vote_auth a USING(sequence) WHERE v.subject_kind=?1 AND v.fingerprint=?2 ORDER BY v.sequence", + )?; + let mut rows = statement.query(params![subject_kind.key(), fingerprint])?; + let mut result = Vec::new(); + while let Some(row) = rows.next()? { + let document: Vec = row.get(1)?; + let vote: Vote = serde_json::from_value(crate::json::parse(&document)?)?; + let authentication = Authentication { + signer: row.get(3)?, + signature_sha256: row.get(4)?, + namespace: row.get(5)?, + decision_sha256: row.get(6)?, + key_sha256: row.get(7)?, + decision_json: document, + signature: row.get(8)?, + }; + validate_authentication(&vote, &authentication)?; + ensure!( + row.get::<_, String>(0)? == authentication.decision_sha256, + "stored vote identity mismatch" + ); + result.push(StoredVote { + id: authentication.decision_sha256, + vote, + accepted_at: DateTime::parse_from_rfc3339(&row.get::<_, String>(2)?)? + .with_timezone(&Utc), + key_sha256: authentication.key_sha256, + }); + } + Ok(result) +} + +/// Re-verifies every stored vote signature at trusted writer acceptance time. +/// +/// # Errors +/// Rejects missing/altered proofs, invalid acceptance times, or untrusted signers. +pub fn verify_all(db: &Connection, allowed_reviewers: &Path) -> anyhow::Result<()> { + let allowed = crate::ssh::sealed_input(allowed_reviewers, 1024 * 1024)?; + let missing: u64 = db.query_row( + "SELECT count(*) FROM votes v LEFT JOIN vote_auth a USING(sequence) WHERE a.sequence IS NULL", + [], + |row| crate::store::unsigned(row, 0), + )?; + ensure!(missing == 0, "generation contains unauthenticated votes"); + let mut statement = db.prepare( + "SELECT v.document_json,v.accepted_at,a.signer,a.signature_sha256,a.namespace,a.decision_sha256,a.key_sha256,a.signature FROM votes v JOIN vote_auth a USING(sequence) ORDER BY v.sequence", + )?; + let mut rows = statement.query([])?; + while let Some(row) = rows.next()? { + let decision: Vec = row.get(0)?; + let vote: Vote = serde_json::from_value(crate::json::parse(&decision)?)?; + let accepted_at = + DateTime::parse_from_rfc3339(&row.get::<_, String>(1)?)?.with_timezone(&Utc); + let authentication = Authentication { + signer: row.get(2)?, + signature_sha256: row.get(3)?, + namespace: row.get(4)?, + decision_sha256: row.get(5)?, + key_sha256: row.get(6)?, + decision_json: decision, + signature: row.get(7)?, + }; + validate_authentication(&vote, &authentication)?; + crate::ssh::verify_at( + SIGNATURE_NAMESPACE, + &authentication.decision_json, + &authentication.signature, + &allowed.bytes, + &authentication.signer, + Some(accepted_at), + )?; + } + Ok(()) +} + +/// Rejects a publisher identity that supplied any vote when separation is enabled. +/// +/// # Errors +/// Returns policy/SQLite failures or a publisher-reviewer conflict. +pub fn verify_publisher_separation(registry: &Registry, publisher: &str) -> anyhow::Result<()> { + if !registry.receipt.review_policy.publisher_reviewer_separation { + return Ok(()); + } + let conflict: bool = registry.db.query_row( + "SELECT EXISTS(SELECT 1 FROM votes WHERE reviewer=?1)", + [publisher], + |row| row.get(0), + )?; + ensure!(!conflict, "release publisher also supplied a reviewer vote"); + Ok(()) +} + +/// Rejects a publisher signature made by any physical key that supplied a vote. +/// +/// # Errors +/// Returns malformed-signature, policy, SQLite, or key-separation failures. +pub(crate) fn verify_publisher_key_separation( + registry: &Registry, + publisher_signature: &[u8], +) -> anyhow::Result<()> { + if !registry.receipt.review_policy.publisher_reviewer_separation { + return Ok(()); + } + let key = signature_key_sha256(publisher_signature)?; + let conflict: bool = registry.db.query_row( + "SELECT EXISTS(SELECT 1 FROM vote_auth WHERE key_sha256=?1)", + [key], + |row| row.get(0), + )?; + ensure!( + !conflict, + "release publisher key also supplied a reviewer vote" + ); + Ok(()) +} diff --git a/crates/argand-site-registry/tests/cli.rs b/crates/argand-site-registry/tests/cli.rs index 2c64cca..afb7e9d 100644 --- a/crates/argand-site-registry/tests/cli.rs +++ b/crates/argand-site-registry/tests/cli.rs @@ -34,16 +34,16 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu fs::create_dir(root)?; } prepare_signer(root)?; + fs::write( + root.join("legacy-policy.json"), + serde_json::to_vec_pretty( + &argand_site_registry::policy::ReviewPolicy::legacy_compatible(), + )?, + )?; let database = root.join("store.sqlite"); import_sources(root, &database)?; let candidate = root.join("candidate"); - let built = run(&[ - "build", - "--database", - text(&database)?, - "--output", - text(&candidate)?, - ])?; + let built = build_legacy(root, &database, &candidate)?; let pin = built["pin"].as_str().context("missing pin")?; let lookup = run(&[ "lookup", @@ -79,13 +79,7 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu record_review(root, &database, &candidate, pin, &decision_path)?; let approved = root.join("approved"); review_identity(root, &database, &candidate, pin, &decision)?; - let built = run(&[ - "build", - "--database", - text(&database)?, - "--output", - text(&approved)?, - ])?; + let built = build_legacy(root, &database, &approved)?; let approved_pin = built["pin"].as_str().context("approved pin")?; assert_eq!( run(&[ @@ -307,13 +301,7 @@ fn release_lifecycle( fs::write(&revocation_path, serde_json::to_vec(&decision)?)?; record_review(root, database, approved, approved_pin, &revocation_path)?; let revoked = root.join("revoked"); - let built = run(&[ - "build", - "--database", - text(database)?, - "--output", - text(&revoked)?, - ])?; + let built = build_legacy(root, database, &revoked)?; let revoked_pin = built["pin"].as_str().context("revoked pin")?; assert!( run(&[ @@ -513,6 +501,18 @@ fn prepare_signer(root: &Path) -> anyhow::Result<()> { Ok(()) } +fn build_legacy(root: &Path, database: &Path, output: &Path) -> anyhow::Result { + run(&[ + "build", + "--database", + text(database)?, + "--output", + text(output)?, + "--policy", + text(&root.join("legacy-policy.json"))?, + ]) +} + fn sign_review(root: &Path, decision: &Path) -> anyhow::Result { let status = Command::new("ssh-keygen") .args([ diff --git a/crates/argand-site-registry/tests/common/mod.rs b/crates/argand-site-registry/tests/common/mod.rs index 9ec6c54..e6be439 100644 --- a/crates/argand-site-registry/tests/common/mod.rs +++ b/crates/argand-site-registry/tests/common/mod.rs @@ -76,6 +76,7 @@ pub fn manifest(source: Source, format: Format, bytes: &[u8]) -> anyhow::Result< compression: Compression::None, snapshot: "synthetic-fixture-v1".into(), scope: "fixture".into(), + coverage: None, source_url: source_url.into(), license: source.license().into(), license_url: source.license_url().into(), @@ -135,7 +136,11 @@ pub fn fixture(root: &Path) -> anyhow::Result { pub fn build(db: &rusqlite::Connection, root: &Path, name: &str) -> anyhow::Result { let generation = root.join(name); - argand_site_registry::build::build(db, &generation)?; + argand_site_registry::build::build_with_policy( + db, + &generation, + &argand_site_registry::policy::ReviewPolicy::legacy_compatible(), + )?; Registry::open( &generation, &argand_site_registry::file_digest(&generation.join("COMPLETE.json"))?, diff --git a/crates/argand-site-registry/tests/compatibility.rs b/crates/argand-site-registry/tests/compatibility.rs new file mode 100644 index 0000000..9fb0a9f --- /dev/null +++ b/crates/argand-site-registry/tests/compatibility.rs @@ -0,0 +1,51 @@ +// By Nic Weyand! +//! Frozen v0.3 public-contract coordinates used by migration and rollback tests. + +#[test] +fn v03_contract_golden_is_explicit_and_unchanged() -> anyhow::Result<()> { + let contract: serde_json::Value = + serde_json::from_str(include_str!("fixtures/v03-contract.json"))?; + assert_eq!(contract["crate_version"], "0.3.0"); + assert_eq!( + contract["signed_commit"], + "ac8282093d8a815c6227cff86e1f40714d510bcd" + ); + assert_eq!(contract["writer_schema"], 3); + assert_eq!(contract["rules"], "argand.site-rules/v3"); + assert_eq!(contract["source_manifest_schema"], "argand.site-source/v1"); + assert_eq!( + contract["generation_receipt_schema"], + "argand.site-registry/v1" + ); + assert_eq!(contract["export_schema"], "argand.site-export/v1"); + assert_eq!(contract["current_pointer_schema"], "argand.site-current/v1"); + assert_eq!(contract["diff_schema"], "argand.site-diff/v3"); + assert_eq!(contract["selection_schema"], "argand.site-selection/v1"); + assert_eq!( + contract["review_signature_namespace"], + argand_site_registry::review::SIGNATURE_NAMESPACE + ); + assert_eq!( + contract["release_signature_namespace"], + "argand-site-registry" + ); + assert_eq!( + contract["receipt_fields"].as_array().map(Vec::len), + Some(11) + ); + assert_eq!(contract["lookup_fields"].as_array().map(Vec::len), Some(6)); + assert_eq!( + contract["candidate_fields"].as_array().map(Vec::len), + Some(14) + ); + assert_eq!( + contract["resolution_fields"].as_array().map(Vec::len), + Some(5) + ); + assert_eq!( + contract["resolution_statuses"].as_array().map(Vec::len), + Some(8) + ); + assert_eq!(contract["review_fields"].as_array().map(Vec::len), Some(10)); + Ok(()) +} diff --git a/crates/argand-site-registry/tests/coverage.rs b/crates/argand-site-registry/tests/coverage.rs new file mode 100644 index 0000000..708823a --- /dev/null +++ b/crates/argand-site-registry/tests/coverage.rs @@ -0,0 +1,133 @@ +// By Nic Weyand! +//! Active coverage, delta masking, and audit-retention acceptance proof. +#[allow(dead_code)] +mod common; + +use argand_site_registry::{ + model::{CoverageKind, Format, Source, SourceCoverage}, + query::Registry, + store, +}; +use serde_json::json; +use std::fs; + +#[test] +#[allow(clippy::too_many_lines)] // One scenario proves full, delta, tombstone, active, and audit behavior. +fn typed_full_and_delta_replace_active_records_but_preserve_audit_history() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let mut db = common::fixture(root.path())?; + let original_bytes = serde_json::to_vec(&common::wikidata())?; + let original = common::manifest(Source::Wikidata, Format::WikidataEntities, &original_bytes)?; + + let full_bytes = serde_json::to_vec(&common::wikidata())?; + let mut full = common::manifest(Source::Wikidata, Format::WikidataEntities, &full_bytes)?; + full.schema = "argand.site-source/v2".into(); + full.snapshot = "synthetic-full-v2".into(); + full.scope = "entities-full".into(); + full.retrieved_at += chrono::Duration::hours(1); + full.coverage = Some(SourceCoverage { + collection: "entities".into(), + kind: CoverageKind::Full, + partition: None, + base: None, + sequence: None, + supersedes: vec![original.id()?], + }); + let full_path = root.path().join("wikidata-full.json"); + fs::write(&full_path, &full_bytes)?; + store::import(&mut db, &full, &full_path)?; + + let delta_bytes = serde_json::to_vec(&json!({"entities":{"Q355":common::entity( + "Q355", + "Facebook", + &["Meta FB"], + &["https://facebook.com/"] + )}}))?; + let mut delta = common::manifest(Source::Wikidata, Format::WikidataEntities, &delta_bytes)?; + delta.schema = "argand.site-source/v2".into(); + delta.snapshot = "synthetic-delta-v2".into(); + delta.scope = "entities-delta-1".into(); + delta.retrieved_at += chrono::Duration::hours(2); + delta.coverage = Some(SourceCoverage { + collection: "entities".into(), + kind: CoverageKind::Delta, + partition: None, + base: Some(full.id()?), + sequence: Some(1), + supersedes: vec![full.id()?], + }); + let delta_path = root.path().join("wikidata-delta.json"); + fs::write(&delta_path, &delta_bytes)?; + store::import(&mut db, &delta, &delta_path)?; + + let generation = root.path().join("generation"); + argand_site_registry::build::build_with_policy( + &db, + &generation, + &argand_site_registry::policy::ReviewPolicy::legacy_compatible(), + )?; + let pin = argand_site_registry::file_digest(&generation.join("COMPLETE.json"))?; + let registry = Registry::open(&generation, &pin)?; + assert_eq!(registry.lookup("FB", 20)?.total_entities, 0); + assert_eq!(registry.lookup("Meta FB", 20)?.total_entities, 1); + assert_eq!(registry.lookup("Atlas", 20)?.total_entities, 1); + let stats = registry.stats(common::timestamp()?)?; + assert_eq!(stats.selected_sources, 6); + assert_eq!(stats.superseded_source_snapshots, 1); + + let active = root.path().join("active.jsonl"); + let audit = root.path().join("audit.jsonl"); + argand_site_registry::release::export(®istry, &active, false)?; + argand_site_registry::release::export_audit(®istry, &audit, false)?; + let active = fs::read_to_string(active)?; + let audit = fs::read_to_string(audit)?; + assert!(!active.contains("\"text\":\"FB\"")); + assert!(active.contains("\"text\":\"Meta FB\"")); + assert!(audit.contains("\"selection_state\":\"superseded\"")); + assert!(audit.contains("\"text\":\"FB\"")); + + let tombstone_bytes = serde_json::to_vec(&json!({"entities":{"Q355":{ + "id":"Q355", + "missing":"" + }}}))?; + let mut tombstone = + common::manifest(Source::Wikidata, Format::WikidataEntities, &tombstone_bytes)?; + tombstone.schema = "argand.site-source/v2".into(); + tombstone.snapshot = "synthetic-delta-v2-tombstone".into(); + tombstone.scope = "entities-delta-2".into(); + tombstone.retrieved_at += chrono::Duration::hours(3); + tombstone.coverage = Some(SourceCoverage { + collection: "entities".into(), + kind: CoverageKind::Delta, + partition: None, + base: Some(delta.id()?), + sequence: Some(2), + supersedes: vec![delta.id()?], + }); + let tombstone_path = root.path().join("wikidata-tombstone.json"); + fs::write(&tombstone_path, &tombstone_bytes)?; + store::import(&mut db, &tombstone, &tombstone_path)?; + let retired_path = root.path().join("retired"); + argand_site_registry::build::build_with_policy( + &db, + &retired_path, + &argand_site_registry::policy::ReviewPolicy::legacy_compatible(), + )?; + let retired_pin = argand_site_registry::file_digest(&retired_path.join("COMPLETE.json"))?; + let retired = Registry::open(&retired_path, &retired_pin)?; + assert_eq!(retired.lookup("Facebook", 20)?.total_entities, 0); + assert_eq!(retired.lookup("Meta FB", 20)?.total_entities, 0); + let retired_active = root.path().join("retired-active.jsonl"); + let retired_audit = root.path().join("retired-audit.jsonl"); + argand_site_registry::release::export(&retired, &retired_active, false)?; + argand_site_registry::release::export_audit(&retired, &retired_audit, false)?; + assert!( + !fs::read_to_string(retired_active)? + .contains("\"subject\":\"argand:entity:wikidata:Q355\"") + ); + let retired_audit = fs::read_to_string(retired_audit)?; + assert!(retired_audit.contains("\"type\":\"tombstone\"")); + assert!(retired_audit.contains("\"selection_state\":\"tombstoned\"")); + assert!(retired_audit.contains("\"source_identifier\":\"Q355\"")); + Ok(()) +} diff --git a/crates/argand-site-registry/tests/failures.rs b/crates/argand-site-registry/tests/failures.rs index 3a86da1..519705d 100644 --- a/crates/argand-site-registry/tests/failures.rs +++ b/crates/argand-site-registry/tests/failures.rs @@ -23,7 +23,7 @@ fn version_one_store_migrates_without_losing_review_history() -> anyhow::Result< let migrated = store::open(&path)?; assert_eq!( migrated.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?, - 3 + 5 ); assert_eq!( migrated.query_row("SELECT rules FROM registry_metadata", [], |row| row @@ -43,6 +43,40 @@ fn version_one_store_migrates_without_losing_review_history() -> anyhow::Result< Ok(()) } +#[test] +fn every_prior_writer_schema_migrates_to_v04() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + for version in 1..=4 { + let path = root.path().join(format!("v{version}.sqlite")); + let db = rusqlite::Connection::open(&path)?; + db.execute_batch(include_str!("../migrations/001.sql"))?; + if version >= 2 { + db.execute_batch(include_str!("../migrations/002.sql"))?; + } + if version >= 3 { + db.execute_batch(include_str!("../migrations/003.sql"))?; + } + if version >= 4 { + db.execute_batch(include_str!("../migrations/004.sql"))?; + } + drop(db); + let migrated = store::open(&path)?; + assert_eq!( + migrated.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?, + 5 + ); + for table in ["votes", "vote_auth", "observation_batches", "observations"] { + let exists: bool = migrated.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name=?1)", + [table], + |row| row.get(0), + )?; + assert!(exists, "{table} missing after schema {version} migration"); + } + } + Ok(()) +} + #[test] fn externally_signed_unauthenticated_reviews_are_rejected() -> anyhow::Result<()> { let root = tempfile::tempdir()?; @@ -236,6 +270,7 @@ fn activation_accepts_a_pinned_v1_previous_generation() -> anyhow::Result<()> { drop(old); let receipt_path = root.path().join("legacy-current/COMPLETE.json"); let mut receipt: serde_json::Value = argand_site_registry::read_json(&receipt_path)?; + receipt["schema"] = json!("argand.site-registry/v1"); receipt["rules"] = json!("argand.site-rules/v1"); fs::write(&receipt_path, serde_json::to_vec_pretty(&receipt)?)?; let old_pin = argand_site_registry::file_digest(&receipt_path)?; @@ -597,6 +632,11 @@ async fn repeated_update_reuses_generation_and_failure_preserves_it() -> anyhow: )?; inputs.push(CachedSource { input, manifest }); } + let review_policy = root.path().join("policy.json"); + fs::write( + &review_policy, + serde_json::to_vec(&argand_site_registry::policy::ReviewPolicy::legacy_compatible())?, + )?; let config = argand_site_registry::update::Config { cache: root.path().join("cache"), database: root.path().join("data.sqlite"), @@ -604,6 +644,9 @@ async fn repeated_update_reuses_generation_and_failure_preserves_it() -> anyhow: downloads: vec![], inputs, crux: vec![], + review_policy: Some(review_policy), + reviewer_trust: None, + auto_supersede_typed_snapshots: false, }; let first = argand_site_registry::update::run(&config).await?; let second = argand_site_registry::update::run(&config).await?; diff --git a/crates/argand-site-registry/tests/fixtures/v03-contract.json b/crates/argand-site-registry/tests/fixtures/v03-contract.json new file mode 100644 index 0000000..da9004a --- /dev/null +++ b/crates/argand-site-registry/tests/fixtures/v03-contract.json @@ -0,0 +1,21 @@ +{ + "crate_version": "0.3.0", + "signed_commit": "ac8282093d8a815c6227cff86e1f40714d510bcd", + "writer_schema": 3, + "rules": "argand.site-rules/v3", + "source_manifest_schema": "argand.site-source/v1", + "generation_receipt_schema": "argand.site-registry/v1", + "export_schema": "argand.site-export/v1", + "current_pointer_schema": "argand.site-current/v1", + "diff_schema": "argand.site-diff/v3", + "selection_schema": "argand.site-selection/v1", + "review_signature_namespace": "argand-site-registry-review", + "release_signature_namespace": "argand-site-registry", + "generation_files": ["ATTRIBUTION.json", "COMPLETE.json", "LICENSE_SOURCES.md", "registry.sqlite"], + "receipt_fields": ["schema", "rules", "database_sha256", "licenses_sha256", "attribution_sha256", "psl_source", "sources", "entities", "properties", "edges", "rejected"], + "lookup_fields": ["query", "total_entities", "total_edges", "truncated", "candidates", "attribution"], + "candidate_fields": ["identity_provenance", "entity", "entity_id", "canonical_name", "url", "web_property", "property_scopes", "relation", "confidence", "fingerprint", "evidence", "provenance", "review", "eligible"], + "resolution_fields": ["query", "status", "destination", "counts", "attribution"], + "resolution_statuses": ["resolved", "no_name_match", "ambiguous_identity", "safety_limit_exceeded", "no_eligible_destination", "no_active_review", "region_mismatch", "ambiguous_destination"], + "review_fields": ["fingerprint", "decision", "reviewer", "reason", "evidence", "reviewed_at", "expires_at", "role", "locale", "country"] +} diff --git a/crates/argand-site-registry/tests/identity.rs b/crates/argand-site-registry/tests/identity.rs index a0d8225..b9685f2 100644 --- a/crates/argand-site-registry/tests/identity.rs +++ b/crates/argand-site-registry/tests/identity.rs @@ -173,7 +173,7 @@ fn name_collision_remains_ambiguous_until_review_and_changes_invalidate_link() - } #[test] -fn metadata_changes_invalidate_identity_fingerprint() -> anyhow::Result<()> { +fn metadata_changes_preserve_material_identity_and_refresh_evidence() -> anyhow::Result<()> { let root = tempfile::tempdir()?; let mut db = common::fixture(root.path())?; let baseline = common::build(&db, root.path(), "metadata-baseline")?; @@ -182,6 +182,7 @@ fn metadata_changes_invalidate_identity_fingerprint() -> anyhow::Result<()> { .entity_id .clone(); let before = identity::propose(&baseline, &wiki, &curlie)?; + let before_bundle = argand_site_registry::bundle::equivalence(&baseline, &before)?; let mut entity = common::entity("Q355", "Facebook", &["FB"], &["https://facebook.com/"]); entity["claims"]["P17"] = json!([{"id":"Q355$country","rank":"normal","mainsnak":{"property":"P17","snaktype":"value","datavalue":{"type":"wikibase-entityid","value":{"id":"Q30"}}}}]); @@ -193,6 +194,8 @@ fn metadata_changes_invalidate_identity_fingerprint() -> anyhow::Result<()> { store::import(&mut db, &source, &input)?; let changed = common::build(&db, root.path(), "metadata-changed")?; let after = identity::propose(&changed, &wiki, &curlie)?; - assert_ne!(before.fingerprint, after.fingerprint); + let after_bundle = argand_site_registry::bundle::equivalence(&changed, &after)?; + assert_eq!(before.fingerprint, after.fingerprint); + assert_ne!(before_bundle.id, after_bundle.id); Ok(()) } diff --git a/crates/argand-site-registry/tests/registry.rs b/crates/argand-site-registry/tests/registry.rs index 194e7b3..e020d7d 100644 --- a/crates/argand-site-registry/tests/registry.rs +++ b/crates/argand-site-registry/tests/registry.rs @@ -255,7 +255,7 @@ fn ambiguity_survives_limits_and_same_named_domains_do_not_merge() -> anyhow::Re } #[test] -fn changed_names_and_urls_invalidate_approval_but_history_survives() -> anyhow::Result<()> { +fn added_alias_preserves_edge_approval_and_history() -> anyhow::Result<()> { let dir = tempfile::tempdir()?; let mut db = fixture(dir.path())?; let initial = build(&db, dir.path(), "initial")?; @@ -280,14 +280,16 @@ fn changed_names_and_urls_invalidate_approval_but_history_survives() -> anyhow:: std::fs::write(&path, bytes)?; store::import(&mut db, &m, &path)?; let r = build(&db, dir.path(), "changed")?; - assert!( + assert_eq!( r.resolve( "Facebook", None, None, timestamp()? + chrono::Duration::days(1) )? - .is_none() + .context("granular website approval survives an unrelated alias")? + .url, + "https://facebook.com/" ); let sources: i64 = db.query_row( "SELECT count(*) FROM sources WHERE source='wikidata' AND complete=1", diff --git a/crates/argand-site-registry/tests/v04.rs b/crates/argand-site-registry/tests/v04.rs new file mode 100644 index 0000000..f3b515c --- /dev/null +++ b/crates/argand-site-registry/tests/v04.rs @@ -0,0 +1,1109 @@ +// By Nic Weyand! +//! v0.4 trust, observation, drift, and signed-release acceptance proof. +#[allow(dead_code)] +mod common; + +use anyhow::{Context, ensure}; +use argand_site_registry::{ + observation::ObservationKind, + observer::{Capture, Hop}, + policy::{ReviewPolicy, SubjectKind}, + query::Registry, + vote::{Vote, VoteDecision}, +}; +use chrono::Utc; +use std::{fs, path::Path, process::Command}; + +#[test] +#[allow(clippy::too_many_lines)] // One acceptance flow preserves the exact trust and evidence transition order. +fn strict_quorum_observation_drift_revocation_and_release() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let (trust, keys) = reviewers( + root.path(), + &["reviewer-one", "reviewer-two", "incident-reviewer"], + )?; + let initial = strict_build(&db, root.path(), "initial", &trust)?; + assert_eq!( + initial + .resolve_explained("FB", None, None, Utc::now())? + .status, + argand_site_registry::ResolutionStatus::NoActiveNameReview + ); + let queue = argand_site_registry::queue::review_queue(&initial, Utc::now(), 100, 1000)?; + let name = queue + .items + .iter() + .find(|item| item.subject_kind == SubjectKind::Name && item.display == "FB") + .context("FB alias queue item")?; + let lookup = initial.lookup("FB", 20)?; + let edge = lookup + .candidates + .iter() + .find(|candidate| candidate.url == "https://facebook.com/") + .context("Facebook edge candidate")?; + let name_fingerprint = name.fingerprint.clone(); + let edge_fingerprint = edge.fingerprint.clone(); + + approve( + &db, + &initial, + root.path(), + &keys[0], + &trust, + "reviewer-one", + SubjectKind::Name, + &name_fingerprint, + "unspecified", + &[], + )?; + approve( + &db, + &initial, + root.path(), + &keys[0], + &trust, + "reviewer-one", + SubjectKind::Edge, + &edge_fingerprint, + "primary", + &[], + )?; + let one_reviewer = strict_build(&db, root.path(), "one-reviewer", &trust)?; + let resolution = one_reviewer.resolve_explained("FB", None, None, Utc::now())?; + assert!(resolution.destination.is_none()); + assert_eq!( + resolution.status, + argand_site_registry::ResolutionStatus::NoActiveNameReview + ); + + approve( + &db, + &initial, + root.path(), + &keys[1], + &trust, + "reviewer-two", + SubjectKind::Name, + &name_fingerprint, + "unspecified", + &[], + )?; + approve( + &db, + &initial, + root.path(), + &keys[1], + &trust, + "reviewer-two", + SubjectKind::Edge, + &edge_fingerprint, + "primary", + &[], + )?; + let other_entity = initial + .lookup_web("facebook.com", 20)? + .matches + .into_iter() + .map(|matched| matched.candidate.entity_id) + .find(|entity| entity != &edge.entity_id) + .context("independent Curlie entity for the shared domain")?; + let equivalence = + argand_site_registry::identity::propose(&initial, &edge.entity_id, &other_entity)?; + for (identity, key) in [("reviewer-one", &keys[0]), ("reviewer-two", &keys[1])] { + approve_equivalence( + &db, + &initial, + root.path(), + key, + &trust, + identity, + &equivalence, + )?; + } + let approved = strict_build(&db, root.path(), "approved", &trust)?; + assert_eq!( + approved + .resolve("FB", None, None, Utc::now())? + .context("two-reviewer route")? + .url, + "https://facebook.com/" + ); + let mut correlated_policy = ReviewPolicy::reference(); + correlated_policy.reviewer_groups = std::collections::BTreeMap::from([ + ("reviewer-one".into(), "same-organization".into()), + ("reviewer-two".into(), "same-organization".into()), + ]); + let correlated_path = root.path().join("correlated-reviewers"); + argand_site_registry::build::build_with_policy_and_trust( + &db, + &correlated_path, + &correlated_policy, + Some(&trust), + )?; + let correlated_pin = argand_site_registry::file_digest(&correlated_path.join("COMPLETE.json"))?; + let correlated = Registry::open(&correlated_path, &correlated_pin)?; + let policy_stale = argand_site_registry::vote::decision( + &correlated, + SubjectKind::Name, + &name_fingerprint, + Utc::now(), + )?; + assert_eq!( + policy_stale.status, + argand_site_registry::vote::DecisionStatus::StalePolicy + ); + assert_eq!(policy_stale.stale_policy, 2); + for (identity, key) in [("reviewer-one", &keys[0]), ("reviewer-two", &keys[1])] { + approve( + &db, + &correlated, + root.path(), + key, + &trust, + identity, + SubjectKind::Name, + &name_fingerprint, + "unspecified", + &[], + )?; + approve( + &db, + &correlated, + root.path(), + key, + &trust, + identity, + SubjectKind::Edge, + &edge_fingerprint, + "primary", + &[], + )?; + } + let correlated_voted_path = root.path().join("correlated-voted"); + argand_site_registry::build::build_with_policy_and_trust( + &db, + &correlated_voted_path, + &correlated_policy, + Some(&trust), + )?; + let correlated_voted_pin = + argand_site_registry::file_digest(&correlated_voted_path.join("COMPLETE.json"))?; + assert_eq!( + Registry::open(&correlated_voted_path, &correlated_voted_pin)? + .resolve_explained("FB", None, None, Utc::now())? + .status, + argand_site_registry::ResolutionStatus::NoActiveNameReview + ); + + let first_capture = capture_fixture( + root.path(), + "capture-one", + &edge_fingerprint, + "https://facebook.com/", + None, + Utc::now(), + )?; + let (first_jsonl, first_manifest) = + replay_fixture(&approved, root.path(), &first_capture, "one")?; + let manifest = argand_site_registry::read_json(&first_manifest)?; + let first_batch_id = argand_site_registry::observation::BatchManifest::id(&manifest)?; + argand_site_registry::observation::import(&db, &approved, &manifest, &first_jsonl)?; + assert!( + db.execute( + "UPDATE observation_batches SET records=records+1 WHERE id=?1", + [&first_batch_id], + ) + .is_err() + ); + assert!( + db.execute( + "INSERT INTO observations SELECT ?1,batch_id,subject_kind,subject_fingerprint,document_json FROM observations WHERE batch_id=?2 LIMIT 1", + rusqlite::params![argand_site_registry::digest(b"late observation"), first_batch_id], + ) + .is_err() + ); + assert!( + db.execute( + "DELETE FROM observation_batches WHERE id=?1", + [&first_batch_id], + ) + .is_err() + ); + let observed = strict_build(&db, root.path(), "observed", &trust)?; + let observations = + argand_site_registry::observation::lookup(&observed, &edge_fingerprint, 100)?; + assert!(observations.total >= 6); + let reverse = + argand_site_registry::observation::reverse_lookup(&observed, "facebook.com", 100)?; + assert_eq!(reverse.total, observations.total); + let stale = observed.resolve_explained("FB", None, None, Utc::now())?; + assert!(stale.destination.is_none()); + assert_eq!(stale.counts.stale_evidence, 1); + + for (identity, key) in [("reviewer-one", &keys[0]), ("reviewer-two", &keys[1])] { + approve( + &db, + &observed, + root.path(), + key, + &trust, + identity, + SubjectKind::Edge, + &edge_fingerprint, + "primary", + &[], + )?; + } + let observed_approved = strict_build(&db, root.path(), "observed-approved", &trust)?; + assert!( + observed_approved + .resolve("FB", None, None, Utc::now())? + .is_some() + ); + + let second_capture = capture_fixture( + root.path(), + "capture-two", + &edge_fingerprint, + "https://facebook.com/", + Some("https://example.org/"), + Utc::now() + chrono::Duration::seconds(1), + )?; + let (second_jsonl, second_manifest) = + replay_fixture(&observed_approved, root.path(), &second_capture, "two")?; + let manifest = argand_site_registry::read_json(&second_manifest)?; + argand_site_registry::observation::import(&db, &observed_approved, &manifest, &second_jsonl)?; + let drifted = strict_build(&db, root.path(), "drifted", &trust)?; + let drift = argand_site_registry::queue::drift(&drifted, &edge_fingerprint)?; + assert!(drift.revocation_candidate); + assert!( + drift + .classes + .contains(&argand_site_registry::queue::DriftClass::CrossDomainRedirect) + ); + assert!( + drift + .classes + .contains(&argand_site_registry::queue::DriftClass::ContentChanged) + ); + assert_eq!( + argand_site_registry::queue::revocation_candidates(&drifted, 1000)?.len(), + 1 + ); + assert!(drifted.resolve("FB", None, None, Utc::now())?.is_none()); + + let revocation = revoke( + &db, + &drifted, + root.path(), + &keys[2], + &trust, + "incident-reviewer", + &edge_fingerprint, + )?; + let revoked = strict_build(&db, root.path(), "revoked", &trust)?; + let resolution = revoked.resolve_explained("FB", None, None, Utc::now())?; + assert_eq!(resolution.counts.revoked, 1); + let mut changed_policy = ReviewPolicy::reference(); + changed_policy.name = "argand-reference-next-epoch".into(); + let changed_policy_path = root.path().join("changed-policy-revocation"); + argand_site_registry::build::build_with_policy_and_trust( + &db, + &changed_policy_path, + &changed_policy, + Some(&trust), + )?; + let changed_policy_pin = + argand_site_registry::file_digest(&changed_policy_path.join("COMPLETE.json"))?; + let changed_policy_registry = Registry::open(&changed_policy_path, &changed_policy_pin)?; + assert_eq!( + argand_site_registry::vote::decision( + &changed_policy_registry, + SubjectKind::Edge, + &edge_fingerprint, + Utc::now(), + )? + .status, + argand_site_registry::vote::DecisionStatus::Revoked + ); + let (publisher, allowed_publishers) = signed_release(root.path(), &revoked, &trust, &keys[0])?; + let first_feed_at = Utc::now() + chrono::Duration::seconds(1); + let first_feed = root.path().join("revocations-one.json"); + let first_signature = root.path().join("revocations-one.sig"); + argand_site_registry::revocation::export(&revoked, &first_feed, first_feed_at)?; + let reviewer_signature = root.path().join("reviewer-revocations.sig"); + assert!( + argand_site_registry::revocation::sign( + &revoked, + &first_feed, + &reviewer_signature, + &keys[2], + &trust, + "publisher-alias", + ) + .is_err() + ); + assert!(!reviewer_signature.exists()); + argand_site_registry::revocation::sign( + &revoked, + &first_feed, + &first_signature, + &publisher, + &trust, + "publisher", + )?; + let verified_feed = argand_site_registry::revocation::verify( + &observed_approved, + &first_feed, + &first_signature, + &allowed_publishers, + "publisher", + first_feed_at, + None, + )?; + assert!(verified_feed.blocks(SubjectKind::Edge, &edge_fingerprint)); + assert!( + observed_approved + .resolve_with_revocations("FB", None, None, first_feed_at, &verified_feed,)? + .is_none() + ); + let tampered_feed = root.path().join("revocations-tampered.json"); + let mut tampered = fs::read(&first_feed)?; + tampered.push(b' '); + fs::write(&tampered_feed, tampered)?; + assert!( + argand_site_registry::revocation::verify( + &observed_approved, + &tampered_feed, + &first_signature, + &allowed_publishers, + "publisher", + first_feed_at, + None, + ) + .is_err() + ); + assert!( + argand_site_registry::revocation::verify( + &observed_approved, + &first_feed, + &first_signature, + &allowed_publishers, + "publisher", + first_feed_at + chrono::Duration::days(8), + None, + ) + .is_err() + ); + assert!( + observed_approved + .resolve_with_revocations( + "FB", + None, + None, + first_feed_at + chrono::Duration::days(8), + &verified_feed, + ) + .is_err() + ); + + for (identity, key) in [("reviewer-one", &keys[0]), ("reviewer-two", &keys[1])] { + approve( + &db, + &revoked, + root.path(), + key, + &trust, + identity, + SubjectKind::Edge, + &edge_fingerprint, + "primary", + &[], + )?; + } + let blocked = strict_build(&db, root.path(), "blocked-reapproval", &trust)?; + assert_eq!( + blocked + .resolve_explained("FB", None, None, Utc::now())? + .counts + .revoked, + 1 + ); + for (identity, key) in [("reviewer-one", &keys[0]), ("reviewer-two", &keys[1])] { + approve( + &db, + &blocked, + root.path(), + key, + &trust, + identity, + SubjectKind::Edge, + &edge_fingerprint, + "primary", + std::slice::from_ref(&revocation), + )?; + } + let dangerous = strict_build(&db, root.path(), "dangerous-reapproval", &trust)?; + let dangerous_decision = argand_site_registry::vote::decision( + &dangerous, + SubjectKind::Edge, + &edge_fingerprint, + Utc::now(), + )?; + assert_eq!( + dangerous_decision.status, + argand_site_registry::vote::DecisionStatus::Probationary + ); + assert!( + dangerous_decision + .policy_holds + .contains(&"dangerous_drift".into()) + ); + + let safe_capture = capture_fixture( + root.path(), + "capture-safe", + &edge_fingerprint, + "https://facebook.com/", + None, + Utc::now() + chrono::Duration::seconds(2), + )?; + let (safe_jsonl, safe_manifest) = + replay_fixture(&dangerous, root.path(), &safe_capture, "safe")?; + let manifest = argand_site_registry::read_json(&safe_manifest)?; + argand_site_registry::observation::import(&db, &dangerous, &manifest, &safe_jsonl)?; + let safe = strict_build(&db, root.path(), "safe-evidence", &trust)?; + for (identity, key) in [("reviewer-one", &keys[0]), ("reviewer-two", &keys[1])] { + approve( + &db, + &safe, + root.path(), + key, + &trust, + identity, + SubjectKind::Edge, + &edge_fingerprint, + "primary", + std::slice::from_ref(&revocation), + )?; + } + let recovered = strict_build(&db, root.path(), "explicit-reapproval", &trust)?; + assert!(recovered.resolve("FB", None, None, Utc::now())?.is_some()); + let second_feed_at = (Utc::now() + chrono::Duration::seconds(1)) + .max(first_feed_at + chrono::Duration::seconds(1)); + let second_feed = root.path().join("revocations-two.json"); + let second_signature = root.path().join("revocations-two.sig"); + argand_site_registry::revocation::export(&recovered, &second_feed, second_feed_at)?; + argand_site_registry::revocation::sign( + &recovered, + &second_feed, + &second_signature, + &publisher, + &trust, + "publisher", + )?; + assert!( + argand_site_registry::revocation::verify( + &observed_approved, + &second_feed, + &second_signature, + &allowed_publishers, + "publisher", + second_feed_at, + Some(&verified_feed), + ) + .is_err() + ); + let prior_for_recovered = argand_site_registry::revocation::verify( + &recovered, + &first_feed, + &first_signature, + &allowed_publishers, + "publisher", + second_feed_at, + None, + )?; + let recovered_feed = argand_site_registry::revocation::verify( + &recovered, + &second_feed, + &second_signature, + &allowed_publishers, + "publisher", + second_feed_at, + Some(&prior_for_recovered), + )?; + assert!(!recovered_feed.blocks(SubjectKind::Edge, &edge_fingerprint)); + assert!( + recovered + .resolve_with_revocations("FB", None, None, second_feed_at, &recovered_feed,)? + .is_some() + ); + Ok(()) +} + +fn strict_build( + db: &rusqlite::Connection, + root: &Path, + name: &str, + trust: &Path, +) -> anyhow::Result { + let path = root.join(name); + argand_site_registry::build::build_with_policy_and_trust( + db, + &path, + &ReviewPolicy::reference(), + Some(trust), + )?; + let pin = argand_site_registry::file_digest(&path.join("COMPLETE.json"))?; + Registry::open(&path, &pin) +} + +#[allow(clippy::too_many_arguments)] +fn approve( + db: &rusqlite::Connection, + registry: &Registry, + root: &Path, + key: &Path, + trust: &Path, + identity: &str, + subject_kind: SubjectKind, + fingerprint: &str, + role: &str, + supersedes: &[String], +) -> anyhow::Result { + record_vote( + db, + registry, + root, + key, + trust, + &Vote { + schema: "argand.site-vote/v1".into(), + subject_kind, + fingerprint: fingerprint.into(), + decision: VoteDecision::Approve, + reviewer: identity.into(), + reason: "synthetic v0.4 acceptance fixture".into(), + evidence_bundle: argand_site_registry::bundle::build( + registry, + subject_kind, + fingerprint, + )? + .id, + policy: registry.receipt.review_policy_sha256.clone(), + reviewed_at: Utc::now() - chrono::Duration::seconds(1), + expires_at: Some(Utc::now() + chrono::Duration::days(30)), + role: role.into(), + locale: String::new(), + country: String::new(), + supersedes: supersedes.to_vec(), + }, + ) +} + +fn revoke( + db: &rusqlite::Connection, + registry: &Registry, + root: &Path, + key: &Path, + trust: &Path, + identity: &str, + fingerprint: &str, +) -> anyhow::Result { + record_vote( + db, + registry, + root, + key, + trust, + &Vote { + schema: "argand.site-vote/v1".into(), + subject_kind: SubjectKind::Edge, + fingerprint: fingerprint.into(), + decision: VoteDecision::Revoke, + reviewer: identity.into(), + reason: "synthetic cross-domain drift revocation".into(), + evidence_bundle: argand_site_registry::bundle::build( + registry, + SubjectKind::Edge, + fingerprint, + )? + .id, + policy: registry.receipt.review_policy_sha256.clone(), + reviewed_at: Utc::now(), + expires_at: None, + role: "unspecified".into(), + locale: String::new(), + country: String::new(), + supersedes: Vec::new(), + }, + ) +} + +fn record_vote( + db: &rusqlite::Connection, + registry: &Registry, + root: &Path, + key: &Path, + trust: &Path, + vote: &Vote, +) -> anyhow::Result { + let path = root.join(format!( + "vote-{}-{}-{}.json", + vote.reviewer, + vote.subject_kind.key(), + argand_site_registry::digest(&serde_json::to_vec(vote)?)[..12].to_owned() + )); + fs::write(&path, serde_json::to_vec_pretty(vote)?)?; + let status = Command::new("ssh-keygen") + .args([ + "-Y", + "sign", + "-n", + argand_site_registry::vote::SIGNATURE_NAMESPACE, + "-f", + ]) + .arg(key) + .arg(&path) + .status()?; + ensure!(status.success(), "sign fixture vote"); + let signature = path.with_extension("json.sig"); + let (vote, authentication) = + argand_site_registry::vote::authenticate(&path, &signature, trust, &vote.reviewer)?; + argand_site_registry::vote::record_authenticated(db, registry, &vote, &authentication) +} + +fn approve_equivalence( + db: &rusqlite::Connection, + registry: &Registry, + root: &Path, + key: &Path, + trust: &Path, + identity: &str, + pair: &argand_site_registry::identity::Equivalence, +) -> anyhow::Result { + let vote = Vote { + schema: "argand.site-vote/v1".into(), + subject_kind: SubjectKind::Equivalence, + fingerprint: pair.fingerprint.clone(), + decision: VoteDecision::Approve, + reviewer: identity.into(), + reason: "synthetic shared-domain entity equivalence".into(), + evidence_bundle: argand_site_registry::bundle::equivalence(registry, pair)?.id, + policy: registry.receipt.review_policy_sha256.clone(), + reviewed_at: Utc::now() - chrono::Duration::seconds(1), + expires_at: Some(Utc::now() + chrono::Duration::days(30)), + role: "unspecified".into(), + locale: String::new(), + country: String::new(), + supersedes: Vec::new(), + }; + let path = root.join(format!( + "vote-{identity}-equivalence-{}.json", + &argand_site_registry::digest(&serde_json::to_vec(&vote)?)[..12] + )); + fs::write(&path, serde_json::to_vec_pretty(&vote)?)?; + ensure!( + Command::new("ssh-keygen") + .args([ + "-Y", + "sign", + "-n", + argand_site_registry::vote::SIGNATURE_NAMESPACE, + "-f", + ]) + .arg(key) + .arg(&path) + .status()? + .success(), + "sign equivalence vote" + ); + let (vote, authentication) = argand_site_registry::vote::authenticate( + &path, + &path.with_extension("json.sig"), + trust, + identity, + )?; + argand_site_registry::identity::record_vote_authenticated( + db, + registry, + pair, + &vote, + &authentication, + ) +} + +fn reviewers( + root: &Path, + identities: &[&str], +) -> anyhow::Result<(std::path::PathBuf, Vec)> { + let mut allowed = String::new(); + let mut keys = Vec::new(); + for identity in identities { + let key = root.join(identity); + ensure!( + Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(&key) + .status()? + .success(), + "generate reviewer key" + ); + allowed.push_str(identity); + allowed.push(' '); + allowed.push_str(&fs::read_to_string(key.with_extension("pub"))?); + keys.push(key); + } + let trust = root.join("allowed-reviewers"); + fs::write(&trust, allowed)?; + Ok((trust, keys)) +} + +fn capture_fixture( + root: &Path, + name: &str, + fingerprint: &str, + start: &str, + redirect: Option<&str>, + retrieved_at: chrono::DateTime, +) -> anyhow::Result { + let directory = root.join(name); + fs::create_dir(&directory)?; + let body = if redirect.is_some() { + b"".as_slice() + } else { + b"US".as_slice() + }; + let dns_one = argand_site_registry::digest(b"public-address-set-one"); + let tls_one = argand_site_registry::digest(b"certificate-one"); + let mut hops = vec![Hop { + url: start.into(), + status: if redirect.is_some() { 301 } else { 200 }, + redirect_to: redirect.map(str::to_owned), + content_type: Some("text/html".into()), + link_headers: Vec::new(), + addresses_sha256: Some(dns_one), + certificate_sha256: Some(tls_one), + }]; + if let Some(redirect) = redirect { + hops.push(Hop { + url: redirect.into(), + status: 200, + redirect_to: None, + content_type: Some("text/html".into()), + link_headers: Vec::new(), + addresses_sha256: Some(argand_site_registry::digest(b"public-address-set-two")), + certificate_sha256: Some(argand_site_registry::digest(b"certificate-two")), + }); + } + let capture = Capture { + schema: "argand.site-observer-capture/v1".into(), + subject_fingerprint: fingerprint.into(), + start_url: start.into(), + retrieved_at, + hops, + body_sha256: argand_site_registry::digest(body), + body_bytes: u64::try_from(body.len())?, + failure: None, + }; + fs::write(directory.join("BODY.bin"), body)?; + fs::write( + directory.join("CAPTURE.json"), + serde_json::to_vec_pretty(&capture)?, + )?; + Ok(directory) +} + +fn replay_fixture( + registry: &Registry, + root: &Path, + capture: &Path, + name: &str, +) -> anyhow::Result<(std::path::PathBuf, std::path::PathBuf)> { + let output = root.join(format!("observations-{name}.jsonl")); + let manifest = root.join(format!("observations-{name}.manifest.json")); + argand_site_registry::observer::replay(registry, capture, &output, &manifest)?; + Ok((output, manifest)) +} + +fn signed_release( + root: &Path, + registry: &Registry, + reviewers: &Path, + reviewer_key: &Path, +) -> anyhow::Result<(std::path::PathBuf, std::path::PathBuf)> { + let generation = root.join("revoked"); + assert!( + argand_site_registry::release::sign_as( + &generation, + reviewer_key, + ®istry.identity, + reviewers, + "publisher-alias", + ) + .is_err() + ); + assert!(!generation.join("COMPLETE.json.sig").exists()); + let publisher = root.join("publisher"); + ensure!( + Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(&publisher) + .status()? + .success(), + "generate publisher key" + ); + let allowed = root.join("allowed-publishers"); + fs::write( + &allowed, + format!( + "publisher {}", + fs::read_to_string(publisher.with_extension("pub"))? + ), + )?; + argand_site_registry::release::sign_as( + &generation, + &publisher, + ®istry.identity, + reviewers, + "publisher", + )?; + let current = root.join("current.json"); + argand_site_registry::release::activate( + &generation, + ¤t, + &allowed, + "publisher", + reviewers, + )?; + let pointer: serde_json::Value = argand_site_registry::read_json(¤t)?; + assert_eq!(pointer["receipt_sha256"], registry.identity); + Ok((publisher, allowed)) +} + +#[test] +fn relation_enum_round_trips_new_network_evidence() -> anyhow::Result<()> { + let values = [ + ObservationKind::DnsResolution { + addresses_sha256: argand_site_registry::digest(b"addresses"), + }, + ObservationKind::TlsCertificate { + certificate_sha256: argand_site_registry::digest(b"certificate"), + }, + ObservationKind::DomainRegistration { + state: "expiry_risk".into(), + expires_at: Some(Utc::now() + chrono::Duration::days(7)), + }, + ObservationKind::MalwarePolicy { + policy: "synthetic-policy-v1".into(), + result: "malicious".into(), + }, + ]; + for value in values { + let json = serde_json::to_vec(&value)?; + assert_eq!(serde_json::from_slice::(&json)?, value); + } + Ok(()) +} + +#[test] +fn aliases_for_one_ssh_key_cannot_satisfy_quorum() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let (_, keys) = reviewers(root.path(), &["physical-key"])?; + let public = fs::read_to_string(keys[0].with_extension("pub"))?; + let trust = root.path().join("aliased-reviewers"); + fs::write(&trust, format!("alias-one {public}alias-two {public}"))?; + let initial = strict_build(&db, root.path(), "alias-initial", &trust)?; + let fingerprint = argand_site_registry::queue::review_queue(&initial, Utc::now(), 100, 1000)? + .items + .into_iter() + .find(|item| item.subject_kind == SubjectKind::Name && item.display == "FB") + .context("FB name")? + .fingerprint; + for identity in ["alias-one", "alias-two"] { + approve( + &db, + &initial, + root.path(), + &keys[0], + &trust, + identity, + SubjectKind::Name, + &fingerprint, + "unspecified", + &[], + )?; + } + let generation = strict_build(&db, root.path(), "aliased-key", &trust)?; + let decision = argand_site_registry::vote::decision( + &generation, + SubjectKind::Name, + &fingerprint, + Utc::now(), + )?; + assert_eq!(decision.approvals, 1); + assert_eq!( + decision.status, + argand_site_registry::vote::DecisionStatus::InsufficientReview + ); + Ok(()) +} + +#[test] +fn vote_signature_tampering_and_future_decisions_fail_closed() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let (trust, keys) = reviewers(root.path(), &["future-one", "future-two"])?; + let initial = strict_build(&db, root.path(), "future-initial", &trust)?; + let fingerprint = argand_site_registry::queue::review_queue(&initial, Utc::now(), 100, 1000)? + .items + .into_iter() + .find(|item| item.subject_kind == SubjectKind::Name && item.display == "FB") + .context("FB name")? + .fingerprint; + let reviewed_at = Utc::now() + chrono::Duration::days(1); + for (index, identity) in ["future-one", "future-two"].into_iter().enumerate() { + let vote = Vote { + schema: "argand.site-vote/v1".into(), + subject_kind: SubjectKind::Name, + fingerprint: fingerprint.clone(), + decision: VoteDecision::Approve, + reviewer: identity.into(), + reason: "future dated fixture".into(), + evidence_bundle: argand_site_registry::bundle::build( + &initial, + SubjectKind::Name, + &fingerprint, + )? + .id, + policy: initial.receipt.review_policy_sha256.clone(), + reviewed_at, + expires_at: Some(reviewed_at + chrono::Duration::days(30)), + role: "unspecified".into(), + locale: String::new(), + country: String::new(), + supersedes: Vec::new(), + }; + if index == 0 { + let path = root.path().join("tamper-vote.json"); + let original = serde_json::to_vec_pretty(&vote)?; + fs::write(&path, &original)?; + ensure!( + Command::new("ssh-keygen") + .args([ + "-Y", + "sign", + "-n", + argand_site_registry::vote::SIGNATURE_NAMESPACE, + "-f", + ]) + .arg(&keys[index]) + .arg(&path) + .status()? + .success(), + "sign tamper fixture" + ); + let signature = path.with_extension("json.sig"); + let mut changed = serde_json::to_value(&vote)?; + changed["reason"] = serde_json::json!("altered after signing"); + fs::write(&path, serde_json::to_vec_pretty(&changed)?)?; + assert!( + argand_site_registry::vote::authenticate(&path, &signature, &trust, identity) + .is_err() + ); + fs::write(&path, original)?; + let (verified, authentication) = + argand_site_registry::vote::authenticate(&path, &signature, &trust, identity)?; + argand_site_registry::vote::record_authenticated( + &db, + &initial, + &verified, + &authentication, + )?; + } else { + record_vote(&db, &initial, root.path(), &keys[index], &trust, &vote)?; + } + } + let generation = strict_build(&db, root.path(), "future-votes", &trust)?; + let decision = argand_site_registry::vote::decision( + &generation, + SubjectKind::Name, + &fingerprint, + Utc::now(), + )?; + assert_eq!(decision.not_yet_valid, 2); + assert_eq!( + decision.status, + argand_site_registry::vote::DecisionStatus::InsufficientReview + ); + Ok(()) +} + +#[test] +fn conflicting_current_edge_scopes_are_disputed() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let (trust, keys) = reviewers(root.path(), &["scope-one", "scope-two"])?; + let initial = strict_build(&db, root.path(), "scope-initial", &trust)?; + let fingerprint = initial + .lookup("Facebook", 20)? + .candidates + .into_iter() + .find(|candidate| candidate.url == "https://facebook.com/") + .context("Facebook edge")? + .fingerprint; + approve( + &db, + &initial, + root.path(), + &keys[0], + &trust, + "scope-one", + SubjectKind::Edge, + &fingerprint, + "primary", + &[], + )?; + record_vote( + &db, + &initial, + root.path(), + &keys[1], + &trust, + &Vote { + schema: "argand.site-vote/v1".into(), + subject_kind: SubjectKind::Edge, + fingerprint: fingerprint.clone(), + decision: VoteDecision::Approve, + reviewer: "scope-two".into(), + reason: "conflicting regional scope fixture".into(), + evidence_bundle: argand_site_registry::bundle::build( + &initial, + SubjectKind::Edge, + &fingerprint, + )? + .id, + policy: initial.receipt.review_policy_sha256.clone(), + reviewed_at: Utc::now() - chrono::Duration::seconds(1), + expires_at: Some(Utc::now() + chrono::Duration::days(30)), + role: "regional".into(), + locale: String::new(), + country: "GB".into(), + supersedes: Vec::new(), + }, + )?; + let generation = strict_build(&db, root.path(), "scope-dispute", &trust)?; + assert_eq!( + argand_site_registry::vote::decision( + &generation, + SubjectKind::Edge, + &fingerprint, + Utc::now(), + )? + .status, + argand_site_registry::vote::DecisionStatus::Disputed + ); + Ok(()) +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..fc9d8b5 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,83 @@ +# Architecture + +Argand Site Registry is a local Rust library and CLI around a mutable SQLite +writer and immutable, content-pinned reader generations. Source acquisition, +evidence normalization, editorial decisions, and release authority remain +separate. + +```mermaid +flowchart LR + S[Allowlisted source objects] --> I[Streaming adapters] + I --> W[(Writer store)] + O[Bounded candidate observer] --> B[Immutable observation batches] + B --> W + W --> G[Candidate generation] + G --> Q[Evidence bundles and review queue] + Q --> V[Signed reviewer votes] + V --> W + W --> R[Reviewed generation] + R --> P[Separate publisher signature] + P --> C[Verified lookup and resolve] + R --> E[Signed cumulative revocation feed] + E --> C +``` + +## Data boundaries + +`sources`, `records`, and `facts` preserve provider-native evidence. Typed +coverage selects a coherent active set without deleting older snapshots. +`selected_sources` and `active_records` record that derivation. Names, entities, +web properties, edges, popularity, and rejected facts are deterministic +projections. + +Names, website edges, and entity equivalences have separate material +fingerprints. Adding an alias therefore cannot inherit an approved destination, +and changing unrelated entity metadata does not invalidate an unchanged website +edge. Every evidence bundle contains the exact current assertion, provenance, and +attached observations that a vote signs. + +Observation batches are append-only. Redirects, canonicals, hreflang, JSON-LD +`sameAs`, sitemaps, country selectors, HTTP state, public DNS-set hashes, TLS +certificate hashes, and bounded failures remain observations. They never create +an entity, ownership edge, role, or approval. + +Votes are exact signed JSON documents. The writer records its own `accepted_at`, +the SSH signature, physical public-key digest, evidence-bundle digest, policy +digest, scope, expiry, and any explicitly superseded revocation IDs. Policy +compilation counts independent identities, groups, and physical keys. The +reference policy requires two independent approvals and makes one revocation +sticky. + +## Trust boundaries + +- Source HTTPS and a content digest establish what was imported, not whether the + assertion is true. +- Reviewer signatures establish who made an exact decision, not site safety. +- Publisher signatures authenticate a complete generation, not every assertion. +- `lookup` is an audit surface. `resolve` is the policy-enforced navigation + surface. +- Automated acquisition, observation, and builds stop at candidates. They have no + review, publisher, or activation authority. + +Strict generation receipts bind database bytes, license and attribution files, +selected coverage, the review policy, reviewer trust-root bytes, and the trusted +acceptance-time rule. Readers copy authenticated SQLite bytes into a private +unlinked snapshot before opening them. + +## Main modules + +| Module | Responsibility | +| --- | --- | +| `download`, `crux` | Allowlisted, bounded source acquisition and resumable cache | +| `source`, `store`, adapters | Manifest validation and streaming source-specific import | +| `coverage` | Full/partition/delta graph validation and active-record masking | +| `normalize` | Deterministic URL, hostname, registrable-domain, suffix, and name normalization | +| `build` | Canonical immutable generation and receipt creation | +| `bundle`, `policy`, `vote` | Review evidence, policy epochs, authenticated quorum, revocation | +| `observer`, `observation`, `queue` | Candidate-only collection, replay/import, reverse lookup, drift and queues | +| `query`, `resolution`, `identity`, `catalog` | Audit lookup, equivalence, resolution, statistics | +| `release`, `revocation`, `generation`, `ssh` | Export, signature verification, emergency overlays, activation, rollback protection | + +The database schema lives in ordered migrations. JSON and receipt contracts have +their own schema strings and fail closed on unknown versions. See +[FORMATS.md](FORMATS.md) and [MIGRATING-0.4.md](MIGRATING-0.4.md). diff --git a/docs/CONSUMERS.md b/docs/CONSUMERS.md index c76313a..d6cad0b 100644 --- a/docs/CONSUMERS.md +++ b/docs/CONSUMERS.md @@ -1,65 +1,100 @@ # Consumer and compatibility contract -## Rust library +## Use the policy-enforced reader -Use `release::verify_signed(generation, publisher_signers, publisher_identity, -reviewer_signers)` once per production generation and reuse the returned reader. -`Registry::open(generation, trusted_pin)` is the lower-level path when the pin -distributor is also trusted for the complete review decision. `lookup(query, -limit)` returns evidence and complete ambiguity counts; `resolve_explained` returns a reviewed candidate or -a typed abstention reason with counts. Exact reverse views cover entity IDs, -URLs/domains, popularity and Curlie categories. Check the compiled example and API -docs for exact types. `selection_context` binds the full alternative set for -downstream query review. Preserve returned provenance, scopes, counts and attribution. +For a production generation, call +`release::verify_signed(generation, publisher_signers, publisher_identity, +reviewer_signers)` once and reuse the returned `Registry`. `Registry::open` is the +lower-level API for deployments that already trust an exact `COMPLETE.json` +SHA-256 for the whole publication decision. -For local integration, point a Cargo dependency at -`crates/argand-site-registry` inside an extracted standalone source tree. Once an -upstream repository is published, use its actual Git URL and a full reviewed `rev` -pin. Do not invent a crates.io version or track a mutable branch in production. -Both crates remain in this workspace; the atomic helper is a relative dependency. +`lookup` returns source evidence, candidates, ambiguity counts and attribution. +`resolve_explained` returns either one policy-qualified destination or a typed +abstention. It requires an approved name binding and approved edge under the +receipt's policy. Preserve the full response, especially `destination: null`, +`status`, counts, selected scope, evidence and attribution. + +Reverse views cover entity IDs, exact URLs, hostnames, registrable domains, +source-specific popularity, Curlie categories and observations. They are audit +operations and do not imply ownership or admission. ## CLI and other languages -`lookup`, `resolve`, `entity`, `lookup-web`, `popularity`, `category`, `stats`, -`evaluate`, `verify` and the other commands emit JSON. The Python example -passes arguments directly to the native executable, preserving query text and the -entire response. A nonzero exit is an error. `destination: null` is a successful -abstention, not a request to pick the first lookup candidate. Render source names -as untrusted text and satisfy their source-specific attribution requirements. +Every CLI command emits one JSON value to stdout; typed export and diff commands +write bounded JSONL files. A nonzero exit is an error. A successful resolution +with a null destination is an intentional abstention, not an instruction to use +the first lookup result. -Use a bounded process or service wrapper appropriate to your workload. The Python -example has a 60-second timeout and invokes a fresh reader per request; for repeated -low-latency queries, use the reusable Rust reader. No hosted API or Python package -registry publication is claimed by this repository. +The Python example passes an argument array directly to the native CLI and keeps +its 60-second timeout and complete JSON response. Use the Rust reader for repeated +low-latency queries. No hosted API, crates.io release or Python package is claimed. -## SQLite, JSONL and license scope +Render all imported names, categories, URLs and evidence as untrusted data. Apply +source attribution and application-specific malware/content policy. -Distribute `registry.sqlite`, `COMPLETE.json`, `LICENSE_SOURCES.md` and -`ATTRIBUTION.json` together, plus the publisher signature when applicable. The -database contains audit records and source descriptions. Default JSONL export -omits descriptions and includes fact provenance plus an attribution envelope. -It is an assertion export, not a self-contained signed list of admitted routes. -Raw SQL inspection is useful for audit; it does not implement resolution policy. +## Current contracts -Code version 0.3.0 uses schema version 3 and `argand.site-rules/v3`. -It adds reviewer-trust enforcement for consumers, private authenticated SQLite -snapshots, exact-stream import checks, bounded outputs and metadata-bound identity -decisions. Schema/rule contracts remain versioned independently in -receipts. Unsupported contracts fail closed. Pin source releases, -compile consumers and replay fixed fixtures before upgrades. Preserve import and -review history; never mutate complete generations to migrate them. +Code version 0.4.0 uses writer schema 5 and `argand.site-rules/v4`. +`COMPLETE.json` uses `argand.site-registry/v2` and binds: + +- authenticated `registry.sqlite` bytes; +- the active source coverage graph and exact PSL source; +- review policy and exact reviewer trust-root bytes; +- decision-time policy; +- source license and machine-readable attribution files; and +- entity, property, edge and rejection counts. + +Normal `export` uses `argand.site-export/v2`, contains selected active nonrejected +facts, and marks every assertion `active`. `export-audit` uses the same schema with +mode `audit` and includes superseded/rejected states and rejection reasons. Neither +contains a list of resolver-approved routes. + +Votes use `argand.site-vote/v1` and the OpenSSH namespace +`argand-site-registry-vote`. Consumers compile them under the exact receipt-bound +policy and trusted query time. Unknown schema or rule versions fail closed. + +Emergency feeds use `argand.site-revocations/v1` and the separate +`argand-site-registry-revocations` namespace. Call `revocation::verify` against +the cached `Registry` and publisher trust root, then use +`resolve_explained_with_revocations`. The CLI accepts the same four feed arguments +on `resolve`. It blocks exact revoked names, edges, and equivalences before route +selection and can choose another approved edge. Verify each replacement against +the previously accepted feed to enforce cumulative continuity. Feeds expire after +seven days. A feed from a newer compatible generation may add blocks to a cached +generation, but reinstatement requires installing the exact full generation so +the consumer can recompute the authenticated superseding quorum. + +CLI consumers pass the last feed and signature as `--previous-revocations` and +`--previous-revocation-signature` on subsequent `resolve` calls. The Rust API +passes the last `VerifiedRevocations` to `revocation::verify` before resolution. + +Distribute `registry.sqlite`, `COMPLETE.json`, `LICENSE_SOURCES.md`, +`ATTRIBUTION.json`, and `COMPLETE.json.sig` together. Obtain publisher and reviewer +trust roots independently. Raw SQL copies and JSONL extracts do not implement +resolution policy, expiry, revocation continuity or signature verification. + +## Compatibility + +V1 source manifests remain readable as isolated legacy provider/scope streams. +A deliberate legacy-compatible build policy can replay v0.3 reviews, but strict +0.4 builds require votes and reviewer trust. Current readers accept v2 receipts; +rollback checks may open pinned v1 receipts with rules v1-v3 only to compare +revocation history. + +Before upgrading, pin the source release by full signed Git revision, compile the +consumer and replay fixed fixtures. Preserve import, vote, observation and +revocation history. Never mutate a complete generation to migrate it. See +[MIGRATING-0.4.md](MIGRATING-0.4.md) and [FORMATS.md](FORMATS.md). ## Argand integration -UPSTREAM.json pins the exact Argand source baseline, including the beta agent's -selection-context API. The first standalone extraction preserves runtime Rust and -migration bytes. Argand main commit -`d9dfd1585ce21d9c4136bcc24fa01fe3bfb8ed6e` replaced its embedded workspace -crate with this signed `v0.3.0` release at full Git revision -`ac8282093d8a815c6227cff86e1f40714d510bcd`. +`UPSTREAM.json` records the original Argand extraction baseline and file hashes. +The last recorded downstream integration replaced Argand's embedded crate with +signed v0.3.0 revision `ac8282093d8a815c6227cff86e1f40714d510bcd` at Argand +commit `d9dfd1585ce21d9c4136bcc24fa01fe3bfb8ed6e`. -Develop the library here and update Argand through explicit reviewed revision-pin -changes. Each update must compare the old and new contracts and rerun Argand's -navigation compiler and API gates. Preserve existing registry receipts and public -navigation admission; a source dependency change does not activate a registry -generation or approve a destination. +Version 0.4 is handed off as a signed standalone revision. Argand should update its +full Git `rev` in a separate coordinated source/build window, compare contract +changes, and rerun navigation compiler, native resolver, API, abstention, +revocation and clean-process gates. Changing the code dependency does not activate +a registry generation or approve a public destination. diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 8c10673..14f1842 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -1,23 +1,20 @@ # Resolver evaluation -`evaluate` replays authored judgments through the same native resolver used by -consumers. It opens one externally pinned immutable generation, streams up to the -configured case limit and reports correctness plus native p50/p95 query latency. -Pass an explicit `--at` time when the report must be replayable across approval -expiry boundaries; the chosen clock is included in the report. -The input is bounded to 16 MiB, each line to 64 KiB and case IDs must be unique. - -Each nonempty JSONL line has this form: +`evaluate` streams authored JSONL judgments through the same pinned native reader +used by consumers. It reports correctness and p50/p95 latency. Pass `--at` for a +reproducible policy clock; approval expiry and future votes otherwise depend on +current time. Inputs are bounded to 16 MiB, lines to 64 KiB, and case IDs must be +unique. ```json {"id":"facebook-primary","query":"facebook","locale":null,"country":null,"expected_status":"resolved","expected_entity_id":"argand:entity:SOURCE:ID","expected_url":"https://www.facebook.com/"} ``` -`locale`, `country`, `expected_entity_id` and `expected_url` are optional. Status -is required and is one of `resolved`, `no_name_match`, `ambiguous_identity`, -`safety_limit_exceeded`, `no_eligible_destination`, `no_active_review`, -`region_mismatch` or `ambiguous_destination`. Expected URLs must be copied from a -pinned registry, including normalization such as a trailing slash. +Optional fields are `locale`, `country`, `expected_entity_id`, and `expected_url`. +Current statuses are `resolved`, `no_name_match`, `ambiguous_identity`, +`no_active_name_review`, `safety_limit_exceeded`, `no_eligible_destination`, +`no_active_review`, `region_mismatch`, and `ambiguous_destination`. Copy expected +URLs from a pinned registry, including normalization such as trailing slash. ```bash argand-site-registry evaluate --generation /data/registry/reviewed \ @@ -27,9 +24,14 @@ argand-site-registry evaluate --generation /data/registry/reviewed \ jq -e '.failed == 0 and .passed == .total' /data/evaluation/report.json ``` -Keep the corpus version and digest with the report. Include canonical names, -aliases, Unicode normalization, multiple scripts, every served country/locale, -unknown names, deceptive lookalikes, ambiguous entities, expired/revoked reviews -and tied destinations. Use synthetic or authorized query material; do not commit -private user logs. Latency values are process and hardware measurements. Compare -them only with an equivalent environment and sufficient sample size. +Keep corpus version and digest, registry pin, policy digest, reviewer-trust digest, +process build and hardware with the report. Cover canonical names, aliases, +Unicode normalization, scripts, regions, unknown names, lookalikes, ambiguity, +missing name votes, missing edge votes, correlated reviewers, stale policy, +stale observations, expiry, revocation, sticky supersession and tied routes. +Segment failures by safe abstention and wrong resolved route; any wrong resolved +route is a release blocker. + +Use synthetic or authorized queries and do not commit private user logs. Latency is +process and hardware evidence. Compare only equivalent native configurations with +sufficient samples. diff --git a/docs/FORMATS.md b/docs/FORMATS.md new file mode 100644 index 0000000..3f506fd --- /dev/null +++ b/docs/FORMATS.md @@ -0,0 +1,61 @@ +# Versioned formats + +Version 0.4 uses writer schema 5 and `argand.site-rules/v4`. Schema identifiers +are independent from the crate version. Unknown schemas and rules fail closed. + +| Artifact | Current schema | Purpose | +| --- | --- | --- | +| Source manifest | `argand.site-source/v2` | Exact source object plus typed coverage | +| Generation receipt | `argand.site-registry/v2` | Hash-bound immutable generation contract | +| Active/audit JSONL | `argand.site-export/v2` | Source-bearing assertions with selection state | +| Evidence bundle | `argand.site-evidence-bundle/v1` | Exact evidence signed by reviewer votes | +| Vote | `argand.site-vote/v1` | Authenticated approve/revoke decision | +| Review policy | `argand.site-policy/v1` | Threshold, groups, revocation, and separation rules | +| Observation batch | `argand.site-observation-source/v1` | Pinned JSONL object and rights | +| Observation | `argand.site-observation/v1` | Normalized subject-bound observation | +| Observer capture | `argand.site-observer-capture/v1` | Cache-only replay input | +| Emergency revocations | `argand.site-revocations/v1` | Cumulative publisher-signed offline block overlay | +| Active pointer | `argand.site-current/v2` | Signed generation pin plus revocation continuity | +| Diff | `argand.site-diff/v4` | Typed change stream across generations | + +Source manifest v2 coverage is one of `full`, `partition`, or `delta`. A delta +names its exact base, positive consecutive sequence, and superseded source IDs. +A full source cannot compose with active partitions. Overlap, gaps, cycles, +missing bases, cross-provider supersession, and mixed legacy/typed frontiers fail +the build. + +The normal export emits selected active facts only and excludes rejected facts. +Every assertion has `selection_state: "active"`. Audit export includes active, +superseded, and rejected facts; rejected assertions include their reason. Both +modes redact Curlie descriptions by default and neither represents approved +navigation routes. Use `resolve` for admission. + +Votes bind the subject kind and fingerprint, evidence-bundle digest, policy +digest, reviewer, decision, reason, asserted review time, optional expiry, exact +role/locale/country scope, and revocation supersession IDs. Exact JSON bytes are +signed with OpenSSH namespace `argand-site-registry-vote`. Writer `accepted_at` +is separate and cannot be supplied by the reviewer. + +Policy fields define base thresholds for names, edges, and equivalences; maximum +approval and observation ages; reviewer groups; publisher separation; source and +drift holds; and optional stricter `risk_thresholds` for `source_conflict` and +`dangerous_drift`. Unknown fields and risk classes fail closed. + +Emergency feeds retain every authenticated revocation ID across policy epochs, +active state, any explicit superseding quorum, and a refresh deadline no more +than seven days after the effective time. Exact JSON bytes use OpenSSH namespace +`argand-site-registry-revocations`. A feed can overlay a cached generation only +when rules, policy, and reviewer trust-root digests match. Feed continuity rejects +dropped subjects or vote IDs. A feed from another compatible generation may only +add blocks. Removing a block requires the exact full generation so the consumer +can recompute the signed superseding quorum; expired feeds fail closed. + +Legacy source manifests remain readable as isolated `(source, scope)` streams. +Legacy 0.3 reviews remain available only through the explicit compatibility +policy. New strict builds use votes and a receipt-bound reviewer trust root. + +The contract decisions are recorded in [ADR 0001](adr/0001-typed-source-coverage.md), +[ADR 0002](adr/0002-granular-trust-subjects.md), +[ADR 0003](adr/0003-votes-revocations-and-publishers.md), +[ADR 0004](adr/0004-active-and-audit-views.md), and +[ADR 0005](adr/0005-full-delta-release-identity.md). diff --git a/docs/INDEX.md b/docs/INDEX.md index 5c743d1..7efec22 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -3,6 +3,9 @@ - [README](../README.md): build, examples and scope. - [Operator guide](../crates/argand-site-registry/README.md): all source commands, schema, reviews, regional resolution, releases and update configuration. +- [Architecture](ARCHITECTURE.md): source, observation, vote, policy and release boundaries. +- [Versioned formats](FORMATS.md): current schema identifiers and compatibility. +- [Migrating to 0.4](MIGRATING-0.4.md): writer migration and trust transition. - [Source licenses](../crates/argand-site-registry/LICENSE_SOURCES.md): exact terms and attribution. - [Consumers](CONSUMERS.md): Rust, Python/CLI, data distribution and Argand transition. - [Trust](TRUST.md): enforced checks and publisher/consumer responsibilities. @@ -10,7 +13,17 @@ - [Evaluation](EVALUATION.md): bounded JSONL judgments and result interpretation. - [Releasing](RELEASING.md): CI, source signing and archive verification. - [Validation](VALIDATION.md): independent builds and native acceptance evidence. +- [Version 0.4 security review](SECURITY-REVIEW-0.4.md): threat boundaries, + resolved findings and residual operator responsibilities. - [Contributing](../CONTRIBUTING.md), [governance](../GOVERNANCE.md), [security](../SECURITY.md): proposals, decisions and incidents. - [Extraction design](superpowers/specs/2026-09-12-standalone-design.md) and - [implementation plan](superpowers/plans/2026-09-12-standalone.md). + [initial implementation plan](superpowers/plans/2026-09-12-standalone.md). +- [0.4 and beyond plan](superpowers/plans/2026-09-13-v0.4-and-beyond.md). +- Architecture decisions: [coverage](adr/0001-typed-source-coverage.md), + [trust subjects](adr/0002-granular-trust-subjects.md), + [votes and publishers](adr/0003-votes-revocations-and-publishers.md), + [active/audit views](adr/0004-active-and-audit-views.md), + [release identity](adr/0005-full-delta-release-identity.md), + [source lineage](adr/0006-source-lineage.md), and + [embedding intent](adr/0007-distribution-and-embedding.md). diff --git a/docs/MIGRATING-0.4.md b/docs/MIGRATING-0.4.md new file mode 100644 index 0000000..beb13f6 --- /dev/null +++ b/docs/MIGRATING-0.4.md @@ -0,0 +1,53 @@ +# Migrating from 0.3 to 0.4 + +Back up the writer database, source cache, generations, approvals, and trust files +before upgrading. Do not edit a complete generation in place. + +1. Install the 0.4 binary and run `verify` against every retained 0.3 generation + using their existing trusted pins. +2. Open the writer with 0.4. Migrations add authenticated votes and immutable + observation batches, moving `PRAGMA user_version` from 3 to 5 without removing + source or legacy review history. +3. Keep existing v1 source manifests as isolated legacy scopes. Use v2 manifests + with explicit full/partition/delta coverage for new replacement chains. +4. Use the strict reference policy or supply a reviewed policy JSON. Create an + independent OpenSSH reviewer allowed-signers file. Strict `build` requires + `--reviewer-trust`; its exact bytes are bound into the receipt. +5. Use `review-queue` and `evidence` to find each name, edge, and equivalence that + needs votes. Prepare, sign, verify, and append votes, then rebuild. +6. Expect `resolve` to abstain until the selected name and destination edge each + satisfy the new quorum. Audit `lookup` remains available throughout migration. +7. Compare `stats`, `diff`, `export`, and `export-audit`; replay evaluation; then + sign and activate with a publisher identity and key that did not approve. + +The legacy compatibility policy exists for controlled replay and transition. It +is not the CLI default and does not provide the reference two-reviewer guarantee. +Legacy approvals remain auditable. A publisher should collect new votes rather +than silently translating old reviews into quorum votes. + +## Behavior changes + +- Name bindings and website edges now have separate review identities. A new alias + needs its own name votes and cannot inherit a route. An accepted alias update + does not invalidate an unchanged website edge. +- Votes bind the exact review policy. Changing thresholds or reviewer groups makes + earlier votes stale under the new policy; the source evidence is unchanged. +- Reviewer time is advisory. Effective validity starts at the later of signed + `reviewed_at` and writer `accepted_at`, and ends no later than 90 days after + acceptance. +- One retained authenticated revocation is sticky across policy epochs. Ordinary + later approvals do not clear it. Every approval in a fresh quorum must + explicitly supersede every active revocation ID. +- Publisher-signed cumulative feeds can apply new blocks to an older compatible + pinned registry before the complete replacement arrives. Their policy, rules, + reviewer trust, and prior-feed continuity are verified. +- Active export omits superseded and rejected facts. `export-audit` retains them + with explicit state. +- A strict receipt binds selected coverage, review policy, reviewer trust bytes, + and the decision-time contract. Consumers reject unknown contracts. +- Observation imports and observer caches are evidence only. New or changed + observations make existing edge evidence stale and return the item to review. + +The 0.4 reader accepts current v2 receipts. Rollback validation can open pinned +v1 receipts using rules v1-v3 only to compare retained revocations. Preserve old +generation directories and pins until every consumer has moved successfully. diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index 7fbf23f..d8b7a48 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -1,82 +1,114 @@ # Dataset publisher runbook -This runbook creates a reviewable candidate and a signed activation. Source update, -review and release keys are separate authorities. Scheduled jobs may acquire, -import and build candidates; they do not approve, sign or activate destinations. +Scheduled acquisition and observation jobs create evidence and candidates. Human +reviewers vote on exact bundles. A separate publisher signs and activates an +accepted generation. -## Trust roots and local state +## Prepare trust and state -Keep cache, mutable store, immutable generations, reviewer keys, release keys and -consumer trust files outside the checkout. An OpenSSH reviewer trust file contains -one accepted principal and public key per line: +Keep source caches, captures, writer database, generations, reviewer keys, +publisher keys and consumer trust files outside the checkout. A reviewer +allowed-signers file contains one principal and public key per line. Add OpenSSH +validity options when rotating keys and preserve old keys for retained history. -```text -operator@example.org ssh-ed25519 REVIEWER_PUBLIC_KEY -``` +Create a reviewed policy JSON when the reference policy is not appropriate. The +reference policy requires two independent identities, groups and physical keys +for every name, edge and equivalence; sticky revocations and publisher separation +are mandatory. Record actual group membership in `reviewer_groups`. Optional +`risk_thresholds` can require a larger quorum for `source_conflict` or +`dangerous_drift`; blocking those risks remains separately configurable. -For rotation, retain an old public key with an OpenSSH `valid-before` option -covering its signed decision times. New decisions are checked at append time, so -an expired key cannot submit backdated reviews; historical release verification -uses each authenticated `reviewed_at`. Remove a retired key only after no retained -generation or review log depends on it. +## Build and inspect a candidate -Distribute the release publisher public key to consumers through an independent -authenticated channel. Do not put private keys, production trust files or source -datasets in Git or CI. The isolated source CI runner has none of these files. +1. Import only manifests whose source, format, license and typed coverage were + checked. Keep the object and acquisition receipt. +2. Build with `--reviewer-trust`; record the returned receipt pin. +3. Run `verify`, `stats`, `diff`, `export-audit` and the fixed evaluation corpus. +4. Inspect `review-queue`, `evidence`, exact entity/domain reverse lookups and + source-separated popularity. +5. Run candidate observations on a bounded schedule and import their manifests. + Review `drift` and `revocation-candidates`; rebuild after imports. -## Candidate acceptance +Queue viewing is read-only. Observation import changes evidence bundles but never +route state. -Run `update` or the explicit download/import/build commands from the operator guide. -For every candidate generation: +## Create authenticated votes -1. Verify its externally recorded receipt pin with `verify`. -2. Run `stats` and compare source snapshots, selected sources, facts, rejections, - reviews, database bytes and upcoming expiry with the prior accepted generation. -3. Run `diff` against the prior pin. Investigate every source-selection, identity, - name, property, edge, popularity, review and equivalence change. -4. Use `lookup`, `entity`, `lookup-web`, `popularity` and `category` to inspect exact - source evidence and conflicts. Popularity never proves ownership. -5. Replay the maintained evaluation corpus with `evaluate`; require zero judgment - mismatches and compare latency with a documented hardware/process baseline. - -## Authenticated decisions - -Create a bounded review JSON from the exact candidate fingerprint and evidence. -Sign its exact bytes and append it through the CLI: +Use `prepare-vote` for a name or edge, or `prepare-equivalence-vote` for an exact +entity pair. The command writes canonical JSON containing the current evidence +and policy digests. Review those exact bytes, then sign them: ```bash -ssh-keygen -Y sign -n argand-site-registry-review \ - -f /secure/reviewer-key review.json -argand-site-registry review --database /data/registry/import.sqlite \ +ssh-keygen -Y sign -n argand-site-registry-vote \ + -f /secure/reviewer-one vote.json +argand-site-registry verify-vote \ --generation /data/registry/candidate --pin "$CANDIDATE_PIN" \ - --decision review.json --signature review.json.sig \ + --decision vote.json --signature vote.json.sig \ --allowed-reviewers /secure/reviewer-allowed-signers \ - --identity operator@example.org + --identity reviewer-one +argand-site-registry vote \ + --database /data/registry/import.sqlite \ + --generation /data/registry/candidate --pin "$CANDIDATE_PIN" \ + --decision vote.json --signature vote.json.sig \ + --allowed-reviewers /secure/reviewer-allowed-signers \ + --identity reviewer-one ``` -Identity equivalence decisions use the same signed JSON and reviewer namespace. -Rebuild after appending decisions, then repeat the complete diff and evaluation. -The release command re-verifies every retained reviewer signature against the -current reviewer trust file. Missing, altered or no-longer-trusted proofs stop it. +Repeat with enough independently controlled identities, groups and keys. Use +`equivalence-vote` to append equivalence votes after +`verify-equivalence-vote`. Rebuild and check the compiled decision. Never share +one private key under several reviewer names. + +An approval expires within 90 days and begins no earlier than writer acceptance. +A revocation has no expiry. To restore a revoked subject, every new approval in a +complete quorum must list every active revocation ID in `supersedes`. ## Sign and activate +Run all offline gates and the separate network-enabled dependency audit. The +publisher identity and physical key must not have supplied any reviewer vote. + ```bash argand-site-registry sign --generation /data/registry/reviewed \ - --pin "$REVIEWED_PIN" --key /secure/release-key \ - --allowed-reviewers /secure/reviewer-allowed-signers + --pin "$REVIEWED_PIN" --key /secure/publisher-key \ + --allowed-reviewers /secure/reviewer-allowed-signers \ + --identity registry-publisher argand-site-registry activate --generation /data/registry/reviewed \ --current /data/registry/current.json \ - --allowed-signers /secure/release-allowed-signers \ + --allowed-signers /secure/publisher-allowed-signers \ --allowed-reviewers /secure/reviewer-allowed-signers \ --identity registry-publisher ``` -Record the source commit, candidate and accepted receipt pins, typed diff, evaluation -report, reviewer trust-file digest, release signer identity and activation receipt -in an immutable operator log. Activation refuses a rollback that drops a distributed -revocation. Deliver a new current pointer/pin to every consumer and bound their caches. +Record the source-code revision, provider manifest IDs, candidate and reviewed +pins, policy and reviewer-trust digests, diff, evaluation report, publisher +identity, and activation result in an immutable operator log outside Git. +Distribute the complete generation, signature, and independently authenticated +publisher trust root. Confirm every consumer received the new revocation state. -For an incident, append a signed revocation, rebuild with full history, inspect, -evaluate, sign and activate. Preserve the suspect source bytes, generation and proofs. -Follow [SECURITY.md](../SECURITY.md) for private reporting and key compromise. +For an incident, preserve the suspect evidence, append a signed revocation, +rebuild with full history, evaluate, sign and activate. Use +`revocation-candidates` only as review input; it never signs or appends a vote. +To protect pinned consumers before the full generation arrives, publish a small +cumulative feed under the distinct publisher namespace: + +```bash +argand-site-registry export-revocations \ + --generation /data/registry/revoked --pin "$REVOKED_PIN" \ + --effective-at 2026-09-13T00:00:00Z --output /data/revocations.json +argand-site-registry sign-revocations \ + --generation /data/registry/revoked --pin "$REVOKED_PIN" \ + --input /data/revocations.json --output /data/revocations.json.sig \ + --key /secure/publisher-key \ + --allowed-reviewers /secure/reviewer-allowed-signers \ + --identity registry-publisher +``` + +Signing recomputes the feed from the pinned generation and re-verifies every +reviewer vote. Distribute the exact feed, signature, and publisher trust root. +When replacing a feed, verify it with the prior feed and signature so a mirror +cannot discard a revocation. Refresh feeds before their seven-day deadline. A +newer compatible feed can block an older pinned generation, but clearing that +block requires installing its exact full generation so the consumer can recompute +the authenticated superseding quorum. +Follow [SECURITY.md](../SECURITY.md) for private disclosure and key compromise. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index eca95a5..393b42f 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -87,7 +87,11 @@ receipt; the initial release tool packages source only. Source releases contain no provider datasets or real approvals. Dataset publishers follow the operator guide: import, inspect, review, build, diff, sign and activate. Follow the [publisher runbook](PUBLISHING.md); release signing requires the external -reviewer trust file and re-verifies every stored decision signature. +reviewer trust file and re-verifies every stored decision signature. Version 0.4 +strict releases also require `sign --identity`; the publisher identity and +physical key must not have supplied an approval vote. Run +`cargo audit --deny warnings` as a separate network-enabled gate and record the +result in [VALIDATION.md](VALIDATION.md). The dataset namespace `argand-site-registry` is distinct from the source namespace above. Keep `LICENSE_SOURCES.md` and `ATTRIBUTION.json` with the database and receipt. The weekly update example creates candidates. It never approves, renews, signs or diff --git a/docs/SECURITY-REVIEW-0.4.md b/docs/SECURITY-REVIEW-0.4.md new file mode 100644 index 0000000..1673945 --- /dev/null +++ b/docs/SECURITY-REVIEW-0.4.md @@ -0,0 +1,76 @@ +# Version 0.4 security review + +Reviewed 2026-09-13 after implementation and before release promotion. + +## Scope and trust boundaries + +The review covered source-manifest and acquisition validation, typed coverage, +SQLite migrations and projections, observation import and candidate-site network +access, reviewer votes and policy compilation, generation and revocation signing, +consumer verification, CLI argument relationships, systemd isolation, dependency +advisories, source packaging, and accidental credential or dataset disclosure. + +The design treats provider bytes, websites, DNS answers, redirects, observation +batches, public proposals, mirrors, and lookup results as untrusted. Reviewer and +publisher private keys, the mutable writer database, consumer trust roots, and the +host operating system remain privileged. A source assertion or observation cannot +approve a route. One reviewer cannot satisfy the reference approval quorum, while +one authenticated revocation can stop an exact subject. + +## Findings fixed before release + +1. **Cross-generation revocation reinstatement:** a publisher-signed feed could + claim supersession while an older consumer lacked the reviewer votes needed to + recompute it. Cross-generation feeds now only add blocks. Clearing a block + requires the exact full generation containing the authenticated fresh quorum. +2. **Stale emergency feeds:** feeds previously had no artifact deadline and a + verified object could be reused indefinitely. Feeds now expire within seven + days, and freshness is checked both during verification and every resolution. +3. **Resolver continuity:** `verify-revocations` accepted a previous feed, while + `resolve` had no equivalent input. `resolve` now accepts the last feed and + signature and refuses replacements that discard subjects or vote IDs. +4. **Publisher/reviewer separation:** physical-key and identity checks covered + approval voters only. They now cover every reviewer vote, including emergency + revocations, and remove a rejected signature output. +5. **Observation-batch mutation:** completed batch counters, state, and membership + were not all protected by schema triggers. A batch must now be created open, + can complete once only with its exact row count, and cannot accept later rows, + change, or be deleted. +6. **Policy-epoch revocation bypass:** a new policy epoch retained an old signed + revocation in the audit log but excluded it from compilation. All authenticated + revocations now remain sticky across policy epochs until a fresh quorum under + the active policy explicitly supersedes them. +7. **Special-address observation targets:** the outbound filter omitted several + IPv4 and IPv6 special-use ranges. The observer now also blocks IPv4-compatible, + site-local, translation, discard, benchmarking, ORCHID, documentation, and 6to4 + destinations before constructing a pinned client. + +Regression coverage includes expired verified objects, signed cross-generation +reinstatement attempts, physical reviewer-key reuse by a publisher, immutable +observation batches and late inserts, policy-epoch revocation changes, signature +and feed tampering, redirect loops, private and reserved address ranges, +compressed bodies, malformed markup, extraction caps, +and a deterministic 512-case parser mutation corpus. + +## Review result + +No known critical, high, or medium security finding remains in the reviewed 0.4 +scope. `unsafe` Rust is forbidden workspace-wide. External SSH operations use +argument vectors and descriptor-bound private temporary files. Generation reads +hash a no-follow source into a private unlinked SQLite snapshot before querying. +Network clients disable ambient proxies; source acquisition uses reviewed HTTPS +endpoints and explicit byte limits; the observer pins an entirely public DNS set +per hop and bounds redirects, headers, body bytes, bandwidth, time, and extracted +links. + +`cargo audit --deny warnings` scanned 1,243 RustSec advisories across 272 locked +dependencies without a finding. The complete offline gate separately exercises +strict Clippy, documentation, unit and integration tests, native CLI behavior, +consumer parity, source-package defenses, and the synthetic five-source import. +`systemd-analyze verify` accepted the observer service and timer; its only output +was an unrelated warning from the host's installed `arch-audit.service`. + +This review authenticates software behavior, not provider truth or a public +dataset. Publishers must protect writer and signing authority, inspect evidence, +retain the last accepted feed, refresh it before expiry, and distribute trust +roots through an independent authenticated channel. diff --git a/docs/TRUST.md b/docs/TRUST.md index 99a4f61..03ea4c7 100644 --- a/docs/TRUST.md +++ b/docs/TRUST.md @@ -1,73 +1,115 @@ # Trust and evidence policy +Argand Site Registry is designed to fail closed. Imported assertions and crawler +observations become review evidence. Only policy-qualified votes can make a name +or destination resolvable, and only a separately authenticated generation should +reach consumers. + ## What the implementation enforces -Inputs use reviewed source adapters and explicit manifests. URL normalization, -PSL parsing and stable identities are deterministic. Fact provenance, conflicting -evidence and source-specific popularity remain separate. Names and similar -hostnames never silently join entities. Invalid URLs and malformed imports fail -validation; a partial import does not replace a complete source selection. +Source adapters accept only documented providers and formats. Manifests bind the +exact object, origin URL, source-native snapshot, license, retrieval time, byte +length, digest, and typed coverage. Full, partition and delta graphs reject gaps, +cycles, overlap, cross-provider replacement and ambiguous active branches. Failed +or incomplete imports cannot replace complete evidence. -Destination and identity decisions bind exact evidence fingerprints, including -names, assertions and normalization context. Reviews expire within 90 days. -Changed evidence invalidates earlier approvals. Resolution abstains on ambiguity, -ties, missing approval or ineligible claims; regional scopes must explicitly -match. Reviewer decisions are signed under a dedicated SSH namespace and the local -writer retains their exact decision/signature bytes in the append-only review log. +URL and domain normalization is deterministic and uses the complete retained +Public Suffix List, including PRIVATE rules. Names and hostnames never merge +entities. Source-specific popularity stays separate from identity. Every fact +keeps source, source identifier, selector, license, retrieval time, confidence and +raw evidence needed for audit. -Generations bind the database, license document and attribution to a completion -receipt. Consumers provide a trusted hash or verify an external publisher key. -Release signing re-verifies every stored decision against an external reviewer -trust file. Activation checks publisher signatures, re-verifies every review -against a separately supplied reviewer trust file and refuses rollback that loses -distributed revocations. Reviewer validity epochs are evaluated at decision time -for retained history while new decisions must pass the trust policy at append -time. Updates build candidates and cannot approve, sign or activate them. +Names, entity-to-property edges and entity equivalences have independent material +fingerprints. Under the reference policy, `resolve` needs two independent votes +for the matched name and two for the selected edge. Reviewer groups and physical +SSH public keys are deduplicated, so aliases for one person or key do not satisfy +quorum. Regional roles require explicit locale or country scope. + +A vote signs exact JSON in the `argand-site-registry-vote` namespace and binds the +current assertion, evidence bundle and policy epoch. The writer supplies +`accepted_at`; effective validity starts at the later of acceptance and the +reviewer's time and ends no later than 90 days after acceptance. Future, expired, +wrong-policy, stale-evidence, malformed, untrusted and altered votes do not count. + +A retained authenticated revocation is sticky across policy epochs. It blocks +the exact subject until every member of a complete fresh quorum explicitly +supersedes every active revocation ID. Sequence order alone cannot restore a +route. Activation prevents rollback past retained legacy or vote revocations. + +A cumulative emergency feed carries the same granular revocation identities under +a distinct publisher signature namespace. Compatible pinned consumers apply it +before name, equivalence, and edge selection. Replacement feeds cannot discard +previous subjects or vote IDs. Cross-generation feeds can only add blocks; a block +can be cleared only against the exact full generation containing the authenticated +superseding quorum. Feed artifacts expire after seven days and must be refreshed. + +The observer accepts only an eligible imported edge. It uses public DNS pinning +for every hop, rejects credentials, private/link-local/reserved targets, +nondefault ports and HTTPS downgrade, and bounds time, redirects, response headers, +raw body bytes, bandwidth and extracted links. It rejects compressed bodies and +redirect loops. Capture replay checks the complete chain and +body digest without network access. HTTP, redirect, canonical, hreflang, JSON-LD, +sitemap, country-selector, DNS, TLS and failure records remain observations. They +cannot approve a name, ownership relationship or role. + +The observation contract also accepts rights-reviewed domain-registration state +and malware-policy results without naming a vendor. No such provider is built in; +an operator must verify commercial-reuse terms and preserve its exact source and +rights declaration before importing those records. + +Generations bind authenticated SQLite bytes, selected coverage, policy, reviewer +trust bytes, licenses, attribution, and decision-time rules into `COMPLETE.json`. +Readers verify the receipt pin and copy the database into a private unlinked file +before SQLite opens it. Release signing and activation reverify every stored +signature. The strict policy rejects a publisher identity or physical key used for +any reviewer vote. ## What a publisher must establish -The reviewer name and evidence locator in a decision are operator assertions. -The CLI authenticates exact decision bytes to an allowed SSH signer and validates -their structure and evidence binding; it does not retrieve the cited evidence or -prove website ownership. -Protect the writer database and signing key with separate operating permissions. -Restrict who can author decisions and require human review before release signing. +The software verifies evidence integrity and decision authorization. A publisher +still has to determine that the source and observation evidence support the exact +entity, URL, relationship and role. TLS, DNS control, a redirect, `sameAs`, ccTLD, +popularity or source confidence alone is insufficient. -Publish dated evidence supporting the exact entity, URL, relationship, role and -country/locale. Prefer independently corroborated primary evidence with immutable -capture identifiers. Record contrary evidence and uncertainty. TLS, DNS control, -registrable-domain spelling, redirects, `sameAs` or popularity alone cannot -establish every identity or role claim. Future crawler observations remain inputs -to review. Confidence values are assertion scores, not calibrated probabilities. +Publishers should: -Choose expiry based on volatility, within the enforced maximum. Do not renew -blindly on a timer. Expired approval should lead to abstention until evidence is -reviewed. Disclose editorial conflicts and use an independent reviewer for a -disputed claim when possible. The implementation is a local single-writer tool -with externally authenticated reviewer keys; it does not provide accounts or an -enforced quorum. +- keep acquisition, writer, reviewer and release authorities separate; +- protect reviewer and publisher keys outside the repository and CI; +- configure real organizational reviewer groups instead of relying only on unique + identity strings; +- examine conflicts and current independent evidence in each review bundle; +- choose shorter expiries for volatile or high-risk routes; +- refresh observation evidence independently from source import cadence; +- review drift and produce signed emergency revocations promptly; +- run diff, evaluation, license and signature gates before release; and +- preserve source objects, generations, trust roots, pins and revocations for + audit and recovery. + +Changing policy or reviewer trust produces a different receipt. It does not +silently reinterpret old votes as decisions under the new policy. Removing a key +can also make retained signature verification fail; plan rotations with OpenSSH +validity epochs and immutable history. ## What consumers must preserve -Authenticate a release before opening it. Consumers that rely on reviewer -separation must use `release::verify_signed` or `activate` with independently -distributed publisher and reviewer trust files; a receipt pin alone delegates the -whole release decision to whoever distributed that pin. Keep the full receipt pin -and required attribution with caches and exports. Use `resolve` for reviewed destinations, keep -null as abstention, and enforce application-specific malware/content/navigation -policy separately. A verified signature authenticates the publisher, not the truth -of every assertion. An official website may later be compromised. +Authenticate the publisher and reviewer trust roots independently, or obtain the +full receipt pin over a channel that is already trusted for the complete release +decision. A hash beside an untrusted download authenticates nothing. -Deliver revocations to every active consumer and derived catalogue, and bound cache -lifetimes. The resolver checks review expiry at query time; a detached cached URL -does not recheck itself. Preserve the current review history during rollback. -Copying SQLite rows or the JSONL export into a second resolver can bypass these -checks; use the native API or CLI for admission decisions. +Use `resolve` for navigation. `lookup`, exports, raw SQLite rows and observation +views are audit evidence and can include unreviewed, conflicting, superseded or +malicious claims. Preserve null destinations and typed abstention reasons. Apply +application-specific malware, content and destination policy because a legitimate +site can later be compromised. -## Community changes +Keep license and attribution artifacts with caches and derived datasets. Bound +cache lifetime by vote expiry and revocation delivery. A detached URL copied from +an earlier result no longer performs policy, freshness or rollback checks. -Treat public submissions as untrusted evidence. Do not execute submitted content -or copy review decisions into production automatically. Source allowlist and -attribution changes need both implementation tests and documented rights review. -Reject unknown sources until those checks are complete. Every publisher may apply -stricter admission rules, and must state its actual review and incident procedures. +## Community submissions + +Treat public issues, manifests, source files, captures and vote proposals as +untrusted input. They may suggest evidence but cannot write the protected store, +approve, publish or activate a generation. Unknown providers remain unsupported +until current commercial-reuse rights, format, lineage, attribution and tests are +reviewed. Public contribution does not imply production admission. diff --git a/docs/adr/0001-typed-source-coverage.md b/docs/adr/0001-typed-source-coverage.md new file mode 100644 index 0000000..6724aa4 --- /dev/null +++ b/docs/adr/0001-typed-source-coverage.md @@ -0,0 +1,26 @@ +# ADR 0001: Typed source coverage and supersession + +Status: Accepted, 2026-09-13. + +Source snapshots declare `full`, `partition`, or `delta` coverage. Partitions use +stable disjoint coordinates. Deltas name one exact base, a consecutive sequence, +and every directly superseded object. The build rejects ambiguous frontiers, +cycles, missing bases, cross-source links, overlaps, and mixed legacy/typed active +sets. Older objects and facts remain available for audit. + +Free-form scope strings were insufficient to distinguish replacement from +composition. Explicit coverage makes selection reproducible and prevents a +partial object from silently replacing unrelated source data. + +## Rejected alternatives + +Selecting the newest retrieval time repeats the v0.3 ambiguity and lets clock +skew choose authority. Treating every object as additive retains deleted facts. +Inferring overlap from source URLs or file names is source-specific and unsafe. + +## Compatibility + +V1 manifests remain isolated by exact provider and scope. A typed object may +replace legacy evidence only by naming its exact manifest ID in `supersedes`. +Readers that do not understand v2 must reject it. See the source-manifest and +coverage contracts in [FORMATS.md](../FORMATS.md). diff --git a/docs/adr/0002-granular-trust-subjects.md b/docs/adr/0002-granular-trust-subjects.md new file mode 100644 index 0000000..2b9bb63 --- /dev/null +++ b/docs/adr/0002-granular-trust-subjects.md @@ -0,0 +1,23 @@ +# ADR 0002: Separate names, website edges, and observations + +Status: Accepted, 2026-09-13. + +A name binding, entity-to-property edge, and crawler observation have separate +identities and decisions. A resolver must admit the matched name and the selected +website edge. Observations enter evidence bundles but never create ownership, +identity, or regional roles. + +This prevents a new alias from inheriting an existing route and avoids invalidating +an unchanged route when unrelated entity metadata changes. + +## Rejected alternatives + +One decision over the whole entity made harmless label changes invalidate every +route and let an injected alias inherit old authority. Promoting observer signals +directly would turn redirects or self-authored metadata into ownership claims. + +## Compatibility + +Legacy review rows remain auditable under the explicit compatibility policy. +Strict v0.4 resolution requires separate name and edge votes. V2 edge fingerprints +exclude unrelated entity revision metadata while retaining it in provenance. diff --git a/docs/adr/0003-votes-revocations-and-publishers.md b/docs/adr/0003-votes-revocations-and-publishers.md new file mode 100644 index 0000000..069d47d --- /dev/null +++ b/docs/adr/0003-votes-revocations-and-publishers.md @@ -0,0 +1,38 @@ +# ADR 0003: Signed votes, sticky revocations, and publisher separation + +Status: Accepted, 2026-09-13. + +Reviews are immutable signed votes compiled under a receipt-bound policy epoch. +The reference policy requires two reviewer identities, two independent groups, +and two physical SSH keys. One revocation blocks the exact subject until every +approval in a fresh quorum explicitly references all active revocation IDs. +Publisher identity and key must be separate from every reviewer vote. +Changing the policy epoch never clears a retained revocation; a fresh quorum +under the new policy must explicitly supersede it. + +Trusted writer acceptance time bounds validity. Reviewer timestamps cannot +backdate eligibility or extend an approval beyond 90 days after acceptance. + +Risk-class thresholds may raise the base name, edge, or equivalence quorum for +`source_conflict` and `dangerous_drift`. The reference policy also holds those +edges in disputed or probationary state until the underlying risk clears. + +Emergency feeds are cumulative, use the separate +`argand-site-registry-revocations` SSH namespace, retain superseded revocation +IDs, and can be applied to a compatible pinned generation before its replacement +arrives. Cross-generation feeds only add blocks. Reinstatement requires the exact +full generation containing the signed superseding quorum, and feed artifacts must +be refreshed at least every seven days. + +## Rejected alternatives + +Latest-decision-wins lets one later approval erase a revocation. Counting aliases +of one key as separate reviewers does not provide independence. Letting a release +publisher contribute approvals collapses review and publication into one actor. + +## Compatibility + +Legacy decisions remain available only under the explicit v0.3 compatibility +policy. Strict receipts bind the policy and reviewer trust-root digests. A policy +change creates a new epoch and old approvals become stale rather than being +silently reinterpreted. diff --git a/docs/adr/0004-active-and-audit-views.md b/docs/adr/0004-active-and-audit-views.md new file mode 100644 index 0000000..31e2c15 --- /dev/null +++ b/docs/adr/0004-active-and-audit-views.md @@ -0,0 +1,29 @@ +# ADR 0004: Separate active and audit views + +Status: Accepted, 2026-09-13. + +Normal export contains only selected, nonrejected facts. Audit export preserves +active, superseded, and rejected facts with an explicit state and rejection +reason. Neither export bypasses resolver policy. + +Operational consumers need an unambiguous current evidence view, while +investigators and publishers need conflicting and superseded evidence. One +ambiguous export could be mistaken for an approved route list. + +Version 0.4 keeps the complete audit history inside each generation so rollback, +diff, and incident inspection remain self-contained. Separating a compact runtime +projection from content-addressed cold audit bundles is deferred to v0.5 until +size and latency measurements justify the extra recovery surface. + +## Rejected alternatives + +Deleting superseded facts loses conflict and replacement evidence. Shipping only +the audit view makes accidental use as current state too easy. Splitting storage +before authenticated bundle verification exists risks publishing a runtime index +whose supporting evidence cannot be recovered. + +## Compatibility + +The normal JSONL envelope is v2 and contains active assertions only. Audit mode +uses the same envelope version with explicit active, superseded, rejected, and +tombstoned rows. V1 consumers must reject the new schema and migrate explicitly. diff --git a/docs/adr/0005-full-delta-release-identity.md b/docs/adr/0005-full-delta-release-identity.md new file mode 100644 index 0000000..3a0ce4a --- /dev/null +++ b/docs/adr/0005-full-delta-release-identity.md @@ -0,0 +1,30 @@ +# ADR 0005: Full and delta release identities + +Status: Accepted, 2026-09-13. + +Each source object has its own digest. The selected coverage graph, including +every active full, partition, base, and delta object and its precedence, has a +separate digest bound into the generation receipt. A future distributed dataset +delta must name exact base and target generation identities and preserve +revocation continuity. + +An unauthenticated `latest` locator can be a convenience pointer, but never the +trust root. Consumers authenticate a full receipt pin or publisher signature. + +Version 0.4 implements source-level full/partition/delta identity and cumulative +publisher-signed emergency revocation overlays. General downloadable registry +deltas remain a v0.6 distribution task because they also need mirror-independent +base/target authentication and consumer transaction semantics. + +## Rejected alternatives + +Mutable releases and unpinned `latest` URLs permit substitution and rollback. +Signing only a compressed archive makes alternate packaging unverifiable. Calling +a source delta a registry delta would hide changes introduced by review policy, +normalization, observations, or another provider. + +## Compatibility + +V2 receipts bind the selected coverage digest. Current pointers retain every +legacy and vote revocation across activation. Emergency feeds apply only when +their rules, policy, and reviewer-trust digests match the cached generation. diff --git a/docs/adr/0006-source-lineage.md b/docs/adr/0006-source-lineage.md new file mode 100644 index 0000000..542d99c --- /dev/null +++ b/docs/adr/0006-source-lineage.md @@ -0,0 +1,24 @@ +# ADR 0006: Source lineage and independence + +Status: Accepted design; implementation scheduled for 0.5. + +Corroboration must describe the direct provider, upstream dataset, transformation, +and snapshot. Two providers that copied the same upstream assertion do not count +as independent evidence merely because their URLs differ. Unknown lineage stays +unknown. + +Version 0.4 preserves provider-native provenance and never combines popularity or +same-domain evidence into ownership confidence. Version 0.5 will add explicit +lineage fields and independence-aware corroboration without rewriting history. + +## Rejected alternatives + +Counting provider names as independent evidence rewards copied datasets. Guessing +lineage from matching text creates another unsupported inference. Dropping a +source because lineage is unknown destroys useful conflicting evidence. + +## Compatibility + +The v0.4 policy does not award quorum from source count, so absent lineage cannot +inflate reviewer authority. Future lineage fields must be additive provenance; +old assertions remain byte-identifiable and are never rewritten as independent. diff --git a/docs/adr/0007-distribution-and-embedding.md b/docs/adr/0007-distribution-and-embedding.md new file mode 100644 index 0000000..8f86d32 --- /dev/null +++ b/docs/adr/0007-distribution-and-embedding.md @@ -0,0 +1,27 @@ +# ADR 0007: Distribution and proprietary embedding + +Status: Accepted, 2026-09-13. + +The project prioritizes a standalone AGPL-3.0-or-later CLI and Rust library plus +signed data artifacts. Broad proprietary embedding is not a 0.4 goal. Consumers +must assess AGPL obligations for their deployment and comply independently with +every provider data license and attribution term. + +The supported integration paths are the native CLI, reusable Rust reader, signed +generation format, and documented subprocess protocol. A future change to code +licensing, dual licensing, hosted APIs, or proprietary linking requires a separate +governance and legal decision; this ADR does not grant one. + +## Rejected alternatives + +A second permissively licensed verifier crate was considered for v0.4. It would +duplicate format and signature policy before the contracts have deployment data, +and could imply that provider datasets inherit the verifier's license. A network +API would add account, availability, and traffic-trust requirements to an offline +dataset component. + +## Compatibility + +Rust and subprocess consumers use the same receipt and JSON contracts. Broad +proprietary embedding is outside the supported v0.4 surface. Dataset users must +still follow each source license regardless of how they invoke the verifier. diff --git a/docs/superpowers/plans/2026-09-13-v0.4-and-beyond.md b/docs/superpowers/plans/2026-09-13-v0.4-and-beyond.md new file mode 100644 index 0000000..239f2dd --- /dev/null +++ b/docs/superpowers/plans/2026-09-13-v0.4-and-beyond.md @@ -0,0 +1,880 @@ +# Argand Site Registry v0.4 and Beyond Plan + +> **Status:** Version 0.4 phases 0 through 2 implemented and security-reviewed. +> Phases 3 and later remain the sequenced roadmap. +> +> **Baseline:** Clean `main` at `2861337`; runtime behavior is the tagged +> `v0.3.0` release at `ac82820`. The complete v0.3 acceptance suite passes. +> +> **Execution constraint:** Work in this standalone repository only. Do not use +> subagents, edit Argand's main checkout, share its build cache, acquire paid data, +> publish a dataset, sign with production keys, or deploy without the authority +> already established for that specific action. + +## Goal + +Turn the v0.3 evidence and release substrate into a production-grade, reusable +entity-to-website authority system that can publish a useful signed reference +registry while preserving conflict, provenance, commercial-reuse terms, human +review, abstention, revocation, and deterministic regional resolution. + +The implementation must remain useful in two modes: + +1. A local publisher builds and reviews its own registry from allowed sources. +2. A consumer verifies and queries a separately distributed signed reference + generation without trusting an unauthenticated download location. + +The work is divided into independently releasable stages. Correct active-state +semantics and review controls come before broader ingestion. Provider-scale proof +comes before a public reference dataset. A hosted service remains optional. + +## Product thesis + +Popularity lists answer which domains receive attention. They do not reliably +answer which entity controls a domain, whether a destination is current, or which +regional property is appropriate. Argand Site Registry should compile independent +source assertions, observed site relationships, and authenticated reviewer votes +into a signed generation that resolves only when policy is satisfied. + +The principal product outcome is a signed, explainable answer: + +```text +query + locale/country + -> reviewed name-to-entity binding + -> reviewed entity-to-property edge + -> active regional-selection policy + -> URL or typed abstention +``` + +A larger assertion database is not the goal by itself. Release quality is measured +by resolved-route correctness, useful coverage, freshness, and revocation speed. + +## Non-negotiable invariants + +- Keep raw inputs, normalized facts, crawler observations, reviewer decisions, + policy decisions, popularity, and release authority logically separate. +- Never infer entity equivalence or website ownership from similar names, domains, + redirects, TLS, DNS, `sameAs`, popularity, or shared upstream data alone. +- Preserve conflicting and rejected evidence with its source-native identity. +- Every imported or derived fact retains source, source identifier, license, + license evidence URL, retrieval timestamp, confidence, and derivation version. +- Every source adapter must be rights-reviewed, format-pinned, streaming, bounded, + reproducible, resumable where feasible, and idempotent. +- Automated updates may download, import, evaluate, and build candidates. They may + not approve, renew, sign, activate, or publish them. +- Resolution remains fail closed. Ambiguity, missing policy, expired evidence, + missing votes, unsupported locale semantics, or conflicting equal candidates + produces a typed abstention. +- Popularity remains source-separated evidence and never becomes an ownership vote. +- Existing v0.3 data remains auditable. Migrations must not reinterpret an old + approval, source scope, or timestamp as if it had been created under a new rule. +- Curlie descriptions remain redacted unless the exact distribution surface meets + its attribution obligations. +- Public submissions are untrusted proposals. They never mutate an active release. +- Cloudflare Radar, default Tranco, Cisco Umbrella, and other unverified sources + remain excluded. + +## Target architecture + +```mermaid +flowchart LR + A[Rights-reviewed source adapters] --> B[Immutable source manifests and raw cache] + B --> C[Writer store: records, facts, conflicts, tombstones] + C --> D[Normalized assertion graph] + E[Bounded candidate-site observer] --> F[Immutable observation bundles] + F --> D + D --> G[Review queue and evidence diff] + G --> H[Signed reviewer votes] + H --> I[Versioned publisher policy compiler] + I --> J[Immutable active projection] + J --> K[Signed full generation] + J --> L[Signed delta and revocation feed] + K --> M[Verified local library and CLI] + L --> M + M --> N[Exact lookup and regional resolve] +``` + +The writer store remains append-oriented and audit-capable. Runtime generations +contain the selected facts, active decisions, required proof material, and signed +references to cold audit bundles. They do not duplicate every historical raw row. + +## Release sequence + +| Release | Purpose | Exit condition | +| --- | --- | --- | +| v0.3.x | Correct active-state and review-time semantics | Historical export is explicit, scope overlap fails closed, review acceptance time is trusted, and legacy behavior is regression-tested. | +| v0.4 | Scalable trust and observation pipeline | Quorum policy, granular name/edge decisions, sticky revocations, structured evidence bundles, and review queues work end to end. | +| v0.5 | Provider-scale and source expansion | Real-format scale canaries pass; incremental Wikidata and the first new rights-approved adapters are reproducible and bounded. | +| v0.6 | Signed public reference registry | A reviewed dataset, full/delta releases, revocation delivery, coverage report, and consumer verification are published independently of source releases. | +| Later | Wider resolver/product surface | Locale fallback, typed destination roles, candidate discovery, platform portability, and an optional read-only service are justified by real use. | + +## Phase 0: Freeze contracts and record design decisions + +### Task 0.1: Capture the v0.3 compatibility baseline + +- [x] Record the exact v0.3 CLI output schemas, generation schema, source-manifest + schema, rules version, review JSON, release receipts, and Python/Rust examples. +- [x] Add golden fixtures for a complete v0.3 generation and a mutable schema-v3 + writer database without committing provider data or private keys. +- [x] Verify that current `lookup`, `resolve`, `diff`, `export`, `stats`, signature + verification, rollback refusal, and migrations behave exactly as documented. +- [x] Preserve the current five-source all-synthetic fixture as a compatibility gate. + +Likely files: + +- `crates/argand-site-registry/tests/common/` +- `crates/argand-site-registry/tests/registry.rs` +- `crates/argand-site-registry/tests/failures.rs` +- `crates/argand-site-registry/tests/cli.rs` +- `docs/VALIDATION.md` + +Acceptance: + +- A v0.3 reader fixture remains readable or fails with a precise documented version + error after each later schema change. +- No later test can silently regenerate the baseline fixture from new behavior. + +### Task 0.2: Write architecture decisions before migrations + +- [x] Add an ADR for active source coverage and supersession. +- [x] Add an ADR for separating name bindings, website edges, and observations. +- [x] Add an ADR for reviewer votes, revocation precedence, and publisher policy. +- [x] Add an ADR for runtime projection versus cold audit retention. +- [x] Add an ADR for full and delta dataset release identities. +- [x] Add an ADR for source lineage so copied upstream evidence does not count twice. +- [x] Document whether broad proprietary embedding is a goal. If so, evaluate a + small permissively licensed format/verifier crate without changing the AGPL + publisher engine or source-driven dataset licenses automatically. + +Acceptance: + +- Every later migration and public contract links to an approved decision. +- The ADRs explain rejected alternatives and compatibility consequences. + +## Phase 1: v0.3.x active-state and trust hardening + +### Task 1.1: Replace free-form snapshot replacement semantics with typed coverage + +Problem: v0.3 selects the latest completed snapshot per exact `(source, scope)` +string. Different strings are treated as additive even when one is a full snapshot +or overlaps another partition. + +- [x] Introduce a source-manifest version with explicit coverage semantics: + collection identity, coverage kind, stable partition identity, base snapshot + where applicable, and explicit supersession references. +- [x] Preserve v1 manifest parsing. Map every legacy `(source, scope)` to an isolated + legacy partition so migration does not silently change its active facts. +- [x] Require operators to provide an explicit mapping before a legacy partial + snapshot can join or be replaced by a typed full collection. +- [x] Select one coherent coverage set per source collection: + - a complete full snapshot supersedes older declared partitions; + - disjoint partitions compose only when their identities are explicit; + - deltas require an authenticated base and ordered continuity; + - overlapping or missing coverage relationships abort the build; + - deletions use explicit tombstones and remain auditable. +- [x] Include the complete selected-coverage graph in the build receipt. +- [x] Make `stats` and `diff` report selected, superseded, incomplete, and conflicting + source snapshots separately. + +Likely files: + +- `crates/argand-site-registry/src/model.rs` +- `crates/argand-site-registry/src/store.rs` +- `crates/argand-site-registry/src/build.rs` +- `crates/argand-site-registry/src/diff.rs` +- `crates/argand-site-registry/src/query.rs` +- `crates/argand-site-registry/src/cli.rs` +- `crates/argand-site-registry/migrations/004.sql` + +Tests: + +- Latest snapshot replaces an older snapshot in the same partition. +- A declared full snapshot supersedes earlier partitions. +- Two explicit disjoint partitions compose. +- An undeclared full-plus-partial combination fails. +- Overlapping partitions fail. +- Missing delta bases, skipped deltas, and forked delta histories fail. +- Tombstoned facts disappear from the active projection but remain in audit history. +- Failed or partial imports never replace selected complete coverage. +- Reimporting identical manifests and deltas is byte-for-byte idempotent. + +### Task 1.2: Split active export from audit-history export + +Problem: v0.3 JSONL export emits facts from every complete retained source snapshot, +including superseded snapshots, without a per-row active-state marker. + +- [x] Change the normal export contract to emit selected active facts only. +- [x] Version the export envelope and include generation identity, selected source + IDs, coverage-policy version, derivation version, and attribution identity. +- [x] Add explicit selection state to every assertion. +- [x] Add a separate audit export that includes active, superseded, rejected, and + tombstoned facts with replacement links. +- [x] Keep Curlie descriptions redacted by default in both modes. +- [x] Make consumers reject unknown export schema versions. +- [x] Document that neither export bypasses `resolve` admission policy. + +Likely files: + +- `crates/argand-site-registry/src/release.rs` +- `crates/argand-site-registry/src/cli.rs` +- `crates/argand-site-registry/src/diff.rs` +- `README.md` +- `docs/CONSUMERS.md` +- `docs/TRUST.md` + +Tests: + +- Superseded facts are absent from active export. +- Audit export retains and labels the same facts. +- Active export and native generation agree exactly on selected source IDs. +- Description redaction and attribution survive both modes. +- Python and Rust consumers refuse unknown versions and preserve null abstention. + +### Task 1.3: Bind decisions to trusted acceptance time + +Problem: the reviewer signs `reviewed_at`, but the writer does not persist a trusted +append time. That self-declared time influences approval validity and reviewer-key +validity-epoch checks. + +- [x] Record `accepted_at` from the writer when exact signature verification and + candidate validation succeed. +- [x] Make approval validity begin no earlier than trusted acceptance. +- [x] Bound effective expiry by both the signed review duration and the enforced + maximum measured from acceptance, preventing backdating or future dating from + extending authority. +- [x] Evaluate reviewer eligibility at acceptance for new decisions. Preserve the + signed claimed decision time for audit, without treating it as trusted time. +- [x] Include acceptance time and the time-policy version in generation receipts. +- [x] Retain legacy approvals for audit but require an explicit migration policy or + fresh decision before they are active under the new rules. Never synthesize a + historical acceptance time. Preserve legacy revocations regardless. +- [x] Test clock skew, future dates, backdates, expired keys, removed keys, offline + signing followed by later acceptance, and reproducible builds at fixed clocks. + +Likely files: + +- `crates/argand-site-registry/src/review.rs` +- `crates/argand-site-registry/src/query.rs` +- `crates/argand-site-registry/src/resolution.rs` +- `crates/argand-site-registry/src/release.rs` +- `crates/argand-site-registry/migrations/004.sql` +- `docs/TRUST.md` +- `docs/PUBLISHING.md` + +### Task 1.4: Make revocation precedence explicit before quorum work + +- [x] Treat a valid revocation as sticky for its exact subject. +- [x] Require an explicit signed supersession that references the revocation before + the same destination can become active again. +- [x] Prevent a later ordinary approval from overriding a revocation by sequence + order alone. +- [x] Preserve existing activation rollback checks and strengthen them to compare + revocation identities and supersession relationships. +- [x] Add a machine-readable emergency revocation export suitable for cached + consumers. + +Acceptance: + +- Reordering, forking, or appending an ordinary approval cannot erase a revocation. +- A deliberately superseded revocation remains visible in lookup, diff, and audit. +- An offline consumer can apply the revocation export before receiving a full build. + +### Phase 1 release gate + +- [x] Run `cargo fetch --locked` once, then the complete offline `scripts/check.sh`. +- [x] Run migrations from empty, schema v1, v2, and v3 databases. +- [x] Build the same generation twice and compare every output byte. +- [x] Verify source release determinism from a clean signed commit. +- [x] Update changelog, trust docs, format contracts, and migration guidance. +- [x] Run `cargo audit --deny warnings` and record it separately from the offline gate. +- [x] Confirm no provider data, keys, approval logs, or private paths entered Git. + +## Phase 2: v0.4 scalable review and structured evidence + +### Task 2.1: Separate name-to-entity trust from entity-to-property trust + +Problem: v0.3 binds each website approval to the entity's full name set. This stops +a newly injected alias from inheriting a route, but benign alias or label changes +invalidate every website edge. A global Wikidata revision also changes edge identity +when the P856 statement itself did not materially change. + +- [x] Create independent material fingerprints for: + - source name or alias assertion to stable entity; + - explicit entity equivalence; + - entity-to-normalized-web-property assertion; + - normalized regional role and scope; + - observation bundle; + - reviewer vote and policy result. +- [x] Bind website-edge identity only to material website evidence: exact source + statement identity/value/rank/relevant qualifiers, normalized URL result, and + derivation rule version. +- [x] Retain source record revision and unrelated entity metadata in provenance and + diffs without making them part of website-edge identity. +- [x] Require `resolve` to satisfy both an admitted name binding and an admitted + website edge. `lookup` continues to expose unreviewed names and edges. +- [x] Ensure a newly imported alias cannot inherit an existing approved route. +- [x] Ensure a benign accepted alias update does not invalidate an unchanged route. +- [x] Give canonical labels, aliases, and source-specific names distinct evidence + identities and policy treatment. + +Tests: + +- Adding an unreviewed alias never creates a resolvable query. +- Removing or changing an approved alias affects only that binding. +- An unrelated Wikidata `lastrevid` change preserves the route fingerprint. +- A P856 URL, rank, end qualifier, or relevant regional qualifier change invalidates + the route approval. +- Confusable and bidi-control names remain rejected. +- Explicit entity equivalence never carries unstated name or route authority. + +### Task 2.2: Replace latest-decision-wins with signed votes and policy compilation + +- [x] Represent decisions as immutable signed votes by authenticated reviewer. +- [x] Permit one current vote per signer, subject, decision type, and policy epoch; + preserve superseded votes in the append-only log. +- [x] Add a versioned publisher policy that defines: + - required approval threshold by decision type and risk class; + - revocation threshold and sticky-revocation behavior; + - publisher/reviewer separation; + - reviewer groups or independence constraints where configured; + - maximum approval age and evidence-freshness requirements; + - treatment of source conflicts and unresolved observations. +- [x] Hash the exact policy and selected reviewer trust roots into the release receipt. +- [x] Compile votes deterministically into approved, revoked, expired, disputed, + probationary, or insufficient-review state. +- [x] Keep policy configurable for independent publishers while shipping a strict, + documented reference policy for Argand's own releases. +- [x] Preserve the ability to operate locally without accounts or a network service. + +Tests: + +- One signer cannot satisfy a two-independent-reviewer policy. +- Duplicate keys or aliases for the same reviewer do not create extra votes. +- Publisher/reviewer separation is enforced when enabled. +- One authorized revocation blocks resolution under the reference policy. +- Conflicting votes produce a disputed state and abstention. +- Changing policy invalidates the old compiled result without changing source facts. +- Release verification fails on policy, signer, vote, or receipt tampering. + +### Task 2.3: Turn observation types into a stored evidence pipeline + +- [x] Extend the writer schema with immutable observation batches and observations. +- [x] Preserve the existing distinction between observations and ownership claims. +- [x] Add an adapter contract for externally captured observations, including source, + rights declaration, capture ID, retrieval time, content hash, exact selector, + from/to URLs, relation, and confidence. +- [x] Project redirect, canonical, hreflang, JSON-LD `sameAs`, sitemap, and country + selector evidence without automatically creating an entity edge. +- [x] Add observation lookup, reverse lookup, generation diff, and evidence-bundle + output for review. +- [x] Record negative or failed observations with bounded error classes so absence is + not confused with a fetch that never succeeded. +- [x] Define retention that stores hashes and necessary bounded extracts rather than + copied page bodies unless rights and need are established. + +Likely files: + +- `crates/argand-site-registry/src/observation.rs` +- `crates/argand-site-registry/src/store.rs` +- `crates/argand-site-registry/src/build.rs` +- `crates/argand-site-registry/src/query.rs` +- `crates/argand-site-registry/src/diff.rs` +- `crates/argand-site-registry/migrations/005.sql` + +### Task 2.4: Add a bounded candidate-site observer + +- [x] Observe only imported/reviewed candidate URLs. Do not start an open-web crawler. +- [x] Enforce scheme, redirect-count, response-size, header-size, decompression, + timeout, DNS-result, address-range, and per-host request bounds. +- [x] Block credentials, local/private/link-local targets, unsafe redirect transitions, + non-HTTP protocols, and host confusion. +- [x] Capture redirect chains, final URL, status, canonical, hreflang, sameAs, sitemap + references, country selectors, DNS answers, and TLS certificate fingerprints as + separate evidence classes. +- [x] Do not treat TLS, DNS, redirects, or site self-assertions as ownership proof. +- [x] Emit immutable observation manifests compatible with Task 2.3. +- [x] Support cache-only replay so parser and policy tests never require the network. +- [x] Keep acquisition cadence, concurrency, bandwidth, and data path configurable. + +Security tests: + +- SSRF attempts, redirect loops, compression bombs, oversized markup, malformed HTML, + DNS rebinding, mixed encodings, invalid certificates, and cross-scheme redirects. +- Parser fuzz/property tests for headers, hreflang, canonical, JSON-LD, and sitemaps. +- Deterministic replay from captured fixtures with no network access. + +### Task 2.5: Build review queues and evidence bundles before a graphical UI + +- [x] Add deterministic queue output ordered by risk and material change, not source + popularity alone. +- [x] Queue new routes, new aliases, source conflicts, changed website statements, + redirects across registrable domains, observation drift, expiring approvals, + unresolved regional scopes, and revoked destinations proposed for reinstatement. +- [x] Produce a bounded review bundle containing exact claims, conflicts, observation + diffs, source licenses, capture hashes, requested role, and candidate fingerprint. +- [x] Add commands to prepare a vote, verify exact vote bytes, append it, and show the + compiled policy result. +- [x] Keep the queue read-only and deterministic. Never let viewing evidence mutate + approval state. +- [x] Defer a browser workbench until CLI bundles have proven the workflow. + +### Task 2.6: Monitor approved routes and classify drift + +- [x] Schedule observation refresh independently from source import cadence. +- [x] Classify material changes: unreachable, cross-domain redirect, DNS/TLS change, + content/canonical shift, domain expiry indicators, malware-policy result, or no + material change. +- [x] Use risk policy to shorten review intervals. Never auto-extend approvals. +- [x] Put materially changed routes into probation or revoke them according to signed + publisher policy; default to abstention where evidence is insufficient. +- [x] Record every transition and make revocation candidates immediately exportable. +- [x] Design malware-feed adapters only after exact commercial-reuse and redistribution + terms are verified. Do not hard-code an unreviewed vendor. + +### Phase 2 release gate + +- [x] Complete a signed two-reviewer fixture from source assertion through name vote, + edge vote, observation bundle, policy compilation, release, resolution, drift, + revocation, signed delta, and consumer application. +- [x] Prove a malicious contributor cannot submit directly into active state. +- [x] Prove one compromised reviewer cannot approve under the reference policy. +- [x] Prove an emergency revocation reaches a pinned offline consumer without a full + source reimport. +- [x] Preserve all v0.3 conflict, normalization, regional, and rollback tests. + +Completion note: the version 0.4 implementation satisfies the Phase 0 through 2 +acceptance boundary, including the complete two-reviewer fixture and security +review. The immutable v0.3 contract is frozen by exact signed commit and golden +schema/field coordinates rather than a committed SQLite generation, because this +repository does not admit generated datasets or approval logs. General registry +deltas remain explicitly deferred by ADR 0005; version 0.4 supplies typed source +deltas and a separately signed, cumulative emergency block feed. + +## Phase 3: v0.5 storage, scale, and current-source improvements + +### Task 3.1: Separate runtime projections from cold audit history + +- [ ] Keep raw cache objects immutable and content-addressed outside Git. +- [ ] Package completed source imports into content-addressed audit bundles with + manifest, record/fact indices, hashes, format version, and attribution. +- [ ] Make a runtime generation contain selected facts, normalized projections, + active policy results, required votes/revocations, and signed bundle references. +- [ ] Do not copy all historical records and facts into every runtime generation. +- [ ] Add audit verification that streams referenced bundles and detects absence, + truncation, substitution, or mismatched attribution. +- [ ] Add retention/checkpoint tooling that never deletes the only authenticated copy + of evidence and produces a signed deletion/retention report. +- [ ] Measure query latency and generation size before and after the split. + +### Task 3.2: Add provider-scale benchmark and recovery tooling + +- [ ] Define repeatable small, medium, and provider-representative import profiles. +- [ ] Record wall time, CPU time, peak RSS, compressed and expanded bytes, database + growth, facts/second, checkpoint frequency, restart time, build size, and query + latency. +- [ ] Interrupt imports at multiple checkpoints and prove idempotent resumption. +- [ ] Exercise disk-full, truncated input, cache corruption, duplicate records, and + interrupted generation publication. +- [ ] Make benchmark reports name exact source snapshot hashes and hardware without + committing source data. +- [ ] Treat synthetic performance as development evidence, not provider capacity. + +### Task 3.3: Harden full Wikidata ingestion and add incremental refresh + +- [ ] Confirm current official full and incremental formats from Wikidata documentation + and inspected fixtures before changing the adapter. +- [ ] Replace the assumption that every useful entity fits in one in-memory 16 MiB + line with bounded disk-spooling or an explicitly receipted oversized-record path. +- [ ] Never silently skip an oversized entity that may contain a relevant fact. +- [ ] Add incremental add/change ingestion with authenticated base snapshot identity, + ordered application, checkpoints, and reconciliation against later full dumps. +- [ ] Define how deletions and removed P856 statements become tombstones. +- [ ] Keep full raw assertion/qualifier/reference provenance for consumed fields. +- [ ] Make unrelated `lastrevid` changes visible in audit diffs without invalidating + unchanged material edge fingerprints. +- [ ] Test real-format pathological entities and multistream compression boundaries. + +Authoritative format reference: + +- + +### Task 3.4: Strengthen current-source acquisition verification + +- [ ] Prefer provider-published checksums or signatures when officially available and + bind verification method into the source manifest. +- [ ] Keep HTTPS allowlists, manual redirect validation, byte bounds, strong-validator + resume rules, and immutable local cache behavior. +- [ ] Detect and report source format drift before partial import can replace a source. +- [ ] Add format-version canaries for Majestic, CrUX, Curlie, PSL, and Wikidata. +- [ ] Preserve CrUX billing as explicit opt-in configuration and record job identity, + query, result period, and actual cost outside public fixtures. +- [ ] Continue frequent PSL refresh and include exact PSL hash in normalization proofs. +- [ ] Preserve Curlie attribution and description-redaction tests on every export path. + +### Task 3.5: Add source lineage and independence metadata + +- [ ] Record direct provider, upstream/origin dataset, transformation, snapshot, and + known dependency relationships for each fact source. +- [ ] Prevent policy from counting two assertions as independent corroboration when + one republishes the other. +- [ ] Expose lineage in lookup, review bundles, export, diff, and evaluation. +- [ ] Keep unknown lineage explicit rather than assuming independence. + +## Phase 4: Rights-gated additional source adapters + +Every source follows the same gate: + +1. Verify authoritative download, schema, update cadence, license, attribution, + redistribution, database-right, and commercial-use documentation. +2. Record the exact decision and URLs in `LICENSE_SOURCES.md`. +3. Inspect current official fixtures. Do not infer fields from third-party examples. +4. Add a source enum/format only with a streaming adapter and bounded failure tests. +5. Preserve native IDs, raw relevant records, selectors, lineage, and confidence. +6. Import into a separate logical source layer. +7. Demonstrate that the source cannot auto-create an approved route. +8. Run deterministic, idempotent, interrupted, malformed, and conflict fixtures. + +### Task 4.1: Add ROR first + +Rationale: ROR is CC0 and directly supplies stable organization IDs, names, aliases, +status, locations, links, and domains. It is compact and well aligned with the model. + +- [ ] Verify the current ROR schema version and official release asset from the ROR + data-dump documentation at implementation time. +- [ ] Consume only fields confirmed in that inspected schema. +- [ ] Map names and aliases without merging ROR entities into Wikidata entities unless + an exact external identifier or reviewed equivalence supports the join. +- [ ] Preserve links and domains as ROR assertions, not approvals. +- [ ] Preserve status, type, country/location, external IDs, and upstream lineage. +- [ ] Test domain conflicts, former/inactive organizations, aliases, multiple links, + missing fields, duplicate input, schema drift, and exact-ID equivalence. + +References: + +- +- +- + +### Task 4.2: Add MusicBrainz core snapshots second + +- [ ] Use only the CC0 core database snapshot and its verified checksums/signatures. +- [ ] Do not use the CC BY-NC-SA live replication feed in the commercial-safe default + pipeline. +- [ ] Consume exact URL entities and documented URL relationship types, including + official-homepage and ended-state metadata. +- [ ] Keep artists, labels, places, and events separate by stable MusicBrainz ID. +- [ ] Preserve relationship begin/end dates and link types as material evidence. +- [ ] Treat community-curated URLs as assertions requiring normal Argand review. +- [ ] Test removed URLs, ended relationships, entity redirects/merges, duplicate URLs, + malicious/taken-over sites, and snapshot replacement. + +References: + +- +- +- + +### Task 4.3: Validate GND as the third adapter + +- [ ] Verify the exact current CC0 declaration for the selected GND files. +- [ ] Inspect current JSON-LD or RDF schema and confirm homepage predicates before + implementing an adapter. +- [ ] Preserve authority IDs, preferred/variant names, types, countries, external IDs, + and exact homepage assertions where present. +- [ ] Keep its regional and language focus visible in provenance and evaluation. +- [ ] Stop after the rights/schema spike if homepage coverage does not justify the + adapter cost. + +Reference: + +- + +### Task 4.4: Keep lower-priority candidates behind explicit holds + +- [ ] ORCID: research a low-confidence individuals-only adapter. Its public file is + CC0, but links are self-declared and require privacy, impersonation, and + volatility policy. Never auto-approve. Reference: + . +- [ ] OpenAlex: use only for research-activity/popularity metadata if useful. Record + ROR as upstream lineage and never count its institution website as independent + corroboration. Reference: . +- [ ] OpenStreetMap: do not ingest until an ODbL-compatible distribution and + attribution architecture is approved. Reference: + . +- [ ] Government/corporate registries: assess jurisdiction by jurisdiction. Prefer + stable identity crosswalks; do not infer a website where no authoritative field + exists. +- [ ] DNS, RDAP, certificate transparency, package registries, and web crawl data: + evaluate as observation sources only after exact terms are verified. +- [ ] Open Library and other sources with unresolved underlying rights remain excluded. + +### Phase 4 acceptance + +- Each admitted source has authoritative license evidence, current fixture evidence, + exact fields consumed, attribution behavior, source lineage, and format-drift tests. +- Full source-specific provenance appears in lookup, review bundles, export, and diff. +- Removing any new adapter leaves existing source identities and release verification + deterministic. +- No new source changes an existing edge's approval merely by corroborating it. + +## Phase 5: Resolver, evaluation, and consumer improvements + +### Task 5.1: Canonicalize locale and country semantics + +- [ ] Add standards-based BCP 47 parsing/canonicalization after reviewing the chosen + library and current specification behavior. +- [ ] Define deterministic precedence for exact locale/country, country-only, + language-parent, and global-primary candidates. +- [ ] Abstain on equal candidates at the same specificity. +- [ ] Preserve the requested and normalized locale in the explanation envelope. +- [ ] Add tests for `en-GB`, `en`, script subtags, case, deprecated aliases, malformed + tags, country-only properties, multi-country sites, and conflicting scopes. + +### Task 5.2: Extend property roles without weakening default navigation + +- [ ] Define a versioned role vocabulary covering at least global primary, regional + primary, product, support, developer, careers, login, status, and other reviewed + roles justified by real cases. +- [ ] Keep default `resolve` restricted to the requested navigation role. +- [ ] Require role-specific evidence and votes; a support site cannot become primary + because it shares a domain. +- [ ] Preserve unknown source roles as evidence without admitting them. + +### Task 5.3: Add candidate discovery separately from resolution + +- [ ] Add prefix/fuzzy discovery only as an audit/candidate operation. +- [ ] Preserve exact normalized matching for final admission. +- [ ] Return candidate score components and ambiguity rather than hiding a rewrite. +- [ ] Never let similarity bypass reviewed name-to-entity bindings. +- [ ] Test homographs, typosquatting, short names, multilingual aliases, and popular + entities with colliding names. + +### Task 5.4: Expand evaluation into a release gate + +- [ ] Extend judgments to cover expected abstention reasons, selected name binding, + route edge, regional precedence, and policy state. +- [ ] Report resolved-route errors separately from safe abstentions. +- [ ] Measure coverage, abstention taxonomy, active-evidence age, expiring approvals, + conflict rate, reviewer agreement, review turnaround, observation drift, and + revocation propagation. +- [ ] Add adversarial suites for source poisoning, alias injection, correlated sources, + malicious redirects, domain takeover, compromised reviewer, compromised + publisher, stale cache, rollback, and policy downgrade. +- [ ] Segment evaluation by source, entity type, language, country, popularity band, + destination role, and evidence age without collapsing source signals. +- [ ] Establish release thresholds only after an audited baseline exists. Any known + wrong resolved destination is a release blocker for the reference dataset. + +## Phase 6: v0.6 signed public reference dataset + +### Task 6.1: Define reference publisher governance + +- [ ] Publish reviewer eligibility, independence, conflict-of-interest, evidence, + expiry, appeals, correction, key rotation, incident, and emergency-revocation + policies. +- [ ] Separate source maintainers, reviewers, and release publishers where practical. +- [ ] Publish the exact policy hash and reviewer trust roots with each release. +- [ ] Define a transparent proposal process in which contributors submit signed + evidence bundles rather than direct active-dataset edits. +- [ ] Record disputed claims and abstain until policy is met. +- [ ] Publish change logs and correction history without exposing sensitive reviewer + material unnecessarily. + +### Task 6.2: Build a deliberately bounded starter registry + +- [ ] Select a high-value initial coverage set using source-separated popularity and + declared entity classes only for prioritization. +- [ ] Publish the selection methodology and its biases. +- [ ] Review name bindings, entity equivalences, website edges, regional roles, and + current observations under the reference policy. +- [ ] Do not claim comprehensive web, country, language, or entity-type coverage. +- [ ] Require every resolvable destination to have current votes, unexpired evidence, + complete provenance, and an active monitoring schedule. +- [ ] Run the full evaluation, diff, license, source-lineage, and release gates. + +### Task 6.3: Publish full releases, deltas, and revocations + +- [ ] Keep dataset releases separate from source-code releases and raw provider cache. +- [ ] Produce a deterministic full generation, receipt, attribution bundle, policy, + reviewer trust roots, evaluation report, coverage report, and detached publisher + signature. +- [ ] Produce ordered, signed deltas bound to exact base and target generation IDs. +- [ ] Publish a small signed revocation feed with monotonic continuity and rollback + protection. +- [ ] Make full and delta application atomic and recoverable after interruption. +- [ ] Provide a documented mirror-independent verification procedure. +- [ ] Never make an unauthenticated `latest` URL the trust root. A convenience current + pointer must itself be signed and rollback protected. + +### Task 6.4: Make immediate use simple + +- [ ] Add a beginner workflow that downloads a release, verifies its publisher and + policy, pins it, and resolves `facebook` locally. +- [ ] Show an entity with multiple reviewed regional properties and an abstention. +- [ ] Provide identical native CLI, Rust library, and Python subprocess examples. +- [ ] Add a machine-readable capability/version command. +- [ ] Explain code license separately from each dataset source license and attribution. +- [ ] Provide update and emergency-revocation examples for systemd and cron. + +### Reference release acceptance + +- Every resolved route is reproducible from selected source facts, name/edge votes, + policy, and observations contained in or authenticated by the release. +- Every conflict, rejection, abstention, and supersession remains inspectable. +- An untrusted mirror cannot substitute a generation, policy, reviewer set, delta, or + revocation feed without verification failure. +- A clean consumer machine can verify and query the release without provider + credentials, reviewer keys, publisher private keys, or the mutable writer database. +- The coverage and evaluation reports distinguish implemented capability from actual + reviewed data coverage. + +## Phase 7: Optional service and portability + +### Task 7.1: Cross-platform immutable generation reads + +- [ ] Implement equivalent no-follow, regular-file, bounded-copy, hash-before-open, + and private-temporary-file behavior for macOS and Windows. +- [ ] Add platform CI only on isolated runners with no release authority. +- [ ] Preserve Linux behavior and reject platforms without a safe implementation. + +### Task 7.2: Read-only service only after the dataset proves useful + +- [ ] Expose the existing verified query envelope through a minimal read-only local + HTTP or Unix-socket service if consumer demand justifies it. +- [ ] Pin one generation per process/request context and swap only after full signature, + policy, reviewer, and rollback verification. +- [ ] Bound query size, output size, concurrency, and time. +- [ ] Preserve typed abstention and explanation; do not add silent query rewriting. +- [ ] Keep publication, review, acquisition, and private keys out of the serving process. + +## Operational security and supply-chain work + +- [ ] Add property tests or fuzz targets for URL/domain normalization, PSL behavior, + JSON duplicate keys, each parser, coverage selection, observation parsing, + signed-delta application, and locale handling. +- [ ] Add dependency vulnerability and license-policy checks to a network-enabled, + isolated scheduled workflow. Preserve the offline acceptance gate separately. +- [ ] Pin and document CI images and external actions by immutable identity. +- [ ] Verify official provider checksums/signatures where available. +- [ ] Add threat-model cases for reviewer-key theft, publisher-key theft, source + compromise, malicious proposal bundles, cache substitution, stale mirrors, + rollback, denial of service, and domain takeover. +- [ ] Add key rotation and emergency response drills using disposable test keys. +- [ ] Keep signed receipts and validation artifacts outside the source tree unless they + are deliberately public, non-sensitive release evidence. + +## Documentation deliverables + +- [ ] Keep the README focused on the first successful verified lookup. +- [ ] Add an architecture document explaining assertion, observation, decision, policy, + generation, delta, and consumer boundaries. +- [ ] Expand `LICENSE_SOURCES.md` for every admitted source with exact fields consumed, + source URLs, licenses, attribution, redistribution, update cadence, and lineage. +- [ ] Document active versus audit export semantics. +- [ ] Document migrations and legacy decision handling. +- [ ] Publish the reference review and incident policies. +- [ ] Publish provider-scale measurements without implying serving or corpus coverage. +- [ ] Maintain a source-candidate table showing approved, research, held, and rejected + sources with the reason for each state. + +## SWOT-driven checks + +### Preserve strengths + +- Deterministic, source-separated facts and normalization. +- Fail-closed resolution and explicit abstention. +- Signed reviews/releases and rollback-safe revocations. +- Complete licenses, attribution, provenance, and conflict evidence. +- Streaming, bounded, resumable, idempotent imports. +- Meaningful adversarial and cross-consumer tests. + +### Correct weaknesses + +- Publish an immediately usable dataset after trust and scale gates pass. +- Replace one-latest-decision semantics with policy-compiled votes. +- Reduce benign review churn without allowing alias inheritance. +- Separate active runtime state from unbounded audit history. +- Add provider-scale proof, monitoring, queues, and richer metrics. +- Improve regional semantics and platform support after the core release. + +### Capture opportunities + +- Establish an open evidence-bundle and signed-generation interchange format. +- Let independent publishers share facts while choosing different trust policies. +- Supply explainable navigation authority to search engines, assistants, browsers, + bookmarks, enterprise catalogs, and safety products. +- Publish an evaluation benchmark for entity-to-site resolution and abstention. +- Use ROR, MusicBrainz, and GND to expand high-quality vertical coverage. + +### Mitigate threats + +- Compromised reviewers: quorum, independence, expiry, and sticky revocation. +- Compromised publishers: independent reviewer roots and consumer policy verification. +- Domain takeover: active observation, probation, and rapid revocation delivery. +- Source poisoning: no auto-approval, source lineage, evidence diffs, and review queues. +- Correlated evidence: upstream lineage and independence-aware policy. +- License drift: exact rights gates and fail-closed format/license changes. +- Cache/mirror staleness: signed continuity, expiry, and rollback protection. +- Governance capture: public policies, disputes, appeals, and transparent changes. + +## Explicit deferrals + +Do not prioritize these before a reviewed reference dataset exists: + +- A general hosted API or multi-tenant account system. +- ML or LLM approval of ownership or regional roles. +- Open-web crawling. +- Fuzzy matching in the final resolver. +- Additional popularity feeds merely to enlarge source count. +- Sources with unclear commercial reuse, database rights, or redistribution terms. +- ODbL data without an approved packaging architecture. +- A graphical review UI before deterministic CLI bundles and queues are proven. +- Automatic approval renewal based on unchanged popularity, TLS, DNS, or redirects. + +## Definition of complete + +The roadmap is complete when: + +1. Source coverage and exports cannot confuse historical facts with active facts. +2. Review authority is bound to trusted acceptance time and a signed policy. +3. Name bindings and website edges change independently without creating inherited + authority. +4. Multiple authenticated reviewers and sticky revocations protect the reference + policy from one compromised contributor or reviewer. +5. Candidate-site observations are reproducible, bounded, rights-declared evidence + and material drift reaches reviewers promptly. +6. Runtime generations remain compact while complete audit evidence stays verifiable. +7. Current real source formats import and resume within documented modest-hardware + bounds, with no silent oversized-record loss. +8. Every new source has verified commercial-reuse rights, exact consumed fields, + lineage, attribution, and adversarial tests. +9. Regional resolution uses deterministic locale/country precedence and abstains on + unresolved ambiguity. +10. A separately distributed signed reference dataset produces useful reviewed + results, including `facebook -> Facebook -> facebook.com` and multiple regional + properties, from a clean consumer machine. +11. Full releases, deltas, and revocations authenticate independently of mirrors and + refuse rollback. +12. Published evaluation and coverage reports state what is actually reviewed, + current, relevant, and resolvable without conflating implementation with data. +13. Formatter, compiler checks, strict lints, documentation, unit/integration tests, + adversarial tests, source-release determinism, provider-scale canaries, and a + clean final diff all pass for every release boundary. + +## First implementation slice after approval + +Begin only with Phase 0 and Phase 1. Do not combine the initial migration with the +crawler, new sources, public data acquisition, or dataset publication. + +The first concrete change set should contain: + +1. Frozen v0.3 compatibility fixtures. +2. Approved active-coverage and export ADRs. +3. Typed source coverage with conservative legacy behavior. +4. Active-only export plus explicit audit export. +5. Trusted review acceptance time and sticky revocation semantics. +6. Schema/rules migration, documentation, and complete offline acceptance evidence. + +Review that diff and release it before beginning granular name/edge votes or crawler +observations. This keeps the highest-risk semantic corrections small enough to audit +and makes every later phase build on an unambiguous active registry. diff --git a/scripts/source_release.py b/scripts/source_release.py index c21a7e4..d7fb0e3 100644 --- a/scripts/source_release.py +++ b/scripts/source_release.py @@ -57,8 +57,11 @@ def safe_path(name): raise ValueError("unsafe archive path") allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml", ".service", ".timer"} - if path.suffix not in allowed and path.name not in (".gitignore", "Dockerfile") and name not in ( - "LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore"): + if path.suffix not in allowed and path.name not in ( + ".gitignore", "Dockerfile") and name not in ( + "LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore", + "crates/argand-site-registry/examples/observer.env", + "crates/argand-site-registry/tests/fixtures/v03-contract.json"): raise ValueError(f"file is outside the source release allowlist: {name}") diff --git a/tests/test_source_release.py b/tests/test_source_release.py index 9ca9e3b..d315588 100644 --- a/tests/test_source_release.py +++ b/tests/test_source_release.py @@ -45,6 +45,8 @@ class SourceReleaseTests(unittest.TestCase): "LICENSE": "Synthetic code license fixture\n", "UPSTREAM.json": "{}\n", "crates/argand-site-registry/LICENSE_SOURCES.md": "Synthetic source terms\n", + "crates/argand-site-registry/examples/observer.env": "ARGAND_REGISTRY_PIN=fixture\n", + "crates/argand-site-registry/tests/fixtures/v03-contract.json": "{}\n", "src/lib.rs": "// Synthetic Rust source\n", "src/.gitignore": "*.temporary\n", } @@ -129,8 +131,12 @@ class SourceReleaseTests(unittest.TestCase): release.verify(self.output, release.digest(encoded)) def test_special_source_paths_refused(self): + release.safe_path("crates/argand-site-registry/examples/observer.env") + release.safe_path("crates/argand-site-registry/tests/fixtures/v03-contract.json") for path in ("/tmp/a.rs", "a/../b.rs", "a//b.rs", "a\\b.rs", "data/a.md", - ".git/config.toml", "secret.key", "x\na.rs"): + ".git/config.toml", "secret.key", "secrets.env", + "config/observer.env", "other.json", "fixtures/provider.json", + "x\na.rs"): with self.subTest(path=path), self.assertRaises(ValueError): release.safe_path(path)