From e83f43d00f38fb1a8973316fc045ac4139069aaa Mon Sep 17 00:00:00 2001 From: nicweyand Date: Sun, 13 Sep 2026 01:19:27 -0400 Subject: [PATCH] feat: harden reviewed registry releases --- CHANGELOG.md | 23 + Cargo.lock | 9 +- Cargo.toml | 3 +- GOVERNANCE.md | 4 +- README.md | 14 +- ci/Dockerfile | 13 + ci/README.md | 19 + crates/argand-site-registry/Cargo.toml | 3 +- crates/argand-site-registry/README.md | 53 ++- .../argand-site-registry/migrations/002.sql | 14 + crates/argand-site-registry/src/build.rs | 11 +- crates/argand-site-registry/src/catalog.rs | 427 ++++++++++++++++++ crates/argand-site-registry/src/cli.rs | 197 +++++++- crates/argand-site-registry/src/diff.rs | 174 +++++++ crates/argand-site-registry/src/evaluation.rs | 224 +++++++++ crates/argand-site-registry/src/generation.rs | 168 +++++++ crates/argand-site-registry/src/identity.rs | 73 ++- crates/argand-site-registry/src/lib.rs | 8 + .../argand-site-registry/src/observation.rs | 127 ++++++ crates/argand-site-registry/src/query.rs | 132 +----- crates/argand-site-registry/src/release.rs | 62 +-- crates/argand-site-registry/src/resolution.rs | 238 ++++++++++ crates/argand-site-registry/src/review.rs | 205 ++++++++- crates/argand-site-registry/src/ssh.rs | 64 +++ crates/argand-site-registry/src/store.rs | 16 +- crates/argand-site-registry/tests/cli.rs | 298 ++++++++++-- .../argand-site-registry/tests/evaluation.rs | 105 +++++ crates/argand-site-registry/tests/failures.rs | 190 +++++++- .../argand-site-registry/tests/observation.rs | 60 +++ crates/argand-site-registry/tests/registry.rs | 52 ++- docs/CONSUMERS.md | 18 +- docs/EVALUATION.md | 35 ++ docs/INDEX.md | 2 + docs/PUBLISHING.md | 75 +++ docs/RELEASING.md | 5 +- docs/TRUST.md | 19 +- scripts/source_release.py | 2 +- 37 files changed, 2856 insertions(+), 286 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 ci/Dockerfile create mode 100644 ci/README.md create mode 100644 crates/argand-site-registry/migrations/002.sql create mode 100644 crates/argand-site-registry/src/catalog.rs create mode 100644 crates/argand-site-registry/src/diff.rs create mode 100644 crates/argand-site-registry/src/evaluation.rs create mode 100644 crates/argand-site-registry/src/generation.rs create mode 100644 crates/argand-site-registry/src/resolution.rs create mode 100644 crates/argand-site-registry/src/ssh.rs create mode 100644 crates/argand-site-registry/tests/evaluation.rs create mode 100644 crates/argand-site-registry/tests/observation.rs create mode 100644 docs/EVALUATION.md create mode 100644 docs/PUBLISHING.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a5ee2e5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog + +## 0.2.0 - 2026-09-13 + +- Open generations from an authenticated immutable SQLite file descriptor and + reject sidecars, symlinks and unexpected generation files. +- Authenticate exact reviewer decisions with SSH signatures, retain their proofs + and re-verify every decision before release signing. +- Explain resolver abstentions with typed statuses and decision counts. +- Add exact entity, URL/domain, source-separated popularity, redacted category and + registry-statistics audit views. +- Stream typed diffs across source selections, projections, decisions and explicit + equivalences. +- Add bounded native evaluation with multilingual, regional and deceptive-query + regressions plus latency evidence. +- Normalize rights-bearing future crawler observations without admitting them as + identity or ownership evidence. + +## 0.1.0 - 2026-09-12 + +- Initial independent extraction with five rights-reviewed source adapters, + deterministic normalization, immutable generations and conservative regional + resolution. diff --git a/Cargo.lock b/Cargo.lock index b2cefd0..5ce0190 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,14 +78,14 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "argand-atomic" -version = "0.1.0" +version = "0.2.0" dependencies = [ "tempfile", ] [[package]] name = "argand-site-registry" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "argand-atomic", @@ -95,6 +95,7 @@ dependencies = [ "csv", "flate2", "http", + "libc", "publicsuffix", "reqwest", "rusqlite", @@ -212,9 +213,9 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.0", diff --git a/Cargo.toml b/Cargo.toml index eec1733..c411b22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "3" members = ["crates/argand-atomic", "crates/argand-site-registry"] [workspace.package] -version = "0.1.0" +version = "0.2.0" authors = ["Nic Weyand"] edition = "2024" license = "AGPL-3.0-or-later" @@ -16,6 +16,7 @@ chrono = { version = "0.4.44", features = ["serde"] } clap = { version = "4.5.60", features = ["derive"] } flate2 = "1.1.9" http = "1.4.0" +libc = "0.2.189" reqwest = { version = "0.13.2", default-features = false, features = ["json", "query", "rustls", "stream"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 084769a..18e0ab0 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -15,8 +15,8 @@ a consumer's accepted destinations or signing keys. Policy, license, normalization, source-allowlist and signature changes receive explicit maintainer review and complete acceptance checks. Additional independent review is appropriate for trust-boundary changes when another qualified reviewer -is available. This is a governance expectation; the current software does not -enforce a multi-reviewer quorum or authenticate a free-text reviewer name. +is available. The software authenticates each decision to an allowed SSH reviewer +identity, but does not enforce a multi-reviewer quorum. Corrections and appeals must identify the exact assertion or review fingerprint and supply contrary evidence. Retain the original claim and decision, append the diff --git a/README.md b/README.md index df82cde..43ccc20 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,10 @@ own admission policy. A null destination is a meaningful abstention. The [consumer contract](docs/CONSUMERS.md) covers Rust dependencies, CLI JSON, SQLite/JSONL distribution, compatibility and Argand's eventual upstream cutover. +Exact entity, URL/domain, source-separated popularity and redacted Curlie category +queries support audit tools. `resolve` also reports a stable abstention reason and +decision counts. The bounded `evaluate` command replays JSONL judgments and reports +accuracy plus native p50/p95 latency for one pinned generation. ## Sources and trust @@ -92,7 +96,9 @@ and the Public Suffix List (MPL 2.0) remain logically separate. Read the exact Curlie attribution applies to names and categories as well as descriptions. The [trust policy](docs/TRUST.md) explains enforced checks, publisher responsibilities, -evidence standards, expiry and revocation. [CONTRIBUTING.md](CONTRIBUTING.md), +evidence standards, expiry and revocation. The [publisher runbook](docs/PUBLISHING.md) +covers authenticated reviewer decisions, candidate inspection and activation. +[CONTRIBUTING.md](CONTRIBUTING.md), [GOVERNANCE.md](GOVERNANCE.md) and [SECURITY.md](SECURITY.md) cover contributions, decisions, disputes and incidents. Pull requests cannot directly approve destinations. @@ -109,9 +115,9 @@ documentation, Python release tests, and native Rust/Python consumer parity. verification and rebuilding outside the checkout. The Forgejo workflow requires a dedicated isolated runner; it has no signing or dataset-promotion authority. -The [validation record](docs/VALIDATION.md) reports the initial independent builds -and native acceptance. Hosted CI requires runner provisioning; the workflow is -included and Actions remains disabled until an isolated runner is ready. +The [validation record](docs/VALIDATION.md) reports independent builds and native +acceptance. Hosted checks use the repository's isolated Forgejo runner label and +have no dataset, review, signing or activation authority. The code remains **AGPL-3.0-or-later**; the complete license is in [LICENSE](LICENSE). Original attribution is retained. [UPSTREAM.json](UPSTREAM.json) records the signed diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 0000000..4fa4353 --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,13 @@ +# By Nic Weyand! Pinned toolchain image for the repository-scoped isolated runner. +FROM rust:1.98.0-trixie@sha256:620dbcd124499c59e2406d3741574b5c5838cf9eb9656f0c3a03948f79b02959 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + cmake \ + openssh-client \ + python3 \ + shellcheck \ + && rm -rf /var/lib/apt/lists/* \ + && rustup component add clippy rustfmt + +WORKDIR /workspace diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..08d2907 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,19 @@ +# Isolated Forgejo runner image + +Build the reviewed pinned image and register a repository-scoped runner with only +the container label below: + +```bash +docker build --pull -t argand-site-registry-ci:0.2.0 -f ci/Dockerfile . +forgejo-runner register --no-interactive \ + --instance https://git.argand.org \ + --token REPOSITORY_REGISTRATION_TOKEN \ + --name argand-site-registry-isolated \ + --labels site-registry-isolated:docker://argand-site-registry-ci:0.2.0 +``` + +Run the daemon with capacity one, no host label, no bind-volume allowlist, no +Docker socket inside jobs, `privileged: false`, and container limits of two CPUs, +4 GiB memory and 4 GiB memory plus swap. The workflow fetches the exact public +commit without repository credentials. This runner contains no dataset, reviewer, +release-signing or activation authority. diff --git a/crates/argand-site-registry/Cargo.toml b/crates/argand-site-registry/Cargo.toml index 0842be5..1c4c675 100644 --- a/crates/argand-site-registry/Cargo.toml +++ b/crates/argand-site-registry/Cargo.toml @@ -15,6 +15,7 @@ chrono.workspace = true clap.workspace = true csv = "1.4.0" flate2.workspace = true +libc.workspace = true publicsuffix = "=2.3.0" reqwest.workspace = true rusqlite = { version = "=0.40.2", features = ["bundled"] } @@ -22,13 +23,13 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true tar = "0.4.46" +tempfile.workspace = true tokio.workspace = true toml.workspace = true unicode-normalization.workspace = true url.workspace = true [dev-dependencies] -tempfile.workspace = true http.workspace = true [lints] diff --git a/crates/argand-site-registry/README.md b/crates/argand-site-registry/README.md index 2a4c5e4..25d1492 100644 --- a/crates/argand-site-registry/README.md +++ b/crates/argand-site-registry/README.md @@ -206,11 +206,18 @@ Replace the example evidence and dates; approvals expire within 90 days. Use `role: "primary"` for the independently verified default and `country: "DE"` for a separately verified German regional property. A country-scoped review can leave locale empty. If both are specified, both must match the request. +Sign the exact decision bytes under the dedicated reviewer namespace. The reviewer +identity must match the JSON and an independently maintained OpenSSH allowed-signers +file. The release signing key may be separate from reviewer keys. ```bash +ssh-keygen -Y sign -n argand-site-registry-review \ + -f /secure/reviewer-key review.json argand-site-registry review --database "$ARGAND_SITE_DATA/import.sqlite" \ --generation "$ARGAND_SITE_DATA/generation-1" --pin "$ARGAND_SITE_PIN" \ - --decision review.json + --decision review.json --signature review.json.sig \ + --allowed-reviewers /secure/reviewer-allowed-signers \ + --identity operator@example.org argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \ --output "$ARGAND_SITE_DATA/generation-2" > "$ARGAND_SITE_DATA/build-2.json" export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-2.json")" @@ -221,7 +228,8 @@ argand-site-registry resolve --generation "$ARGAND_SITE_DATA/generation-2" \ After the corresponding real reviews, GB selects the reviewed UK property; DE selects the reviewed German property; otherwise an explicitly reviewed primary may be used. Unknown, expired, tied or entity-ambiguous requests return -`"destination": null`. Result limits never hide ambiguity. Name/alias changes, +`"destination": null` with `status` and rejection counts. Result limits never hide +ambiguity. Name/alias changes, changed statements/revisions, URLs or normalization evidence invalidate reviews. Deprecated, end-dated and non-value statements remain audit evidence and cannot be admitted. An unchanged PSL file with a new retrieval time preserves reviews. @@ -237,9 +245,10 @@ argand-site-registry equivalence --generation "$ARGAND_SITE_DATA/generation-2" \ ``` Use the returned fingerprint in a review JSON with `role: "unspecified"`, empty -locale/country, a reason, immutable identity evidence and an expiry. Then repeat -the command with `--database "$ARGAND_SITE_DATA/import.sqlite" --decision -identity-review.json` and rebuild. `resolve` follows only active, explicitly +locale/country, a reason, immutable identity evidence and an expiry. Sign +`identity-review.json` with the reviewer namespace and repeat the command with +`--database`, `--decision`, `--signature`, `--allowed-reviewers` and `--identity`, +then rebuild. `resolve` follows only active, explicitly reviewed equivalences and includes their provenance. Each destination still needs its own review. The original IDs, raw ambiguity counts and conflicting assertions remain visible. Changed names or website assertions invalidate the @@ -258,13 +267,16 @@ source-bearing JSONL and omits descriptions by default: argand-site-registry export --generation "$ARGAND_SITE_DATA/generation-2" \ --pin "$ARGAND_SITE_PIN" --output "$ARGAND_SITE_DATA/assertions.jsonl" argand-site-registry sign --generation "$ARGAND_SITE_DATA/generation-2" \ - --pin "$ARGAND_SITE_PIN" --key /secure/registry-signing-key + --pin "$ARGAND_SITE_PIN" --key /secure/registry-signing-key \ + --allowed-reviewers /secure/reviewer-allowed-signers argand-site-registry activate --generation "$ARGAND_SITE_DATA/generation-2" \ --current "$ARGAND_SITE_DATA/current.json" \ --allowed-signers /secure/registry-allowed-signers --identity registry-publisher ``` -Use an existing operator-controlled SSH signing key. The external allowed-signers +Before signing, the CLI replays every stored reviewer signature against the supplied +reviewer trust file and rejects missing, forged or altered proofs. Use an existing +operator-controlled SSH release key. The external release allowed-signers file follows OpenSSH syntax: `registry-publisher ssh-ed25519 PUBLIC_KEY`. Neither keys nor the trust file should come from the downloaded dataset. Consumers can open `Registry::open(path, trusted_receipt_sha256)` once and reuse its indexed @@ -272,8 +284,10 @@ queries, or verify a publisher with `release::verify_signed` first. A hash prove integrity only relative to a trusted pin. Signature verification authenticates the publisher, not the truth of a source assertion. -`diff --old PATH --old-pin HASH --new PATH --new-pin HASH` streams added/removed -edge fingerprints. `verify --generation PATH --pin HASH` checks every artifact +`diff --old PATH --old-pin HASH --new PATH --new-pin HASH` streams typed added, +removed and changed source selections, entities, names, properties, edges, +popularity observations, reviews and equivalences. `verify --generation PATH +--pin HASH` checks every artifact bound by the receipt. To revoke, append a review with `decision: "revoke"`, then rebuild, sign and activate. Re-activating an older signed generation supports rollback **only if it retains every distributed revocation**. Otherwise rebuild @@ -299,9 +313,9 @@ downloads. No scheduled job signs, approves, renews approvals or activates links ## Storage and operating limits -Migration `migrations/001.sql` owns schema version 1. `sources`, `records` and +Migrations `migrations/001.sql` and `002.sql` own schema version 2. `sources`, `records` and `facts` preserve snapshot/native IDs, licenses, retrieval times and confidence; -`reviews` is append-only. Complete source selection is latest retrieval time per +`reviews` and their cryptographic `review_auth` proofs are append-only. Complete source selection is latest retrieval time per provider/scope, with digest as the deterministic tie break. Use the **same scope** for a replacement snapshot, and separate scopes for deliberate independent selections. History and conflicts remain stored. A failed source cannot replace @@ -317,7 +331,10 @@ join. The canonical label rule prefers labels, then English, then language/text order. Original labels/aliases are kept. Popularity has its own source, target, observation period and audience scope and never creates an ownership edge. All derivations bind input fact IDs, PSL identity and the -`argand.site-rules/v1` contract through their generation receipt. +`argand.site-rules/v2` contract through their generation receipt. A v1 writer store +migrates in place while retaining review history. Any legacy unauthenticated +decision makes release signing fail closed; start a reviewed v2 store from the +pinned source inputs rather than deleting historical decisions. Imports use transactions of 256 relevant records with durable replay checkpoints. Restart replays the compressed stream and skips committed records. Large source @@ -330,9 +347,15 @@ The full store/history and each generation consume disk; there is no automatic pruning. The original compressed source is hashed before/after import, so expect extra sequential disk reads. These bounds are not a full-dump throughput claim. -The `observation` module defines future crawler evidence for redirects, canonical -links, hreflang, JSON-LD sameAs, sitemaps and country selectors, with capture IDs, -hashes, rights and confidence. It does not crawl or automatically infer ownership. +The `observation` module validates and deterministically normalizes future crawler +evidence for redirects, canonical links, hreflang, JSON-LD sameAs, sitemaps and +country selectors, with capture IDs, hashes, rights and confidence. It does not +crawl, admit a new source or automatically infer ownership. + +Use `entity`, `lookup-web`, `popularity`, `category` and `stats` for reverse/audit +views. Curlie descriptions remain redacted on the category surface. The +`evaluate` command accepts bounded JSONL judgments; see the repository +[evaluation guide](../../docs/EVALUATION.md) and [publisher runbook](../../docs/PUBLISHING.md). Source vandalism, compromised publishers and a domain changing ownership cannot be eliminated by hashes or popularity. Review expiration, exact evidence binding, diff --git a/crates/argand-site-registry/migrations/002.sql b/crates/argand-site-registry/migrations/002.sql new file mode 100644 index 0000000..6c15b48 --- /dev/null +++ b/crates/argand-site-registry/migrations/002.sql @@ -0,0 +1,14 @@ +-- By Nic Weyand! Authenticate operator decisions and move new builds to rules v2. +CREATE TABLE review_auth ( + sequence INTEGER PRIMARY KEY REFERENCES reviews(sequence), + signer TEXT NOT NULL, + signature_sha256 TEXT NOT NULL, + namespace TEXT NOT NULL CHECK(namespace='argand-site-registry-review'), + decision_sha256 TEXT NOT NULL, + decision_json BLOB NOT NULL, + signature BLOB NOT NULL +) STRICT; +CREATE TRIGGER review_auth_no_update BEFORE UPDATE ON review_auth BEGIN SELECT RAISE(ABORT,'review authentication is immutable'); END; +CREATE TRIGGER review_auth_no_delete BEFORE DELETE ON review_auth BEGIN SELECT RAISE(ABORT,'review authentication is immutable'); END; +UPDATE registry_metadata SET rules='argand.site-rules/v2' WHERE singleton=1; +PRAGMA user_version=2; diff --git a/crates/argand-site-registry/src/build.rs b/crates/argand-site-registry/src/build.rs index 081c715..8f1d1ad 100644 --- a/crates/argand-site-registry/src/build.rs +++ b/crates/argand-site-registry/src/build.rs @@ -148,6 +148,11 @@ fn copy_canonical(source: &Connection, destination: &Connection) -> anyhow::Resu 8, ), ("reviews", "SELECT * FROM reviews ORDER BY sequence", 11), + ( + "review_auth", + "SELECT * FROM review_auth ORDER BY sequence", + 7, + ), ( "equivalences", "SELECT * FROM equivalences ORDER BY fingerprint", @@ -342,9 +347,11 @@ fn project_edge( }; let evidence = json!({"source":source,"native_id":native,"selector":selector,"assertion":value,"names_fingerprint":names,"normalization":store::RULE_VERSION}); let mut identity_property = property.clone(); - // Bind review to actual PSL bytes and derived fields, not a fresh timestamp - // for an otherwise identical list. Full retrieval provenance stays on property. + // Bind reviews to the normalization result. The complete PSL identity remains + // on the property, while comment-only or unrelated rule changes do not force + // re-review when this hostname's derived fields are identical. identity_property.domain.psl_source.clear(); + identity_property.domain.psl_sha256.clear(); let fingerprint = crate::digest(&serde_json::to_vec(&( entity, &identity_property, diff --git a/crates/argand-site-registry/src/catalog.rs b/crates/argand-site-registry/src/catalog.rs new file mode 100644 index 0000000..b0fbab3 --- /dev/null +++ b/crates/argand-site-registry/src/catalog.rs @@ -0,0 +1,427 @@ +// By Nic Weyand! +//! Exact reverse registry views. These never infer entity ownership from domains. + +use crate::{ + evidence, + normalize::{Normalizer, WebProperty}, + query::{Candidate, Registry}, +}; +use anyhow::{Context, ensure}; +use rusqlite::{OptionalExtension, params}; +use serde::Serialize; +use serde_json::{Value, json}; + +/// Auditable generation size, evidence and review counts. +#[derive(Debug, Serialize)] +pub struct RegistryStats { + /// Trusted receipt pin used to open this generation. + pub registry: String, + /// Versioned derivation and review contract. + pub rules: String, + /// SQLite logical byte size from page count and page size. + pub database_bytes: u64, + /// All retained complete source snapshots. + pub source_snapshots: u64, + /// Active source selections used for projections. + pub selected_sources: u64, + /// Retained source records. + pub records: u64, + /// Retained source facts. + pub facts: u64, + /// Operator decisions. + pub reviews: u64, + /// Decisions carrying verified reviewer authentication. + pub authenticated_reviews: u64, + /// Explicit cross-source identity proposals. + pub equivalences: u64, + /// Current approvals expiring during the next seven days. + pub approvals_expiring_within_seven_days: u64, + /// Receipt-level entity count. + pub entities: u64, + /// Receipt-level strict URL count. + pub properties: u64, + /// Receipt-level entity/property edge count. + pub edges: u64, + /// Receipt-level rejected fact count. + pub rejected: u64, +} + +/// One entity and all bounded website assertions attached to its stable ID. +#[derive(Debug, Serialize)] +pub struct EntityView { + /// Provenance-bearing names and metadata. + pub entity: evidence::Entity, + /// Complete edge count before the output limit. + pub total_edges: u64, + /// True when candidates were omitted by the limit. + pub truncated: bool, + /// Source assertions; review status remains explicit on each candidate. + pub candidates: Vec, + /// Attribution required when displaying imported fields. + pub attribution: Value, +} + +/// One exact URL/hostname/registrable-domain match. +#[derive(Debug, Serialize)] +pub struct WebMatch { + /// `url`, `hostname`, or `registrable_domain`. + pub matched_by: String, + /// Original entity/property assertion and review state. + pub candidate: Candidate, +} + +/// Reverse property lookup without an ownership inference. +#[derive(Debug, Serialize)] +pub struct WebLookup { + /// Original input. + pub input: String, + /// Strict normalized URL or domain evidence. + pub normalized: Value, + /// Complete property count before the output limit. + pub total_properties: u64, + /// Complete assertion count before the output limit. + pub total_edges: u64, + /// True when assertions were omitted by the limit. + pub truncated: bool, + /// Exact assertion matches. + pub matches: Vec, + /// Attribution required when displaying imported fields. + pub attribution: Value, +} + +/// One source-specific popularity observation with complete provenance. +#[derive(Debug, Serialize)] +pub struct PopularityObservation { + /// Provider namespace; signals never combine implicitly. + pub source: String, + /// Original source target. + pub target: String, + /// Source-native rank/period/audience values. + pub value: Value, + /// PSL-derived hostname/domain evidence. + pub domain: Value, + /// Complete imported fact declaration. + pub provenance: Value, +} + +/// Exact popularity lookup, kept separate from entity ownership. +#[derive(Debug, Serialize)] +pub struct PopularityLookup { + /// Original input. + pub input: String, + /// Strict normalized target. + pub normalized: Value, + /// Complete observation count. + pub total: u64, + /// True when observations were omitted by the limit. + pub truncated: bool, + /// Source-specific observations. + pub observations: Vec, + /// Required provider attribution. + pub attribution: Value, +} + +/// Redacted category metadata and member assertions. +#[derive(Debug, Serialize)] +pub struct CategoryLookup { + /// Exact Curlie category ID. + pub category_id: String, + /// Category facts with descriptions removed from this display surface. + pub metadata: Vec, + /// Complete member assertion count. + pub total_members: u64, + /// True when members were omitted by the limit. + pub truncated: bool, + /// Directory assertions for category members. + pub members: Vec, + /// Curlie and other provider attribution. + pub attribution: Value, +} + +impl Registry { + /// Reports bounded storage and decision-health measurements. + /// + /// # Errors + /// Returns malformed database state or query failures. + pub fn stats(&self, now: chrono::DateTime) -> anyhow::Result { + let count = |table: &str| -> anyhow::Result { + Ok(self + .db + .query_row(&format!("SELECT count(*) FROM {table}"), [], |row| { + crate::store::unsigned(row, 0) + })?) + }; + let page_count: u64 = self.db.query_row("PRAGMA page_count", [], |row| { + crate::store::unsigned(row, 0) + })?; + let page_size: u64 = self + .db + .query_row("PRAGMA page_size", [], |row| crate::store::unsigned(row, 0))?; + let deadline = now + chrono::Duration::days(7); + let approvals_expiring_within_seven_days = self.db.query_row( + "SELECT count(*) FROM reviews r WHERE r.decision='approve' AND r.sequence=(SELECT max(sequence) FROM reviews WHERE fingerprint=r.fingerprint) AND r.expires_at>?1 AND r.expires_at<=?2", + params![now.to_rfc3339(), deadline.to_rfc3339()], + |row| crate::store::unsigned(row, 0), + )?; + Ok(RegistryStats { + registry: self.identity.clone(), + rules: self.receipt.rules.clone(), + database_bytes: page_count.saturating_mul(page_size), + source_snapshots: count("sources")?, + selected_sources: count("selected_sources")?, + records: count("records")?, + facts: count("facts")?, + reviews: count("reviews")?, + authenticated_reviews: count("review_auth")?, + equivalences: count("equivalences")?, + approvals_expiring_within_seven_days, + entities: self.receipt.entities, + properties: self.receipt.properties, + edges: self.receipt.edges, + rejected: self.receipt.rejected, + }) + } + + /// Looks up one source-derived stable entity ID. + /// + /// # Errors + /// Rejects malformed IDs, output limits, absent entities, or corrupt data. + pub fn entity_by_id(&self, id: &str, limit: u32) -> anyhow::Result> { + bound_limit(limit)?; + ensure!( + id.starts_with("argand:entity:") && id.len() <= 256, + "invalid entity ID" + ); + let name: Option = self + .db + .query_row( + "SELECT canonical_name FROM entities WHERE id=?1", + [id], + |row| row.get(0), + ) + .optional()?; + let Some(name) = name else { + return Ok(None); + }; + let total_edges = + self.db + .query_row("SELECT count(*) FROM edges WHERE entity=?1", [id], |row| { + crate::store::unsigned(row, 0) + })?; + let candidates = self.candidates_for( + "SELECT fingerprint FROM edges WHERE entity=?1 ORDER BY property,fingerprint LIMIT ?2", + params![id, limit], + )?; + Ok(Some(EntityView { + entity: evidence::entity(&self.db, id, &name)?, + total_edges, + truncated: total_edges > u64::from(limit), + candidates, + attribution: crate::release::attribution(), + })) + } + + /// Finds source assertions for an exact URL, hostname, or registrable domain. + /// + /// # Errors + /// Rejects malformed targets/limits and corrupt registry evidence. + pub fn lookup_web(&self, input: &str, limit: u32) -> anyhow::Result { + bound_limit(limit)?; + let normalizer = self.normalizer()?; + let (target, properties, total_properties, total_edges) = if input.contains("://") { + let property = normalizer.url(input)?; + let properties = self.property_edges("p.url=?1", params![&property.url], limit)?; + let counts = self.property_counts("url=?1", params![&property.url])?; + ( + serde_json::to_value(property)?, + properties, + counts.0, + counts.1, + ) + } else { + let domain = normalizer.domain(input)?; + let properties = self.property_edges( + "(p.hostname=?1 OR p.domain=?2)", + params![&domain.hostname, &domain.registrable_domain], + limit, + )?; + let counts = self.property_counts( + "hostname=?1 OR domain=?2", + params![&domain.hostname, &domain.registrable_domain], + )?; + ( + serde_json::to_value(domain)?, + properties, + counts.0, + counts.1, + ) + }; + let matches = properties + .into_iter() + .map(|candidate| { + let property: WebProperty = serde_json::from_value(candidate.web_property.clone())?; + let matched_by = if input.contains("://") { + "url" + } else if target["hostname"] == property.domain.hostname { + "hostname" + } else { + "registrable_domain" + }; + Ok(WebMatch { + matched_by: matched_by.into(), + candidate, + }) + }) + .collect::>>()?; + Ok(WebLookup { + input: input.into(), + normalized: target, + total_properties, + total_edges, + truncated: total_edges > u64::from(limit), + matches, + attribution: crate::release::attribution(), + }) + } + + /// Returns exact source-specific popularity observations for a web target. + /// + /// # Errors + /// Rejects malformed targets/limits and corrupt registry evidence. + pub fn popularity(&self, input: &str, limit: u32) -> anyhow::Result { + bound_limit(limit)?; + let normalizer = self.normalizer()?; + let domain = if input.contains("://") { + normalizer.url(input)?.domain + } else { + normalizer.domain(input)? + }; + let total = self.db.query_row( + "SELECT count(*) FROM popularity WHERE hostname=?1 OR domain=?2", + params![&domain.hostname, &domain.registrable_domain], + |row| crate::store::unsigned(row, 0), + )?; + let mut statement = self.db.prepare("SELECT fact,source,target,value,derived_json FROM popularity WHERE hostname=?1 OR domain=?2 ORDER BY source,fact LIMIT ?3")?; + let rows = statement.query_map( + params![&domain.hostname, &domain.registrable_domain, limit], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + }, + )?; + let mut observations = Vec::new(); + for row in rows { + let (fact, source, target, value, derived) = row?; + observations.push(PopularityObservation { + source, + target, + value: serde_json::from_str(&value)?, + domain: serde_json::from_str(&derived)?, + provenance: evidence::fact(&self.db, &fact)?, + }); + } + Ok(PopularityLookup { + input: input.into(), + normalized: serde_json::to_value(domain)?, + total, + truncated: total > u64::from(limit), + observations, + attribution: crate::release::attribution(), + }) + } + + /// Returns one exact Curlie category without exposing copied descriptions. + /// + /// # Errors + /// Rejects malformed category IDs/limits and corrupt registry evidence. + pub fn category(&self, category_id: &str, limit: u32) -> anyhow::Result { + bound_limit(limit)?; + ensure!( + !category_id.is_empty() + && category_id.len() <= 32 + && category_id.bytes().all(|byte| byte.is_ascii_digit()), + "invalid category ID" + ); + let mut statement = self.db.prepare("SELECT f.id FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category' AND json_extract(f.value,'$.category_id')=?1 ORDER BY f.id")?; + let mut metadata = Vec::new(); + for id in statement.query_map([category_id], |row| row.get::<_, String>(0))? { + let mut fact = evidence::fact(&self.db, &id?)?; + if let Some(value) = fact["value"].as_object_mut() { + value.remove("description"); + value.insert("description_redacted".into(), json!(true)); + } + metadata.push(fact); + } + let total_members = self.db.query_row("SELECT count(*) FROM edges e WHERE e.entity IN(SELECT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category_membership' AND json_extract(f.value,'$.category_id')=?1)",[category_id],|row|crate::store::unsigned(row,0))?; + let members = self.candidates_for("SELECT e.fingerprint FROM edges e WHERE e.entity IN(SELECT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category_membership' AND json_extract(f.value,'$.category_id')=?1) ORDER BY e.entity,e.fingerprint LIMIT ?2",params![category_id,limit])?; + Ok(CategoryLookup { + category_id: category_id.into(), + metadata, + total_members, + truncated: total_members > u64::from(limit), + members, + attribution: crate::release::attribution(), + }) + } + + fn normalizer(&self) -> anyhow::Result { + let (source, raw): (String, String) = self.db.query_row("SELECT f.source_id,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl' ORDER BY f.source_id LIMIT 1",[],|row|Ok((row.get(0)?,row.get(1)?))).context("generation has no PSL")?; + let text: String = serde_json::from_str(&raw)?; + Normalizer::new(text.as_bytes(), source) + } + + fn candidates_for( + &self, + sql: &str, + params: P, + ) -> anyhow::Result> { + let mut statement = self.db.prepare(sql)?; + let fingerprints = statement + .query_map(params, |row| row.get::<_, String>(0))? + .collect::, _>>()?; + fingerprints + .iter() + .map(|fingerprint| self.candidate(fingerprint)) + .collect() + } + + fn property_edges( + &self, + predicate: &str, + params: P, + limit: u32, + ) -> anyhow::Result> { + self.candidates_for( + &format!("SELECT e.fingerprint FROM edges e JOIN properties p ON p.id=e.property WHERE {predicate} ORDER BY p.id,e.entity,e.fingerprint LIMIT {limit}"), + params, + ) + } + + fn property_counts( + &self, + predicate: &str, + params: P, + ) -> anyhow::Result<(u64, u64)> { + let properties = self.db.query_row( + &format!("SELECT count(*) FROM properties WHERE {predicate}"), + params.clone(), + |row| crate::store::unsigned(row, 0), + )?; + let edges = self.db.query_row( + &format!("SELECT count(*) FROM edges e JOIN properties p ON p.id=e.property WHERE {predicate}"), + params, + |row| crate::store::unsigned(row, 0), + )?; + Ok((properties, edges)) + } +} + +fn bound_limit(limit: u32) -> anyhow::Result<()> { + ensure!((1..=100).contains(&limit), "lookup limit must be 1..100"); + Ok(()) +} diff --git a/crates/argand-site-registry/src/cli.rs b/crates/argand-site-registry/src/cli.rs index 45b1167..a585637 100644 --- a/crates/argand-site-registry/src/cli.rs +++ b/crates/argand-site-registry/src/cli.rs @@ -1,7 +1,7 @@ // By Nic Weyand! //! Explicit source acquisition, import, review, and immutable generation commands. -use anyhow::ensure; +use anyhow::{Context, ensure}; use argand_site_registry as registry; use clap::{Parser, Subcommand}; use registry::{ @@ -36,6 +36,12 @@ enum Command { decision: Option, #[arg(long)] database: Option, + #[arg(long, requires = "decision")] + signature: Option, + #[arg(long, requires = "decision")] + allowed_reviewers: Option, + #[arg(long, requires = "decision")] + identity: Option, }, /// Download one allowlisted source into an immutable local cache. Download { @@ -111,6 +117,70 @@ enum Command { #[arg(long, default_value_t = 20)] limit: u32, }, + /// Inspect one stable entity ID and its bounded website assertions. + Entity { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + id: String, + #[arg(long, default_value_t = 20)] + limit: u32, + }, + /// Reverse lookup an exact URL, hostname, or registrable domain. + LookupWeb { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + target: String, + #[arg(long, default_value_t = 20)] + limit: u32, + }, + /// Inspect source-separated popularity observations for a web target. + Popularity { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + target: String, + #[arg(long, default_value_t = 20)] + limit: u32, + }, + /// Inspect a Curlie category with copied descriptions redacted. + Category { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + id: String, + #[arg(long, default_value_t = 20)] + limit: u32, + }, + /// Report generation size, evidence counts and upcoming approval expiry. + Stats { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + }, + /// Replay bounded JSONL judgments and report accuracy and native latency. + Evaluate { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + cases: PathBuf, + #[arg(long, default_value_t = 10_000)] + maximum_cases: u32, + #[arg(long)] + at: Option>, + }, /// Resolve only an unambiguous, explicitly reviewed, unexpired property. Resolve { #[arg(long)] @@ -134,6 +204,12 @@ enum Command { pin: String, #[arg(long)] decision: PathBuf, + #[arg(long)] + signature: PathBuf, + #[arg(long)] + allowed_reviewers: PathBuf, + #[arg(long)] + identity: String, }, /// Export facts as streaming JSONL with source licenses and attribution. Export { @@ -172,6 +248,8 @@ enum Command { pin: String, #[arg(long)] key: PathBuf, + #[arg(long)] + allowed_reviewers: PathBuf, }, /// Verify publisher signature and atomically activate (also supports rollback). Activate { @@ -197,6 +275,14 @@ pub(super) async fn run() -> anyhow::Result<()> { | Command::CruxDownload { .. } | Command::Manifest { .. } | Command::Equivalence { .. }) => acquire(command).await?, + command @ (Command::Lookup { .. } + | Command::Entity { .. } + | Command::LookupWeb { .. } + | Command::Popularity { .. } + | Command::Category { .. } + | Command::Stats { .. } + | Command::Evaluate { .. } + | Command::Resolve { .. }) => inspect(command)?, Command::Import { database, input, @@ -209,28 +295,22 @@ pub(super) async fn run() -> anyhow::Result<()> { let receipt = registry::build::build(®istry::store::open(&database)?, &output)?; serde_json::json!({"receipt":receipt,"pin":registry::file_digest(&output.join("COMPLETE.json"))?,"generation":output}) } - Command::Lookup { - generation, - pin, - query, - limit, - } => serde_json::to_value(Registry::open(&generation, &pin)?.lookup(&query, limit)?)?, - Command::Resolve { - generation, - pin, - query, - locale, - country, - } => { - serde_json::json!({"destination":Registry::open(&generation,&pin)?.resolve(&query,locale.as_deref(),country.as_deref(),chrono::Utc::now())?,"attribution":registry::release::attribution()}) - } Command::Review { database, generation, pin, decision, + signature, + allowed_reviewers, + identity, } => { - serde_json::json!({"review_sequence":registry::review::record(®istry::store::open(&database)?,&Registry::open(&generation,&pin)?,®istry::read_json(&decision)?)?,"rebuild_required":true}) + let (review, authentication) = registry::review::authenticate( + &decision, + &signature, + &allowed_reviewers, + &identity, + )?; + serde_json::json!({"review_sequence":registry::review::record_authenticated(®istry::store::open(&database)?,&Registry::open(&generation,&pin)?,&review,&authentication)?,"authenticated_reviewer":identity,"rebuild_required":true}) } Command::Export { generation, @@ -265,8 +345,9 @@ pub(super) async fn run() -> anyhow::Result<()> { generation, pin, key, + allowed_reviewers, } => { - registry::release::sign(&generation, &key, &pin)?; + registry::release::sign(&generation, &key, &pin, &allowed_reviewers)?; serde_json::json!({"signed":generation}) } Command::Activate { @@ -283,14 +364,81 @@ pub(super) async fn run() -> anyhow::Result<()> { serde_json::json!({"candidate":registry::update::run(&config).await?}) } }; + write_json(&value) +} + +fn write_json(value: &serde_json::Value) -> anyhow::Result<()> { writeln!( std::io::stdout().lock(), "{}", - serde_json::to_string_pretty(&value)? + serde_json::to_string_pretty(value)? )?; Ok(()) } +fn inspect(command: Command) -> anyhow::Result { + Ok(match command { + Command::Lookup { + generation, + pin, + query, + limit, + } => serde_json::to_value(Registry::open(&generation, &pin)?.lookup(&query, limit)?)?, + Command::Entity { + generation, + pin, + id, + limit, + } => serde_json::to_value(Registry::open(&generation, &pin)?.entity_by_id(&id, limit)?)?, + Command::LookupWeb { + generation, + pin, + target, + limit, + } => serde_json::to_value(Registry::open(&generation, &pin)?.lookup_web(&target, limit)?)?, + Command::Popularity { + generation, + pin, + target, + limit, + } => serde_json::to_value(Registry::open(&generation, &pin)?.popularity(&target, limit)?)?, + Command::Category { + generation, + pin, + id, + limit, + } => serde_json::to_value(Registry::open(&generation, &pin)?.category(&id, limit)?)?, + Command::Stats { generation, pin } => { + serde_json::to_value(Registry::open(&generation, &pin)?.stats(chrono::Utc::now())?)? + } + Command::Evaluate { + generation, + pin, + cases, + maximum_cases, + at, + } => serde_json::to_value(registry::evaluation::run_file( + &Registry::open(&generation, &pin)?, + &cases, + maximum_cases, + at.unwrap_or_else(chrono::Utc::now), + )?)?, + Command::Resolve { + generation, + pin, + query, + locale, + country, + } => serde_json::to_value(Registry::open(&generation, &pin)?.resolve_explained( + &query, + locale.as_deref(), + country.as_deref(), + chrono::Utc::now(), + )?)?, + _ => anyhow::bail!("expected a registry inspection command"), + }) +} + async fn acquire(command: Command) -> anyhow::Result { Ok(match command { Command::Equivalence { @@ -300,13 +448,22 @@ async fn acquire(command: Command) -> anyhow::Result { right, decision, database, + signature, + allowed_reviewers, + identity, } => { let registry = Registry::open(&generation, &pin)?; let pair = registry::identity::propose(®istry, &left, &right)?; if let Some(decision) = decision { let database = database .ok_or_else(|| anyhow::anyhow!("identity review needs a writer database"))?; - serde_json::json!({"review_sequence":registry::identity::record(®istry::store::open(&database)?,®istry,&pair,®istry::read_json(&decision)?)?,"rebuild_required":true}) + let signature = signature.context("identity review needs --signature")?; + let allowed = + allowed_reviewers.context("identity review needs --allowed-reviewers")?; + let identity = identity.context("identity review needs --identity")?; + let (review, authentication) = + registry::review::authenticate(&decision, &signature, &allowed, &identity)?; + serde_json::json!({"review_sequence":registry::identity::record_authenticated(®istry::store::open(&database)?,®istry,&pair,&review,&authentication)?,"authenticated_reviewer":identity,"rebuild_required":true}) } else { serde_json::to_value(pair)? } diff --git a/crates/argand-site-registry/src/diff.rs b/crates/argand-site-registry/src/diff.rs new file mode 100644 index 0000000..3b68ba1 --- /dev/null +++ b/crates/argand-site-registry/src/diff.rs @@ -0,0 +1,174 @@ +// By Nic Weyand! +//! Streaming, typed comparison of evidence, decisions and derived registry state. + +use crate::query::Registry; +use rusqlite::{Connection, OptionalExtension}; +use serde_json::{Value, json}; +use std::io::Write; + +struct Table { + subject: &'static str, + scan: &'static str, + find: &'static str, +} + +const TABLES: &[Table] = &[ + Table { + subject: "source_selection", + scan: "SELECT s.id,json_object('source_id',s.id,'manifest',json(s.manifest)) FROM sources s JOIN selected_sources a USING(id) ORDER BY s.id", + find: "SELECT json_object('source_id',s.id,'manifest',json(s.manifest)) FROM sources s JOIN selected_sources a USING(id) WHERE s.id=?1", + }, + Table { + subject: "entity", + scan: "SELECT id,json_object('id',id,'canonical_name',canonical_name,'names_fingerprint',names_fingerprint) FROM entities ORDER BY id", + find: "SELECT json_object('id',id,'canonical_name',canonical_name,'names_fingerprint',names_fingerprint) FROM entities WHERE id=?1", + }, + Table { + subject: "name", + scan: "SELECT fact,json_object('entity',entity,'fact',fact,'key',key,'text',text,'language',language,'kind',kind) FROM names ORDER BY fact", + find: "SELECT json_object('entity',entity,'fact',fact,'key',key,'text',text,'language',language,'kind',kind) FROM names WHERE fact=?1", + }, + Table { + subject: "fact", + scan: "SELECT f.id,json_object('id',f.id,'source_id',f.source_id,'subject',f.subject,'predicate',f.predicate,'value',json(f.value),'selector',f.selector,'confidence',f.confidence) FROM facts f JOIN selected_sources s ON s.id=f.source_id ORDER BY f.id", + find: "SELECT json_object('id',f.id,'source_id',f.source_id,'subject',f.subject,'predicate',f.predicate,'value',json(f.value),'selector',f.selector,'confidence',f.confidence) FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.id=?1", + }, + Table { + subject: "property", + scan: "SELECT id,json_object('id',id,'url',url,'hostname',hostname,'domain',domain,'suffix',suffix,'derived',json(derived_json)) FROM properties ORDER BY id", + find: "SELECT json_object('id',id,'url',url,'hostname',hostname,'domain',domain,'suffix',suffix,'derived',json(derived_json)) FROM properties WHERE id=?1", + }, + Table { + subject: "edge", + scan: "SELECT fingerprint,json_object('fingerprint',fingerprint,'entity',entity,'property',property,'relation',relation,'facts',json(facts),'evidence',json(evidence),'eligible',json(eligible)) FROM edges ORDER BY fingerprint", + find: "SELECT json_object('fingerprint',fingerprint,'entity',entity,'property',property,'relation',relation,'facts',json(facts),'evidence',json(evidence),'eligible',json(eligible)) FROM edges WHERE fingerprint=?1", + }, + Table { + subject: "popularity", + scan: "SELECT fact,json_object('fact',fact,'source',source,'target',target,'hostname',hostname,'domain',domain,'value',json(value),'derived',json(derived_json)) FROM popularity ORDER BY fact", + find: "SELECT json_object('fact',fact,'source',source,'target',target,'hostname',hostname,'domain',domain,'value',json(value),'derived',json(derived_json)) FROM popularity WHERE fact=?1", + }, + Table { + subject: "rejected_fact", + scan: "SELECT fact,json_object('fact',fact,'reason',reason) FROM rejected ORDER BY fact", + find: "SELECT json_object('fact',fact,'reason',reason) FROM rejected WHERE fact=?1", + }, + Table { + subject: "review", + scan: "SELECT CAST(r.sequence AS TEXT),json_object('sequence',r.sequence,'fingerprint',r.fingerprint,'decision',r.decision,'reviewer',r.reviewer,'reason',r.reason,'evidence',r.evidence,'reviewed_at',r.reviewed_at,'expires_at',r.expires_at,'role',r.role,'locale',r.locale,'country',r.country,'authentication',CASE WHEN a.sequence IS NULL THEN NULL ELSE json_object('identity',a.signer,'signature_sha256',a.signature_sha256,'decision_sha256',a.decision_sha256,'namespace',a.namespace) END) FROM reviews r LEFT JOIN review_auth a USING(sequence) ORDER BY r.sequence", + find: "SELECT json_object('sequence',r.sequence,'fingerprint',r.fingerprint,'decision',r.decision,'reviewer',r.reviewer,'reason',r.reason,'evidence',r.evidence,'reviewed_at',r.reviewed_at,'expires_at',r.expires_at,'role',r.role,'locale',r.locale,'country',r.country,'authentication',CASE WHEN a.sequence IS NULL THEN NULL ELSE json_object('identity',a.signer,'signature_sha256',a.signature_sha256,'decision_sha256',a.decision_sha256,'namespace',a.namespace) END) FROM reviews r LEFT JOIN review_auth a USING(sequence) WHERE r.sequence=CAST(?1 AS INTEGER)", + }, + Table { + subject: "equivalence", + scan: "SELECT fingerprint,json_object('fingerprint',fingerprint,'left_entity',left_entity,'right_entity',right_entity,'left_signature',left_signature,'right_signature',right_signature) FROM equivalences ORDER BY fingerprint", + find: "SELECT json_object('fingerprint',fingerprint,'left_entity',left_entity,'right_entity',right_entity,'left_signature',left_signature,'right_signature',right_signature) FROM equivalences WHERE fingerprint=?1", + }, +]; + +/// Streams every material change between two authenticated generations. +/// +/// # Errors +/// Returns database, JSON, or output failures. +pub fn write(old: &Registry, new: &Registry, output: &mut dyn Write) -> anyhow::Result<()> { + writeln!( + output, + "{}", + json!({"schema":"argand.site-diff/v2","type":"header","old":old.identity,"new":new.identity}) + )?; + let mut changes = 0_u64; + for table in TABLES { + changes += removed_or_changed(table, &old.db, &new.db, output)?; + changes += added(table, &new.db, &old.db, output)?; + } + writeln!( + output, + "{}", + json!({"schema":"argand.site-diff/v2","type":"summary","changes":changes}) + )?; + Ok(()) +} + +fn removed_or_changed( + table: &Table, + from: &Connection, + to: &Connection, + output: &mut dyn Write, +) -> anyhow::Result { + let mut count = 0; + let mut scan = from.prepare(table.scan)?; + let mut find = to.prepare(table.find)?; + let rows = scan.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + for row in rows { + let (key, before) = row?; + let after: Option = find.query_row([&key], |row| row.get(0)).optional()?; + match after { + None => { + event(output, table.subject, "removed", &key, Some(&before), None)?; + count += 1; + } + Some(after) if after != before => { + event( + output, + table.subject, + "changed", + &key, + Some(&before), + Some(&after), + )?; + count += 1; + } + Some(_) => {} + } + } + Ok(count) +} + +fn added( + table: &Table, + from: &Connection, + to: &Connection, + output: &mut dyn Write, +) -> anyhow::Result { + let mut count = 0; + let mut scan = from.prepare(table.scan)?; + let mut find = to.prepare(table.find)?; + let rows = scan.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + for row in rows { + let (key, after) = row?; + let exists: bool = find + .query_row([&key], |_| Ok(true)) + .optional()? + .unwrap_or(false); + if !exists { + event(output, table.subject, "added", &key, None, Some(&after))?; + count += 1; + } + } + Ok(count) +} + +fn event( + output: &mut dyn Write, + subject: &str, + change: &str, + key: &str, + before: Option<&str>, + after: Option<&str>, +) -> anyhow::Result<()> { + let parse = |value: Option<&str>| -> anyhow::Result> { + value + .map(serde_json::from_str) + .transpose() + .map_err(Into::into) + }; + writeln!( + output, + "{}", + json!({"schema":"argand.site-diff/v2","type":"change","subject":subject,"change":change,"key":key,"before":parse(before)?,"after":parse(after)?}) + )?; + Ok(()) +} diff --git a/crates/argand-site-registry/src/evaluation.rs b/crates/argand-site-registry/src/evaluation.rs new file mode 100644 index 0000000..754979b --- /dev/null +++ b/crates/argand-site-registry/src/evaluation.rs @@ -0,0 +1,224 @@ +// By Nic Weyand! +//! Bounded, replayable destination-resolution evaluation. + +use crate::{ResolutionStatus, query::Registry}; +use anyhow::{Context, ensure}; +use serde::{Deserialize, Serialize}; +use std::{ + io::{BufRead, BufReader}, + path::Path, + time::Instant, +}; + +const MAXIMUM_INPUT_BYTES: usize = 16 * 1024 * 1024; +const MAXIMUM_LINE_BYTES: usize = 64 * 1024; + +/// One authored judgment against the native resolver. +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Case { + /// Stable case identifier. + pub id: String, + /// Exact entity name or alias query. + pub query: String, + /// Optional requested language/locale. + #[serde(default)] + pub locale: Option, + /// Optional requested two-letter country. + #[serde(default)] + pub country: Option, + /// Required resolver outcome. + pub expected_status: ResolutionStatus, + /// Optional stable entity assertion. + #[serde(default)] + pub expected_entity_id: Option, + /// Optional exact normalized destination assertion. + #[serde(default)] + pub expected_url: Option, +} + +/// A judgment that differed from the native result. +#[derive(Debug, Serialize)] +pub struct Failure { + /// Case identifier. + pub id: String, + /// Expected resolver status. + pub expected_status: ResolutionStatus, + /// Actual resolver status. + pub actual_status: ResolutionStatus, + /// Expected entity when one was asserted. + pub expected_entity_id: Option, + /// Actual selected entity, if any. + pub actual_entity_id: Option, + /// Expected exact URL when one was asserted. + pub expected_url: Option, + /// Actual selected URL, if any. + pub actual_url: Option, +} + +/// Aggregate correctness and native per-query latency evidence. +#[derive(Debug, Serialize)] +pub struct Report { + /// Versioned evaluation output contract. + pub schema: String, + /// Exact pinned generation under evaluation. + pub registry: String, + /// Explicit resolver clock used for every case. + pub evaluated_at: chrono::DateTime, + /// Parsed cases. + pub total: u64, + /// Cases matching every asserted field. + pub passed: u64, + /// Cases with at least one mismatch. + pub failed: u64, + /// Median native resolution time in microseconds. + pub latency_p50_us: u64, + /// 95th-percentile native resolution time in microseconds. + pub latency_p95_us: u64, + /// Complete mismatch list; bounded by `maximum_cases`. + pub failures: Vec, +} + +/// Evaluates newline-delimited cases against one already verified registry. +/// +/// # Errors +/// Rejects oversized/malformed inputs, duplicate IDs and unsafe case bounds. +pub fn run( + registry: &Registry, + input: impl BufRead, + maximum_cases: u32, + evaluated_at: chrono::DateTime, +) -> anyhow::Result { + ensure!( + (1..=100_000).contains(&maximum_cases), + "maximum cases must be 1..100000" + ); + let mut ids = std::collections::BTreeSet::new(); + let mut durations = Vec::new(); + let mut failures = Vec::new(); + let mut total = 0_u64; + for line in input.lines() { + let line = line?; + if line.trim().is_empty() { + continue; + } + ensure!( + line.len() <= MAXIMUM_LINE_BYTES, + "evaluation line exceeds 64 KiB" + ); + ensure!( + total < u64::from(maximum_cases), + "evaluation exceeds maximum cases" + ); + let case: Case = serde_json::from_value(crate::json::parse(line.as_bytes())?)?; + validate(&case)?; + ensure!(ids.insert(case.id.clone()), "duplicate evaluation case ID"); + total += 1; + let started = Instant::now(); + let actual = registry.resolve_explained( + &case.query, + case.locale.as_deref(), + case.country.as_deref(), + evaluated_at, + )?; + durations.push(u64::try_from(started.elapsed().as_micros())?); + let entity = actual + .destination + .as_ref() + .map(|value| value.entity_id.clone()); + let url = actual.destination.as_ref().map(|value| value.url.clone()); + if actual.status != case.expected_status + || case + .expected_entity_id + .as_ref() + .is_some_and(|value| Some(value) != entity.as_ref()) + || case + .expected_url + .as_ref() + .is_some_and(|value| Some(value) != url.as_ref()) + { + failures.push(Failure { + id: case.id, + expected_status: case.expected_status, + actual_status: actual.status, + expected_entity_id: case.expected_entity_id, + actual_entity_id: entity, + expected_url: case.expected_url, + actual_url: url, + }); + } + } + ensure!(total > 0, "evaluation contains no cases"); + durations.sort_unstable(); + let failed = u64::try_from(failures.len())?; + Ok(Report { + schema: "argand.site-evaluation/v1".into(), + registry: registry.identity.clone(), + evaluated_at, + total, + passed: total - failed, + failed, + latency_p50_us: percentile(&durations, 50), + latency_p95_us: percentile(&durations, 95), + failures, + }) +} + +/// Opens a bounded regular JSONL file and runs [`run`]. +/// +/// # Errors +/// Returns secure-open, input, or evaluation errors. +pub fn run_file( + registry: &Registry, + path: &Path, + maximum_cases: u32, + evaluated_at: chrono::DateTime, +) -> anyhow::Result { + let input = crate::ssh::sealed_input(path, MAXIMUM_INPUT_BYTES) + .with_context(|| format!("open evaluation {}", path.display()))?; + run( + registry, + BufReader::new(input.file), + maximum_cases, + evaluated_at, + ) +} + +fn validate(case: &Case) -> anyhow::Result<()> { + ensure!( + !case.id.trim().is_empty() && case.id.len() <= 256, + "evaluation case ID is required and bounded" + ); + ensure!( + !case.query.trim().is_empty() && case.query.len() <= 4096, + "evaluation query is required and bounded" + ); + ensure!( + case.locale.as_ref().is_none_or(|value| value.len() <= 64), + "evaluation locale is too long" + ); + ensure!( + case.country.as_ref().is_none_or(|value| { + value.len() == 2 && value.bytes().all(|byte| byte.is_ascii_uppercase()) + }), + "evaluation country must be uppercase two-letter code" + ); + ensure!( + case.expected_entity_id + .as_ref() + .is_none_or(|value| value.starts_with("argand:entity:") && value.len() <= 256), + "invalid expected entity ID" + ); + ensure!( + case.expected_url + .as_ref() + .is_none_or(|value| value.len() <= 8192), + "expected URL is too long" + ); + Ok(()) +} + +fn percentile(values: &[u64], percentile: usize) -> u64 { + let index = values.len().saturating_mul(percentile).div_ceil(100); + values[index.saturating_sub(1).min(values.len() - 1)] +} diff --git a/crates/argand-site-registry/src/generation.rs b/crates/argand-site-registry/src/generation.rs new file mode 100644 index 0000000..58c3b14 --- /dev/null +++ b/crates/argand-site-registry/src/generation.rs @@ -0,0 +1,168 @@ +// By Nic Weyand! +//! Exact immutable generation opening and descriptor-bound SQLite verification. + +use crate::{build::Receipt, query::Registry}; +use anyhow::ensure; +use rusqlite::Connection; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeSet, + fs::{File, OpenOptions}, + io::{Read, Seek}, + path::Path, +}; + +#[cfg(target_os = "linux")] +use std::os::{fd::AsRawFd, unix::fs::OpenOptionsExt}; + +impl Registry { + /// Opens only a complete, externally pinned generation. + /// + /// # Errors + /// Rejects altered receipts/databases and unsupported contracts. + pub fn open(path: &Path, expected_pin: &str) -> anyhow::Result { + ensure!( + crate::model::valid_digest(expected_pin), + "provide a full trusted receipt SHA-256" + ); + verify_layout(path)?; + let receipt_bytes = read_sealed(&path.join("COMPLETE.json"), 1024 * 1024)?; + ensure!( + crate::digest(&receipt_bytes) == expected_pin, + "receipt pin mismatch" + ); + let receipt: Receipt = serde_json::from_value(crate::json::parse(&receipt_bytes)?)?; + let licenses = read_sealed(&path.join("LICENSE_SOURCES.md"), 1024 * 1024)?; + let attribution = read_sealed(&path.join("ATTRIBUTION.json"), 1024 * 1024)?; + ensure!( + crate::digest(&licenses) == receipt.licenses_sha256 + && crate::digest(&attribution) == receipt.attribution_sha256, + "registry license or attribution digest mismatch" + ); + ensure!( + receipt.schema == "argand.site-registry/v1" + && crate::store::supported_rule_version(&receipt.rules), + "unsupported registry contract" + ); + let database = path.join("registry.sqlite"); + let (db, database) = open_authenticated_database(&database, &receipt.database_sha256)?; + crate::store::configure(&db)?; + Ok(Self { + db, + _database: database, + identity: expected_pin.into(), + receipt, + }) + } +} + +fn verify_layout(path: &Path) -> anyhow::Result<()> { + ensure!( + std::fs::symlink_metadata(path)?.is_dir(), + "generation must be a real directory" + ); + let allowed = BTreeSet::from([ + "ATTRIBUTION.json", + "COMPLETE.json", + "COMPLETE.json.sig", + "LICENSE_SOURCES.md", + "registry.sqlite", + ]); + let mut found = BTreeSet::new(); + for entry in std::fs::read_dir(path)? { + let entry = entry?; + let name = entry + .file_name() + .into_string() + .map_err(|_| anyhow::anyhow!("non-UTF8 generation entry"))?; + ensure!( + allowed.contains(name.as_str()), + "unexpected generation entry: {name}" + ); + ensure!( + entry.file_type()?.is_file(), + "generation entries must be regular files" + ); + found.insert(name); + } + for required in [ + "ATTRIBUTION.json", + "COMPLETE.json", + "LICENSE_SOURCES.md", + "registry.sqlite", + ] { + ensure!(found.contains(required), "generation is missing {required}"); + } + Ok(()) +} + +fn read_sealed(path: &Path, maximum: usize) -> anyhow::Result> { + let mut file = open_no_follow(path)?; + ensure!( + file.metadata()?.is_file(), + "generation artifact must be a regular file" + ); + let mut bytes = Vec::new(); + file.by_ref() + .take(u64::try_from(maximum)? + 1) + .read_to_end(&mut bytes)?; + ensure!(bytes.len() <= maximum, "generation metadata exceeds 1 MiB"); + Ok(bytes) +} + +#[cfg(target_os = "linux")] +pub(crate) fn open_no_follow(path: &Path) -> anyhow::Result { + Ok( + OpenOptions::new() // atomic-writes: allow read-only no-follow descriptor open + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path)?, + ) +} + +#[cfg(not(target_os = "linux"))] +pub(crate) fn open_no_follow(_path: &Path) -> anyhow::Result { + anyhow::bail!("secure immutable generation reads currently require Linux") +} + +#[cfg(target_os = "linux")] +fn open_authenticated_database(path: &Path, expected: &str) -> anyhow::Result<(Connection, File)> { + let mut file = open_no_follow(path)?; + ensure!( + file.metadata()?.is_file(), + "database must be a regular file" + ); + let mut hash = Sha256::new(); + let mut buffer = [0; 8192]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + hash.update(&buffer[..count]); + } + ensure!( + format!("{:x}", hash.finalize()) == expected, + "registry database digest mismatch" + ); + file.rewind()?; + let descriptor = format!("/proc/self/fd/{}", file.as_raw_fd()); + let mut uri = url::Url::from_file_path(descriptor) + .map_err(|()| anyhow::anyhow!("cannot construct immutable SQLite URI"))?; + uri.query_pairs_mut() + .append_pair("mode", "ro") + .append_pair("immutable", "1"); + let flags = rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY + | rusqlite::OpenFlags::SQLITE_OPEN_URI + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX; + let db = Connection::open_with_flags(uri.as_str(), flags)?; + Ok((db, file)) +} + +#[cfg(not(target_os = "linux"))] +fn open_authenticated_database( + _path: &Path, + _expected: &str, +) -> anyhow::Result<(Connection, File)> { + anyhow::bail!("secure immutable generation reads currently require Linux") +} diff --git a/crates/argand-site-registry/src/identity.rs b/crates/argand-site-registry/src/identity.rs index c87e520..31de350 100644 --- a/crates/argand-site-registry/src/identity.rs +++ b/crates/argand-site-registry/src/identity.rs @@ -69,6 +69,32 @@ pub fn record( registry: &Registry, pair: &Equivalence, review: &Review, +) -> anyhow::Result { + record_inner(db, registry, pair, review, None) +} + +/// Appends a cryptographically authenticated identity decision. +/// +/// # Errors +/// Rejects stale evidence, invalid scope, missing source identities, or proof +/// that does not match the decision's reviewer. +pub fn record_authenticated( + db: &Connection, + registry: &Registry, + pair: &Equivalence, + review: &Review, + authentication: &crate::review::Authentication, +) -> anyhow::Result { + crate::review::validate_authentication(review, authentication)?; + record_inner(db, registry, pair, review, Some(authentication)) +} + +fn record_inner( + db: &Connection, + registry: &Registry, + pair: &Equivalence, + review: &Review, + authentication: Option<&crate::review::Authentication>, ) -> anyhow::Result { crate::review::validate(review)?; let expected = propose(registry, &pair.entities[0], &pair.entities[1])?; @@ -98,6 +124,9 @@ pub fn record( ], )?; let sequence = crate::review::append(db, review)?; + if let Some(authentication) = authentication { + crate::review::append_authentication(db, sequence, authentication)?; + } transaction.commit()?; Ok(sequence) } @@ -181,11 +210,25 @@ pub(crate) fn expand( Ok(Some((entities, evidence.into_values().collect()))) } -pub(crate) fn candidates( +pub(crate) enum CandidateSearch { + Ready { + matched_entities: u64, + candidates: Vec, + }, + NoMatch, + AmbiguousIdentity { + matched_entities: u64, + }, + SafetyLimitExceeded { + matched_entities: u64, + }, +} + +pub(crate) fn candidate_search( registry: &Registry, query: &str, now: DateTime, -) -> anyhow::Result> { +) -> anyhow::Result { let key = crate::normalize::name_key(query)?; let mut statement = registry .db @@ -193,14 +236,23 @@ pub(crate) fn candidates( let matched = statement .query_map([key], |r| r.get::<_, String>(0))? .collect::, _>>()?; - if matched.is_empty() || matched.len() > 64 { - return Ok(Vec::new()); + if matched.is_empty() { + return Ok(CandidateSearch::NoMatch); + } + if matched.len() > 64 { + return Ok(CandidateSearch::SafetyLimitExceeded { + matched_entities: u64::try_from(matched.len())?, + }); } let Some((entities, evidence)) = expand(registry, &matched[0], now)? else { - return Ok(Vec::new()); + return Ok(CandidateSearch::SafetyLimitExceeded { + matched_entities: u64::try_from(matched.len())?, + }); }; if matched.iter().any(|id| !entities.contains(id)) { - return Ok(Vec::new()); + return Ok(CandidateSearch::AmbiguousIdentity { + matched_entities: u64::try_from(matched.len())?, + }); } let mut output = Vec::new(); for entity in entities { @@ -212,9 +264,14 @@ pub(crate) fn candidates( candidate.identity_provenance.clone_from(&evidence); output.push(candidate); if output.len() > 100 { - return Ok(Vec::new()); + return Ok(CandidateSearch::SafetyLimitExceeded { + matched_entities: u64::try_from(matched.len())?, + }); } } } - Ok(output) + Ok(CandidateSearch::Ready { + matched_entities: u64::try_from(matched.len())?, + candidates: output, + }) } diff --git a/crates/argand-site-registry/src/lib.rs b/crates/argand-site-registry/src/lib.rs index d327d77..a186dc5 100644 --- a/crates/argand-site-registry/src/lib.rs +++ b/crates/argand-site-registry/src/lib.rs @@ -3,9 +3,13 @@ pub mod adapters; pub mod build; +pub mod catalog; pub mod crux; +pub mod diff; pub mod download; +pub mod evaluation; pub mod evidence; +mod generation; pub mod identity; mod json; pub mod model; @@ -13,8 +17,12 @@ pub mod normalize; pub mod observation; pub mod query; pub mod release; +mod resolution; pub mod review; +mod ssh; pub mod store; + +pub use resolution::{Resolution, ResolutionCounts, ResolutionStatus}; pub mod update; use sha2::{Digest, Sha256}; diff --git a/crates/argand-site-registry/src/observation.rs b/crates/argand-site-registry/src/observation.rs index c3c5409..f345f2c 100644 --- a/crates/argand-site-registry/src/observation.rs +++ b/crates/argand-site-registry/src/observation.rs @@ -58,3 +58,130 @@ pub struct Observation { /// Confidence on the same 0..10000 policy scale as source facts. pub confidence: u16, } + +/// Deterministic crawler evidence prepared for a later rights-reviewed adapter. +#[derive(Clone, Debug, Serialize)] +pub struct NormalizedObservation { + /// Versioned extension contract. + pub schema: String, + /// Stable fingerprint over all normalized evidence fields. + pub fingerprint: String, + /// Observed relationship without any ownership inference. + pub relation: ObservationKind, + /// Strict normalized source page. + pub from: crate::normalize::WebProperty, + /// Strict normalized target page. + pub to: crate::normalize::WebProperty, + /// Source provider/capture collection. + pub source: String, + /// Immutable source-native capture identifier. + pub source_identifier: String, + /// Rights declaration retained with the observation. + pub license: String, + /// Rights evidence URL retained with the observation. + pub license_url: String, + /// Retrieval instant. + pub retrieved_at: DateTime, + /// Captured source-content hash. + pub content_sha256: String, + /// Exact assertion locator. + pub selector: String, + /// Source confidence, separate from destination review. + pub confidence: u16, +} + +impl Observation { + /// Validates and normalizes an observation without admitting it as ownership. + /// + /// # Errors + /// Rejects malformed evidence coordinates, URL values and relation scopes. + pub fn normalize( + &self, + normalizer: &crate::normalize::Normalizer, + ) -> anyhow::Result { + use anyhow::ensure; + ensure!( + !self.source.trim().is_empty() + && self.source.len() <= 128 + && !self.source_identifier.trim().is_empty() + && self.source_identifier.len() <= 1024, + "observation source coordinates are required and bounded" + ); + ensure!( + !self.license.trim().is_empty() + && self.license.len() <= 128 + && self.license_url.len() <= 2048, + "observation rights declaration is required and bounded" + ); + let license_url = url::Url::parse(&self.license_url)?; + ensure!( + license_url.scheme() == "https" && license_url.host_str().is_some(), + "observation license evidence must be an HTTPS URL" + ); + ensure!( + crate::model::valid_digest(&self.content_sha256), + "invalid observation content digest" + ); + ensure!( + !self.selector.trim().is_empty() && self.selector.len() <= 2048, + "observation selector is required and bounded" + ); + ensure!(self.confidence <= 10_000, "invalid observation confidence"); + validate_relation(&self.relation)?; + let from = normalizer.url(&self.from_url)?; + let to = normalizer.url(&self.to_url)?; + let evidence = ( + "argand.site-observation/v1", + &self.relation, + &from, + &to, + &self.source, + &self.source_identifier, + &self.license, + &self.license_url, + self.retrieved_at, + &self.content_sha256, + &self.selector, + self.confidence, + ); + Ok(NormalizedObservation { + schema: "argand.site-observation/v1".into(), + fingerprint: crate::digest(&serde_json::to_vec(&evidence)?), + relation: self.relation.clone(), + from, + to, + source: self.source.clone(), + source_identifier: self.source_identifier.clone(), + license: self.license.clone(), + license_url: self.license_url.clone(), + retrieved_at: self.retrieved_at, + content_sha256: self.content_sha256.clone(), + selector: self.selector.clone(), + confidence: self.confidence, + }) + } +} + +fn validate_relation(relation: &ObservationKind) -> anyhow::Result<()> { + use anyhow::ensure; + match relation { + ObservationKind::Redirect { status } => ensure!( + matches!(status, 301 | 302 | 303 | 307 | 308), + "invalid HTTP redirect status" + ), + ObservationKind::Hreflang { locale } => ensure!( + !locale.is_empty() + && locale.len() <= 64 + && locale + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-'), + "invalid hreflang locale" + ), + ObservationKind::CountrySelector { country } => ensure!( + country.len() == 2 && country.bytes().all(|byte| byte.is_ascii_uppercase()), + "invalid country selector scope" + ), + ObservationKind::Canonical | ObservationKind::JsonLdSameAs | ObservationKind::Sitemap => {} + } + Ok(()) +} diff --git a/crates/argand-site-registry/src/query.rs b/crates/argand-site-registry/src/query.rs index d31f298..48bb0ac 100644 --- a/crates/argand-site-registry/src/query.rs +++ b/crates/argand-site-registry/src/query.rs @@ -2,16 +2,19 @@ //! Bounded native lookup; ambiguity is counted before limits or policy filtering. use crate::{build::Receipt, normalize::name_key}; -use anyhow::{Context, ensure}; -use chrono::{DateTime, Utc}; +use anyhow::ensure; use rusqlite::{Connection, OptionalExtension, params}; use serde::Serialize; use serde_json::{Value, json}; -use std::path::Path; +use std::fs::File; + +pub use crate::resolution::{Resolution, ResolutionCounts, ResolutionStatus}; /// Open verified generation. Hashes are checked once, outside the query path. pub struct Registry { pub(crate) db: Connection, + // Keep the authenticated inode alive for the SQLite /proc/self/fd reader. + pub(crate) _database: File, /// External receipt pin supplied by the caller. pub identity: String, /// Verified manifest. @@ -138,49 +141,6 @@ impl Registry { }) } - /// Opens only a complete, externally pinned generation. - /// - /// # Errors - /// Rejects altered receipts/databases and unsupported contracts. - pub fn open(path: &Path, expected_pin: &str) -> anyhow::Result { - ensure!( - crate::model::valid_digest(expected_pin), - "provide a full trusted receipt SHA-256" - ); - ensure!( - crate::file_digest(&path.join("COMPLETE.json"))? == expected_pin, - "receipt pin mismatch" - ); - let receipt: Receipt = crate::read_json(&path.join("COMPLETE.json"))?; - ensure!( - crate::file_digest(&path.join("LICENSE_SOURCES.md"))? == receipt.licenses_sha256 - && crate::file_digest(&path.join("ATTRIBUTION.json"))? - == receipt.attribution_sha256, - "registry license or attribution digest mismatch" - ); - ensure!( - receipt.schema == "argand.site-registry/v1" - && receipt.rules == crate::store::RULE_VERSION, - "unsupported registry contract" - ); - let database = path.join("registry.sqlite"); - ensure!( - std::fs::symlink_metadata(&database)?.is_file(), - "database must be a regular file" - ); - ensure!( - crate::file_digest(&database)? == receipt.database_sha256, - "registry database digest mismatch" - ); - let db = Connection::open_with_flags(database, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?; - crate::store::configure(&db)?; - Ok(Self { - db, - identity: expected_pin.into(), - receipt, - }) - } - /// Indexed exact-name/alias lookup, with counts independent of result limits. /// /// # Errors @@ -227,7 +187,11 @@ impl Registry { for id in ids { provenance.push(crate::evidence::fact(&self.db, &id)?); } - let review=self.db.query_row("SELECT sequence,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country FROM reviews WHERE fingerprint=?1 ORDER BY sequence DESC LIMIT 1",[fingerprint],|r|Ok(json!({"sequence":crate::store::unsigned(r,0)?,"decision":r.get::<_,String>(1)?,"reviewer":r.get::<_,String>(2)?,"reason":r.get::<_,String>(3)?,"evidence":r.get::<_,String>(4)?,"retrieved_at":r.get::<_,String>(5)?,"expires_at":r.get::<_,String>(6)?,"role":r.get::<_,String>(7)?,"locale":r.get::<_,String>(8)?,"country":r.get::<_,String>(9)?,"source":"argand_operator_review","source_identifier":format!("{}:{}",fingerprint,crate::store::unsigned(r,0)?),"license":"CC0-1.0","license_url":"https://creativecommons.org/publicdomain/zero/1.0/","confidence":9000}))).optional()?; + let review=self.db.query_row("SELECT r.sequence,r.decision,r.reviewer,r.reason,r.evidence,r.reviewed_at,r.expires_at,r.role,r.locale,r.country,a.signer,a.signature_sha256,a.namespace FROM reviews r LEFT JOIN review_auth a ON a.sequence=r.sequence WHERE r.fingerprint=?1 ORDER BY r.sequence DESC LIMIT 1",[fingerprint],|r|{ + let sequence=crate::store::unsigned(r,0)?; + let signer:Option=r.get(10)?; + Ok(json!({"sequence":sequence,"decision":r.get::<_,String>(1)?,"reviewer":r.get::<_,String>(2)?,"reason":r.get::<_,String>(3)?,"evidence":r.get::<_,String>(4)?,"retrieved_at":r.get::<_,String>(5)?,"expires_at":r.get::<_,String>(6)?,"role":r.get::<_,String>(7)?,"locale":r.get::<_,String>(8)?,"country":r.get::<_,String>(9)?,"authentication":signer.map(|identity|json!({"identity":identity,"signature_sha256":r.get::<_,String>(11).unwrap_or_default(),"namespace":r.get::<_,String>(12).unwrap_or_default()})),"source":"argand_operator_review","source_identifier":format!("{}:{sequence}",fingerprint),"license":"CC0-1.0","license_url":"https://creativecommons.org/publicdomain/zero/1.0/","confidence":9000})) + }).optional()?; Ok(Candidate { identity_provenance: Vec::new(), entity: crate::evidence::entity(&self.db, &entity, &name)?, @@ -255,78 +219,4 @@ impl Registry { eligible, }) } - - /// Chooses an approved regional property or an explicitly reviewed primary. - /// Never returns a destination for ambiguous entities or tied properties. - /// - /// # Errors - /// Returns malformed requests/data or database failures. - pub fn resolve( - &self, - query: &str, - locale: Option<&str>, - country: Option<&str>, - now: DateTime, - ) -> anyhow::Result> { - let candidates = crate::identity::candidates(self, query, now)?; - let mut best = None; - let mut score = 0; - let mut ambiguous = false; - for candidate in candidates { - if !candidate.eligible { - continue; - } - let Some(review) = candidate.review.as_ref() else { - continue; - }; - if review["decision"] != "approve" { - continue; - } - let expires = DateTime::parse_from_rfc3339( - review["expires_at"] - .as_str() - .context("invalid review expiry")?, - )?; - let starts = DateTime::parse_from_rfc3339( - review["retrieved_at"] - .as_str() - .context("invalid review timestamp")?, - )?; - if now < starts || now >= expires { - continue; - } - let region = review["country"].as_str().unwrap_or_default(); - let language = review["locale"].as_str().unwrap_or_default(); - let matches_country = - !region.is_empty() && country.is_some_and(|c| c.eq_ignore_ascii_case(region)); - let matches_locale = - !language.is_empty() && locale.is_some_and(|l| l.eq_ignore_ascii_case(language)); - let current = if review["role"] == "regional" { - // All asserted dimensions must match; a language alone cannot - // override an explicit country mismatch. - if (!region.is_empty() && !matches_country) - || (!language.is_empty() && !matches_locale) - { - continue; - } - 2 + u8::from(matches_country) + u8::from(matches_locale) - } else if review["role"] == "primary" { - 1 - } else { - continue; - }; - if current > score { - best = Some(candidate); - score = current; - ambiguous = false; - } else if current == score - && best - .as_ref() - .is_some_and(|b: &Candidate| b.url != candidate.url) - { - ambiguous = true; - } - } - Ok(if ambiguous { None } else { best }) - } } diff --git a/crates/argand-site-registry/src/release.rs b/crates/argand-site-registry/src/release.rs index ad15b4f..62400b4 100644 --- a/crates/argand-site-registry/src/release.rs +++ b/crates/argand-site-registry/src/release.rs @@ -8,7 +8,7 @@ use std::{ fs::{self, File, OpenOptions}, io::{BufWriter, Write}, path::Path, - process::{Command, Stdio}, + process::Command, }; /// Source terms shipped and authenticated with every generation. @@ -80,8 +80,14 @@ fn export_inner( /// /// # Errors /// Returns missing key, existing signature, and signing process failures. -pub fn sign(generation: &Path, key: &Path, pin: &str) -> anyhow::Result<()> { - Registry::open(generation, pin)?; +pub fn sign( + generation: &Path, + key: &Path, + pin: &str, + allowed_reviewers: &Path, +) -> anyhow::Result<()> { + let registry = Registry::open(generation, pin)?; + crate::review::verify_all(®istry.db, allowed_reviewers)?; ensure!( !generation.join("COMPLETE.json.sig").exists(), "signature already exists" @@ -106,19 +112,20 @@ pub fn verify_signed( signers: &Path, identity: &str, ) -> anyhow::Result { - let pin = crate::file_digest(&generation.join("COMPLETE.json"))?; - let status = Command::new("ssh-keygen") - .args(["-Y", "verify", "-n", "argand-site-registry", "-f"]) - .arg(signers) - .arg("-I") - .arg(identity) - .arg("-s") - .arg(generation.join("COMPLETE.json.sig")) - .stdin(Stdio::from(File::open(generation.join("COMPLETE.json"))?)) - .stdout(Stdio::null()) - .status()?; - ensure!(status.success(), "untrusted registry signature"); - Registry::open(generation, &pin) + let receipt = crate::ssh::sealed_input(&generation.join("COMPLETE.json"), 1024 * 1024)?; + let signature = crate::ssh::sealed_input(&generation.join("COMPLETE.json.sig"), 64 * 1024)?; + let signers = crate::ssh::sealed_input(signers, 1024 * 1024)?; + let pin = crate::digest(&receipt.bytes); + crate::ssh::verify( + "argand-site-registry", + &receipt.bytes, + &signature.bytes, + &signers.bytes, + identity, + )?; + let registry = Registry::open(generation, &pin)?; + crate::review::ensure_all_authenticated(®istry.db)?; + Ok(registry) } /// Activates a verified generation using one durable pointer. Refuses rollback @@ -212,26 +219,5 @@ fn preserve_revocations(old: &Registry, new: &Registry) -> anyhow::Result<()> { /// # Errors /// Returns query or output errors. pub fn diff(old: &Registry, new: &Registry, output: &mut dyn Write) -> anyhow::Result<()> { - for (kind, from, to) in [("removed", old, new), ("added", new, old)] { - let mut stmt = from - .db - .prepare("SELECT fingerprint,entity,property FROM edges ORDER BY fingerprint")?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let fingerprint: String = row.get(0)?; - let exists: bool = to.db.query_row( - "SELECT EXISTS(SELECT 1 FROM edges WHERE fingerprint=?1)", - [&fingerprint], - |r| r.get(0), - )?; - if !exists { - writeln!( - output, - "{}", - json!({"change":kind,"fingerprint":fingerprint,"entity":row.get::<_,String>(1)?,"property":row.get::<_,String>(2)?}) - )?; - } - } - } - Ok(()) + crate::diff::write(old, new, output) } diff --git a/crates/argand-site-registry/src/resolution.rs b/crates/argand-site-registry/src/resolution.rs new file mode 100644 index 0000000..f54ae09 --- /dev/null +++ b/crates/argand-site-registry/src/resolution.rs @@ -0,0 +1,238 @@ +// By Nic Weyand! +//! Conservative destination selection with machine-readable abstention reasons. + +use crate::{ + normalize::name_key, + query::{Candidate, Registry}, +}; +use anyhow::Context; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// Explainable outcome from conservative destination resolution. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResolutionStatus { + /// One reviewed destination was selected. + Resolved, + /// No entity name or alias matched exactly after normalization. + NoNameMatch, + /// Matching source entities lack an active reviewed equivalence chain. + AmbiguousIdentity, + /// Identity or edge expansion exceeded defensive bounds. + SafetyLimitExceeded, + /// Matching entities have no currently eligible website assertion. + NoEligibleDestination, + /// Eligible assertions exist but none has a current approval. + NoActiveReview, + /// Active regional approvals do not match the requested scope. + RegionMismatch, + /// Equally specific active approvals point to different URLs. + AmbiguousDestination, +} + +/// Counts explaining why resolution selected or rejected candidates. +#[derive(Clone, Debug, Default, Serialize)] +pub struct ResolutionCounts { + /// Source entities matching the normalized name or alias. + pub matched_entities: u64, + /// Assertion candidates examined after identity review. + pub considered: u64, + /// Candidates eligible for destination review. + pub eligible: u64, + /// Eligible candidates without any review entry. + pub missing_review: u64, + /// Candidates whose latest decision is a revocation. + pub revoked: u64, + /// Approvals whose validity interval has ended. + pub expired: u64, + /// Approvals whose validity interval has not begun. + pub not_yet_valid: u64, + /// Current approvals that do not match requested locale/country. + pub region_mismatch: u64, + /// Current approvals considered for final selection. + pub active_approvals: u64, +} + +/// Resolution envelope; a null destination always has a machine-readable reason. +#[derive(Debug, Serialize)] +pub struct Resolution { + /// Normalized query key. + pub query: String, + /// Selection outcome. + pub status: ResolutionStatus, + /// Reviewed destination, present only for [`ResolutionStatus::Resolved`]. + pub destination: Option, + /// Complete bounded decision counts. + pub counts: ResolutionCounts, + /// Source attribution required when displaying an imported name. + pub attribution: serde_json::Value, +} + +impl Registry { + /// Chooses an approved regional property or an explicitly reviewed primary. + /// Never returns a destination for ambiguous entities or tied properties. + /// + /// # Errors + /// Returns malformed requests/data or database failures. + pub fn resolve( + &self, + query: &str, + locale: Option<&str>, + country: Option<&str>, + now: DateTime, + ) -> anyhow::Result> { + Ok(self + .resolve_explained(query, locale, country, now)? + .destination) + } + + /// Resolves a destination and explains every conservative abstention. + /// + /// # Errors + /// Returns malformed requests/data or database failures. + pub fn resolve_explained( + &self, + query: &str, + locale: Option<&str>, + country: Option<&str>, + now: DateTime, + ) -> anyhow::Result { + let key = name_key(query)?; + let mut counts = ResolutionCounts::default(); + let candidates = match crate::identity::candidate_search(self, query, now)? { + crate::identity::CandidateSearch::Ready { + matched_entities, + candidates, + } => { + counts.matched_entities = matched_entities; + candidates + } + crate::identity::CandidateSearch::NoMatch => { + return Ok(result(key, ResolutionStatus::NoNameMatch, None, counts)); + } + crate::identity::CandidateSearch::AmbiguousIdentity { matched_entities } => { + counts.matched_entities = matched_entities; + return Ok(result( + key, + ResolutionStatus::AmbiguousIdentity, + None, + counts, + )); + } + crate::identity::CandidateSearch::SafetyLimitExceeded { matched_entities } => { + counts.matched_entities = matched_entities; + return Ok(result( + key, + ResolutionStatus::SafetyLimitExceeded, + None, + counts, + )); + } + }; + let (status, destination) = choose(candidates, locale, country, now, &mut counts)?; + Ok(result(key, status, destination, counts)) + } +} + +fn choose( + candidates: Vec, + locale: Option<&str>, + country: Option<&str>, + now: DateTime, + counts: &mut ResolutionCounts, +) -> anyhow::Result<(ResolutionStatus, Option)> { + let mut best = None; + let mut score = 0; + let mut ambiguous = false; + for candidate in candidates { + counts.considered += 1; + if !candidate.eligible { + continue; + } + counts.eligible += 1; + let Some(review) = candidate.review.as_ref() else { + counts.missing_review += 1; + continue; + }; + if review["decision"] != "approve" { + counts.revoked += 1; + continue; + } + let expires = DateTime::parse_from_rfc3339( + review["expires_at"] + .as_str() + .context("invalid review expiry")?, + )?; + let starts = DateTime::parse_from_rfc3339( + review["retrieved_at"] + .as_str() + .context("invalid review timestamp")?, + )?; + if now < starts || now >= expires { + if now < starts { + counts.not_yet_valid += 1; + } else { + counts.expired += 1; + } + continue; + } + let region = review["country"].as_str().unwrap_or_default(); + let language = review["locale"].as_str().unwrap_or_default(); + let matches_country = + !region.is_empty() && country.is_some_and(|c| c.eq_ignore_ascii_case(region)); + let matches_locale = + !language.is_empty() && locale.is_some_and(|l| l.eq_ignore_ascii_case(language)); + let current = if review["role"] == "regional" { + if (!region.is_empty() && !matches_country) || (!language.is_empty() && !matches_locale) + { + counts.region_mismatch += 1; + continue; + } + 2 + u8::from(matches_country) + u8::from(matches_locale) + } else if review["role"] == "primary" { + 1 + } else { + counts.region_mismatch += 1; + continue; + }; + counts.active_approvals += 1; + if current > score { + best = Some(candidate); + score = current; + ambiguous = false; + } else if current == score + && best + .as_ref() + .is_some_and(|b: &Candidate| b.url != candidate.url) + { + ambiguous = true; + } + } + Ok(if ambiguous { + (ResolutionStatus::AmbiguousDestination, None) + } else if let Some(candidate) = best { + (ResolutionStatus::Resolved, Some(candidate)) + } else if counts.eligible == 0 { + (ResolutionStatus::NoEligibleDestination, None) + } else if counts.region_mismatch > 0 { + (ResolutionStatus::RegionMismatch, None) + } else { + (ResolutionStatus::NoActiveReview, None) + }) +} + +fn result( + query: String, + status: ResolutionStatus, + destination: Option, + counts: ResolutionCounts, +) -> Resolution { + Resolution { + query, + status, + destination, + counts, + attribution: crate::release::attribution(), + } +} diff --git a/crates/argand-site-registry/src/review.rs b/crates/argand-site-registry/src/review.rs index 8480ebd..8c7823a 100644 --- a/crates/argand-site-registry/src/review.rs +++ b/crates/argand-site-registry/src/review.rs @@ -6,9 +6,13 @@ use anyhow::ensure; use chrono::{DateTime, Utc}; use rusqlite::{Connection, params}; use serde::{Deserialize, Serialize}; +use std::path::Path; + +/// SSH signature namespace for exact operator decision JSON bytes. +pub const SIGNATURE_NAMESPACE: &str = "argand-site-registry-review"; /// Explicit review input, independent of imported source confidence. -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] #[serde(deny_unknown_fields)] pub struct Review { /// Exact fingerprint printed by pinned lookup. @@ -33,12 +37,95 @@ pub struct Review { pub country: String, } +/// Authentication attached to one immutable review-log entry. +#[derive(Clone, Debug)] +pub struct Authentication { + /// Identity selected from the external allowed-reviewers file. + pub signer: String, + /// SHA-256 of the detached SSH signature bytes. + pub signature_sha256: String, + /// Domain-separated SSH signature namespace. + pub namespace: String, + /// SHA-256 of the exact signed decision JSON. + pub decision_sha256: String, + decision_json: Vec, + signature: Vec, +} + +/// Verifies exact decision bytes against an independently supplied reviewer key. +/// +/// # Errors +/// Rejects oversized/non-regular inputs, malformed decisions, identity mismatch, +/// or an untrusted detached SSH signature. +pub fn authenticate( + decision: &Path, + signature: &Path, + allowed_reviewers: &Path, + identity: &str, +) -> anyhow::Result<(Review, Authentication)> { + ensure!( + !identity.trim().is_empty() && identity.len() <= 256, + "reviewer identity is required and bounded" + ); + let decision = crate::ssh::sealed_input(decision, 1024 * 1024)?; + let signature = crate::ssh::sealed_input(signature, 64 * 1024)?; + let allowed_reviewers = crate::ssh::sealed_input(allowed_reviewers, 1024 * 1024)?; + let review: Review = serde_json::from_value(crate::json::parse(&decision.bytes)?)?; + ensure!( + review.reviewer == identity, + "reviewer must equal the authenticated identity" + ); + crate::ssh::verify( + SIGNATURE_NAMESPACE, + &decision.bytes, + &signature.bytes, + &allowed_reviewers.bytes, + identity, + )?; + let signature_sha256 = crate::digest(&signature.bytes); + Ok(( + review, + Authentication { + signer: identity.into(), + signature_sha256, + namespace: SIGNATURE_NAMESPACE.into(), + decision_sha256: crate::digest(&decision.bytes), + decision_json: decision.bytes, + signature: signature.bytes, + }, + )) +} + +/// Records a verified review and its authentication atomically. +/// +/// # Errors +/// Returns the same validation failures as [`record`] and rejects invalid proof. +pub fn record_authenticated( + db: &Connection, + registry: &Registry, + review: &Review, + authentication: &Authentication, +) -> anyhow::Result { + validate_authentication(review, authentication)?; + let transaction = db.unchecked_transaction()?; + validate_candidate(db, registry, review)?; + let sequence = append(db, review)?; + append_authentication(db, sequence, authentication)?; + transaction.commit()?; + Ok(sequence) +} + /// Appends a decision after verifying the exact generation and fingerprint. /// Rebuild and promote to distribute it; existing immutable artifacts never mutate. /// /// # Errors /// Rejects missing evidence, expired/overlong approval, and ineligible assertions. pub fn record(db: &Connection, registry: &Registry, review: &Review) -> anyhow::Result { + validate_candidate(db, registry, review)?; + append(db, review) +} + +fn validate_candidate(db: &Connection, registry: &Registry, review: &Review) -> anyhow::Result<()> { validate(review)?; let candidate = registry.candidate(&review.fingerprint)?; if review.decision == "approve" { @@ -57,7 +144,7 @@ pub fn record(db: &Connection, registry: &Registry, review: &Review) -> anyhow:: )?; ensure!(exists, "review source is absent from this store"); } - append(db, review) + Ok(()) } pub(crate) fn validate(review: &Review) -> anyhow::Result<()> { @@ -114,3 +201,117 @@ pub(crate) fn append(db: &Connection, review: &Review) -> anyhow::Result { db.execute("INSERT INTO reviews(fingerprint,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",params![review.fingerprint,review.decision,review.reviewer,review.reason,review.evidence,review.reviewed_at.to_rfc3339(),review.expires_at.to_rfc3339(),review.role,review.locale,review.country])?; Ok(u64::try_from(db.last_insert_rowid())?) } + +pub(crate) fn validate_authentication( + review: &Review, + authentication: &Authentication, +) -> anyhow::Result<()> { + validate(review)?; + ensure!( + authentication.decision_json.len() <= 1024 * 1024 + && authentication.signature.len() <= 64 * 1024, + "review authentication proof exceeds its bound" + ); + ensure!( + review.reviewer == authentication.signer + && authentication.namespace == SIGNATURE_NAMESPACE + && crate::model::valid_digest(&authentication.signature_sha256) + && authentication.signature_sha256 == crate::digest(&authentication.signature) + && crate::model::valid_digest(&authentication.decision_sha256) + && authentication.decision_sha256 == crate::digest(&authentication.decision_json), + "review authentication does not match the decision" + ); + let signed: Review = + serde_json::from_value(crate::json::parse(&authentication.decision_json)?)?; + ensure!( + &signed == review, + "signed decision differs from stored review" + ); + Ok(()) +} + +pub(crate) fn append_authentication( + db: &Connection, + sequence: u64, + authentication: &Authentication, +) -> anyhow::Result<()> { + db.execute( + "INSERT INTO review_auth VALUES(?1,?2,?3,?4,?5,?6,?7)", + params![ + i64::try_from(sequence)?, + authentication.signer, + authentication.signature_sha256, + authentication.namespace, + authentication.decision_sha256, + authentication.decision_json, + authentication.signature + ], + )?; + Ok(()) +} + +/// Re-verifies every stored reviewer signature against external trust roots. +/// +/// # Errors +/// Rejects missing/altered proofs, review-row mismatches or untrusted signers. +pub fn verify_all(db: &Connection, allowed_reviewers: &Path) -> anyhow::Result<()> { + let allowed = crate::ssh::sealed_input(allowed_reviewers, 1024 * 1024)?; + verify_rows(db, Some(&allowed.bytes)) +} + +pub(crate) fn ensure_all_authenticated(db: &Connection) -> anyhow::Result<()> { + verify_rows(db, None) +} + +fn verify_rows(db: &Connection, allowed_reviewers: Option<&[u8]>) -> anyhow::Result<()> { + let missing: u64 = db.query_row( + "SELECT count(*) FROM reviews r LEFT JOIN review_auth a USING(sequence) WHERE a.sequence IS NULL", + [], + |row| crate::store::unsigned(row, 0), + )?; + ensure!( + missing == 0, + "generation contains unauthenticated operator reviews" + ); + let mut statement = db.prepare( + "SELECT r.fingerprint,r.decision,r.reviewer,r.reason,r.evidence,r.reviewed_at,r.expires_at,r.role,r.locale,r.country,a.signer,a.signature_sha256,a.namespace,a.decision_sha256,a.decision_json,a.signature FROM reviews r JOIN review_auth a USING(sequence) ORDER BY r.sequence", + )?; + let mut rows = statement.query([])?; + while let Some(row) = rows.next()? { + let authentication = Authentication { + signer: row.get(10)?, + signature_sha256: row.get(11)?, + namespace: row.get(12)?, + decision_sha256: row.get(13)?, + decision_json: row.get(14)?, + signature: row.get(15)?, + }; + let review = Review { + fingerprint: row.get(0)?, + decision: row.get(1)?, + reviewer: row.get(2)?, + reason: row.get(3)?, + evidence: row.get(4)?, + reviewed_at: parse_time(&row.get::<_, String>(5)?)?, + expires_at: parse_time(&row.get::<_, String>(6)?)?, + role: row.get(7)?, + locale: row.get(8)?, + country: row.get(9)?, + }; + validate_authentication(&review, &authentication)?; + if let Some(allowed) = allowed_reviewers { + crate::ssh::verify( + SIGNATURE_NAMESPACE, + &authentication.decision_json, + &authentication.signature, + allowed, + &authentication.signer, + )?; + } + } + Ok(()) +} + +fn parse_time(value: &str) -> anyhow::Result> { + Ok(DateTime::parse_from_rfc3339(value)?.with_timezone(&Utc)) +} diff --git a/crates/argand-site-registry/src/ssh.rs b/crates/argand-site-registry/src/ssh.rs new file mode 100644 index 0000000..dff7bf3 --- /dev/null +++ b/crates/argand-site-registry/src/ssh.rs @@ -0,0 +1,64 @@ +// By Nic Weyand! +//! Descriptor-bound inputs for external OpenSSH signature verification. + +use anyhow::ensure; +use std::{ + fs::File, + io::{Read, Seek, Write}, + path::Path, + process::{Command, Stdio}, +}; + +pub(crate) struct SealedInput { + pub file: File, + pub bytes: Vec, +} + +pub(crate) fn sealed_input(path: &Path, maximum: usize) -> anyhow::Result { + let mut file = crate::generation::open_no_follow(path)?; + ensure!( + file.metadata()?.is_file(), + "signature input must be a regular file" + ); + let mut bytes = Vec::new(); + Read::by_ref(&mut file) + .take(u64::try_from(maximum)? + 1) + .read_to_end(&mut bytes)?; + ensure!(bytes.len() <= maximum, "signature input exceeds its bound"); + file.rewind()?; + Ok(SealedInput { file, bytes }) +} + +pub(crate) fn verify( + namespace: &str, + message: &[u8], + signature: &[u8], + allowed_signers: &[u8], + identity: &str, +) -> anyhow::Result<()> { + // OpenSSH requires paths for its signature and trust file. Private, + // create-new temporary copies bind the command to the exact bytes already + // checked by this process instead of reopening attacker-controlled paths. + let mut message_file = tempfile::NamedTempFile::new()?; + message_file.write_all(message)?; + message_file.flush()?; + let mut signature_file = tempfile::NamedTempFile::new()?; + signature_file.write_all(signature)?; + signature_file.flush()?; + let mut allowed_file = tempfile::NamedTempFile::new()?; + allowed_file.write_all(allowed_signers)?; + allowed_file.flush()?; + + let status = Command::new("ssh-keygen") + .args(["-Y", "verify", "-n", namespace, "-f"]) + .arg(allowed_file.path()) + .arg("-I") + .arg(identity) + .arg("-s") + .arg(signature_file.path()) + .stdin(Stdio::from(message_file.reopen()?)) + .stdout(Stdio::null()) + .status()?; + ensure!(status.success(), "untrusted SSH signature"); + Ok(()) +} diff --git a/crates/argand-site-registry/src/store.rs b/crates/argand-site-registry/src/store.rs index d93a419..0002204 100644 --- a/crates/argand-site-registry/src/store.rs +++ b/crates/argand-site-registry/src/store.rs @@ -15,7 +15,13 @@ use std::{ }; /// Adapter/normalization contract recorded in all generation identities. -pub const RULE_VERSION: &str = "argand.site-rules/v1"; +pub const RULE_VERSION: &str = "argand.site-rules/v2"; + +/// Whether a signed immutable generation uses a reader-compatible rule contract. +#[must_use] +pub fn supported_rule_version(version: &str) -> bool { + version == RULE_VERSION +} /// Opens or migrates the local assertion store with bounded page cache. /// @@ -32,9 +38,15 @@ pub fn open(path: &Path) -> anyhow::Result { 0 => { db.execute_batch("BEGIN IMMEDIATE")?; db.execute_batch(include_str!("../migrations/001.sql"))?; + db.execute_batch(include_str!("../migrations/002.sql"))?; db.execute_batch("COMMIT")?; } - 1 => {} + 1 => { + db.execute_batch("BEGIN IMMEDIATE")?; + db.execute_batch(include_str!("../migrations/002.sql"))?; + db.execute_batch("COMMIT")?; + } + 2 => {} _ => anyhow::bail!("unsupported registry schema {version}"), } let rules: String = db.query_row( diff --git a/crates/argand-site-registry/tests/cli.rs b/crates/argand-site-registry/tests/cli.rs index 21dd87d..cd5c4cc 100644 --- a/crates/argand-site-registry/tests/cli.rs +++ b/crates/argand-site-registry/tests/cli.rs @@ -33,6 +33,7 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu if configured.is_some() { fs::create_dir(root)?; } + prepare_signer(root)?; let database = root.join("store.sqlite"); import_sources(root, &database)?; let candidate = root.join("candidate"); @@ -58,33 +59,24 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu lookup["candidates"][0]["web_property"]["domain"]["registrable_domain"], "facebook.com" ); - assert!( - run(&[ - "resolve", - "--generation", - text(&candidate)?, - "--pin", - pin, - "--query", - "facebook" - ])?["destination"] - .is_null() - ); - let now = chrono::Utc::now() - chrono::Duration::seconds(1); - let decision = json!({"fingerprint":lookup["candidates"][0]["fingerprint"],"decision":"approve","reviewer":"synthetic fixture reviewer","reason":"E2E test only, not actual site verification","evidence":"synthetic:fixture","reviewed_at":now,"expires_at":now+chrono::Duration::days(1),"role":"primary","locale":"","country":""}); - let decision_path = root.join("review.json"); - fs::write(&decision_path, serde_json::to_vec(&decision)?)?; - run(&[ - "review", - "--database", - text(&database)?, + inspect_commands(&candidate, pin, &lookup)?; + let unresolved = run(&[ + "resolve", "--generation", text(&candidate)?, "--pin", pin, - "--decision", - text(&decision_path)?, + "--query", + "facebook", ])?; + assert!(unresolved["destination"].is_null()); + assert_eq!(unresolved["status"], "no_active_review"); + let now = chrono::Utc::now() - chrono::Duration::seconds(1); + let decision = json!({"fingerprint":lookup["candidates"][0]["fingerprint"],"decision":"approve","reviewer":"fixture","reason":"E2E test only, not actual site verification","evidence":"synthetic:fixture","reviewed_at":now,"expires_at":now+chrono::Duration::days(1),"role":"primary","locale":"","country":""}); + let decision_path = root.join("review.json"); + fs::write(&decision_path, serde_json::to_vec(&decision)?)?; + reject_tampered_review(root, &database, &candidate, pin, &decision)?; + record_review(root, &database, &candidate, pin, &decision_path)?; let approved = root.join("approved"); review_identity(root, &database, &candidate, pin, &decision)?; let built = run(&[ @@ -107,6 +99,18 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu ])?["destination"]["url"], "https://facebook.com/" ); + assert_eq!( + run(&[ + "lookup", + "--generation", + text(&approved)?, + "--pin", + approved_pin, + "--query", + "FB", + ])?["candidates"][0]["review"]["authentication"]["identity"], + "fixture" + ); let (revoked, revoked_pin) = release_lifecycle( root, &database, @@ -120,6 +124,129 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu Ok(()) } +fn inspect_commands(generation: &Path, pin: &str, lookup: &Value) -> anyhow::Result<()> { + let entity_id = lookup["candidates"][0]["entity_id"] + .as_str() + .context("entity ID")?; + assert_eq!( + run(&[ + "entity", + "--generation", + text(generation)?, + "--pin", + pin, + "--id", + entity_id, + ])?["entity"]["canonical_name"], + "Facebook" + ); + let reverse = run(&[ + "lookup-web", + "--generation", + text(generation)?, + "--pin", + pin, + "--target", + "facebook.com", + ])?; + assert_eq!(reverse["total_edges"], 2); + assert!(reverse["matches"].as_array().is_some_and(|matches| { + matches + .iter() + .any(|entry| entry["candidate"]["canonical_name"] == "Facebook") + })); + assert_eq!( + run(&[ + "popularity", + "--generation", + text(generation)?, + "--pin", + pin, + "--target", + "facebook.com", + ])?["total"], + 2 + ); + let category = run(&[ + "category", + "--generation", + text(generation)?, + "--pin", + pin, + "--id", + "42", + ])?; + assert_eq!(category["total_members"], 1); + assert_eq!( + category["metadata"][0]["value"]["description_redacted"], + true + ); + assert!( + category["metadata"][0]["value"] + .get("description") + .is_none() + ); + let stats = run(&["stats", "--generation", text(generation)?, "--pin", pin])?; + assert_eq!(stats["selected_sources"], 5); + assert_eq!(stats["authenticated_reviews"], 0); + let cases = generation + .parent() + .context("generation parent")? + .join("cli-evaluation.jsonl"); + fs::write( + &cases, + b"{\"id\":\"missing\",\"query\":\"not a fixture entity\",\"expected_status\":\"no_name_match\"}\n", + )?; + let report = run(&[ + "evaluate", + "--generation", + text(generation)?, + "--pin", + pin, + "--cases", + text(&cases)?, + ])?; + assert_eq!(report["passed"], 1); + assert_eq!(report["failed"], 0); + Ok(()) +} + +fn reject_tampered_review( + root: &Path, + database: &Path, + generation: &Path, + pin: &str, + decision: &Value, +) -> anyhow::Result<()> { + let path = root.join("tampered-review.json"); + fs::write(&path, serde_json::to_vec(decision)?)?; + let signature = sign_review(root, &path)?; + let mut altered = decision.clone(); + altered["reason"] = json!("changed after signature"); + fs::write(&path, serde_json::to_vec(&altered)?)?; + assert!( + run(&[ + "review", + "--database", + text(database)?, + "--generation", + text(generation)?, + "--pin", + pin, + "--decision", + text(&path)?, + "--signature", + text(&signature)?, + "--allowed-reviewers", + text(&root.join("allowed_signers"))?, + "--identity", + "fixture", + ]) + .is_err() + ); + Ok(()) +} + fn import_sources(root: &Path, database: &Path) -> anyhow::Result<()> { let sources = [ ( @@ -174,31 +301,11 @@ fn release_lifecycle( mut decision: Value, decision_path: &Path, ) -> anyhow::Result<(std::path::PathBuf, String)> { - let key = root.join("signer"); - let status = Command::new("ssh-keygen") - .args(["-q", "-t", "ed25519", "-N", "", "-f"]) - .arg(&key) - .status()?; - ensure!(status.success(), "generate test key"); - let allowed = root.join("allowed_signers"); - fs::write( - &allowed, - format!("fixture {}", fs::read_to_string(key.with_extension("pub"))?), - )?; sign_and_activate(root, approved, approved_pin)?; decision["decision"] = json!("revoke"); - fs::write(decision_path, serde_json::to_vec(&decision)?)?; - run(&[ - "review", - "--database", - text(database)?, - "--generation", - text(approved)?, - "--pin", - approved_pin, - "--decision", - text(decision_path)?, - ])?; + let revocation_path = decision_path.with_file_name("revocation.json"); + fs::write(&revocation_path, serde_json::to_vec(&decision)?)?; + record_review(root, database, approved, approved_pin, &revocation_path)?; let revoked = root.join("revoked"); let built = run(&[ "build", @@ -241,6 +348,28 @@ fn release_lifecycle( fn sign_and_activate(root: &Path, approved: &Path, approved_pin: &str) -> anyhow::Result<()> { let key = root.join("signer"); let allowed = root.join("allowed_signers"); + let wrong_reviewers = root.join("wrong_reviewers"); + fs::write( + &wrong_reviewers, + format!( + "untrusted {}", + fs::read_to_string(key.with_extension("pub"))? + ), + )?; + assert!( + run(&[ + "sign", + "--generation", + text(approved)?, + "--pin", + approved_pin, + "--key", + text(&key)?, + "--allowed-reviewers", + text(&wrong_reviewers)?, + ]) + .is_err() + ); run(&[ "sign", "--generation", @@ -249,6 +378,8 @@ fn sign_and_activate(root: &Path, approved: &Path, approved_pin: &str) -> anyhow approved_pin, "--key", text(&key)?, + "--allowed-reviewers", + text(&allowed)?, ])?; let current = root.join("current.json"); assert!( @@ -328,11 +459,86 @@ fn review_identity( let path = root.join("identity-review.json"); fs::write(&path, serde_json::to_vec(&decision)?)?; let mut args = args.to_vec(); - args.extend(["--database", text(database)?, "--decision", text(&path)?]); + let signature = sign_review(root, &path)?; + let allowed = root.join("allowed_signers"); + args.extend([ + "--database", + text(database)?, + "--decision", + text(&path)?, + "--signature", + text(&signature)?, + "--allowed-reviewers", + text(&allowed)?, + "--identity", + "fixture", + ]); run(&args)?; Ok(()) } +fn prepare_signer(root: &Path) -> anyhow::Result<()> { + let key = root.join("signer"); + let status = Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(&key) + .status()?; + ensure!(status.success(), "generate test key"); + fs::write( + root.join("allowed_signers"), + format!("fixture {}", fs::read_to_string(key.with_extension("pub"))?), + )?; + Ok(()) +} + +fn sign_review(root: &Path, decision: &Path) -> anyhow::Result { + let status = Command::new("ssh-keygen") + .args([ + "-Y", + "sign", + "-n", + argand_site_registry::review::SIGNATURE_NAMESPACE, + "-f", + ]) + .arg(root.join("signer")) + .arg(decision) + .status()?; + ensure!(status.success(), "sign fixture review"); + Ok(std::path::PathBuf::from(format!( + "{}.sig", + decision.display() + ))) +} + +fn record_review( + root: &Path, + database: &Path, + generation: &Path, + pin: &str, + decision: &Path, +) -> anyhow::Result<()> { + let signature = sign_review(root, decision)?; + let allowed = root.join("allowed_signers"); + run(&[ + "review", + "--database", + text(database)?, + "--generation", + text(generation)?, + "--pin", + pin, + "--decision", + text(decision)?, + "--signature", + text(&signature)?, + "--allowed-reviewers", + text(&allowed)?, + "--identity", + "fixture", + ])?; + Ok(()) +} + fn export_fixture(root: &Path, revoked: &Path, revoked_pin: &str) -> anyhow::Result<()> { let export = root.join("registry.jsonl"); run(&[ diff --git a/crates/argand-site-registry/tests/evaluation.rs b/crates/argand-site-registry/tests/evaluation.rs new file mode 100644 index 0000000..cad6db0 --- /dev/null +++ b/crates/argand-site-registry/tests/evaluation.rs @@ -0,0 +1,105 @@ +// By Nic Weyand! +//! Multilingual and adversarial resolver judgments over authored fixtures. +#[allow(dead_code)] +mod common; + +use argand_site_registry::{ResolutionStatus, evaluation}; +use serde_json::json; +use std::io::Cursor; + +#[test] +fn multilingual_regional_and_deceptive_queries_are_replayed() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let candidate = common::build(&db, root.path(), "candidate")?; + common::approve( + &db, + &candidate, + "FB", + "https://facebook.com/", + "primary", + "", + )?; + common::approve( + &db, + &candidate, + "Atlas", + "https://atlas.example.co.uk/", + "regional", + "GB", + )?; + common::approve( + &db, + &candidate, + "Atlas", + "https://atlas.example.de/", + "regional", + "DE", + )?; + let registry = common::build(&db, root.path(), "reviewed")?; + let facebook = registry.lookup("FB", 1)?.candidates.remove(0).entity_id; + let rows = [ + json!({"id":"canonical","query":"Facebook","expected_status":"resolved","expected_entity_id":facebook,"expected_url":"https://facebook.com/"}), + json!({"id":"casefold-alias","query":"fb","expected_status":"resolved"}), + json!({"id":"combining-accent","query":"Cafe\u{301} Atlas","country":"GB","expected_status":"resolved","expected_url":"https://atlas.example.co.uk/"}), + json!({"id":"gb-region","query":"Atlas","country":"GB","expected_status":"resolved","expected_url":"https://atlas.example.co.uk/"}), + json!({"id":"de-region","query":"Atlas","country":"DE","expected_status":"resolved","expected_url":"https://atlas.example.de/"}), + json!({"id":"wrong-region","query":"Atlas","country":"US","expected_status":"region_mismatch"}), + json!({"id":"cyrillic-lookalike","query":"Fасebook","expected_status":"no_name_match"}), + json!({"id":"domain-is-not-an-identity","query":"facebook.com","expected_status":"no_name_match"}), + ]; + let input = rows + .iter() + .map(serde_json::to_string) + .collect::, _>>()? + .join("\n"); + let report = evaluation::run(®istry, Cursor::new(input), 100, common::timestamp()?)?; + assert_eq!(report.total, 8); + assert_eq!(report.passed, 8); + assert_eq!(report.failed, 0); + assert_eq!(report.evaluated_at, common::timestamp()?); + assert!(report.failures.is_empty()); + + let incorrect = serde_json::to_string(&json!({ + "id":"wrong-judgment", + "query":"FB", + "expected_status":"no_name_match" + }))?; + let report = evaluation::run(®istry, Cursor::new(incorrect), 1, common::timestamp()?)?; + assert_eq!(report.failed, 1); + assert_eq!(report.failures[0].actual_status, ResolutionStatus::Resolved); + Ok(()) +} + +#[test] +fn malformed_duplicate_and_over_limit_cases_fail_closed() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let registry = common::build(&db, root.path(), "generation")?; + let duplicate_key = + r#"{"id":"one","id":"two","query":"FB","expected_status":"no_active_review"}"#; + assert!( + evaluation::run( + ®istry, + Cursor::new(duplicate_key), + 1, + common::timestamp()? + ) + .is_err() + ); + let duplicate_id = r#"{"id":"one","query":"FB","expected_status":"no_active_review"} +{"id":"one","query":"Atlas","expected_status":"no_active_review"}"#; + assert!( + evaluation::run( + ®istry, + Cursor::new(duplicate_id), + 2, + common::timestamp()? + ) + .is_err() + ); + let two = r#"{"id":"one","query":"FB","expected_status":"no_active_review"} +{"id":"two","query":"Atlas","expected_status":"no_active_review"}"#; + assert!(evaluation::run(®istry, Cursor::new(two), 1, common::timestamp()?).is_err()); + Ok(()) +} diff --git a/crates/argand-site-registry/tests/failures.rs b/crates/argand-site-registry/tests/failures.rs index 6849c51..cbe4972 100644 --- a/crates/argand-site-registry/tests/failures.rs +++ b/crates/argand-site-registry/tests/failures.rs @@ -9,7 +9,119 @@ use argand_site_registry::{ store, }; use serde_json::json; -use std::{fs, io::Write}; +use std::{fs, io::Write, process::Command}; + +#[test] +fn version_one_store_migrates_without_losing_review_history() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let path = root.path().join("v1.sqlite"); + let db = rusqlite::Connection::open(&path)?; + db.execute_batch(include_str!("../migrations/001.sql"))?; + db.execute("INSERT INTO reviews VALUES(1,?1,'revoke','legacy','reason','evidence','2026-01-01T00:00:00Z','2026-01-01T00:00:00Z','unspecified','','')", ["0".repeat(64)])?; + drop(db); + + let migrated = store::open(&path)?; + assert_eq!( + migrated.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?, + 2 + ); + assert_eq!( + migrated.query_row("SELECT rules FROM registry_metadata", [], |row| row + .get::<_, String>(0))?, + store::RULE_VERSION + ); + assert_eq!( + migrated.query_row("SELECT count(*) FROM reviews", [], |row| row + .get::<_, i64>(0))?, + 1 + ); + assert_eq!( + migrated.query_row("SELECT count(*) FROM review_auth", [], |row| row + .get::<_, i64>(0))?, + 0 + ); + Ok(()) +} + +#[test] +fn externally_signed_unauthenticated_reviews_are_rejected() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let initial = common::build(&db, root.path(), "initial")?; + common::approve(&db, &initial, "FB", "https://facebook.com/", "primary", "")?; + let generation = common::build(&db, root.path(), "generation")?; + let key = root.path().join("key"); + assert!( + Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(&key) + .status()? + .success() + ); + fs::write( + root.path().join("allowed"), + format!("fixture {}", fs::read_to_string(key.with_extension("pub"))?), + )?; + assert!( + argand_site_registry::release::sign( + &root.path().join("generation"), + &key, + &generation.identity, + &root.path().join("allowed") + ) + .is_err() + ); + assert!( + Command::new("ssh-keygen") + .args(["-Y", "sign", "-n", "argand-site-registry", "-f"]) + .arg(&key) + .arg(root.path().join("generation/COMPLETE.json")) + .status()? + .success() + ); + assert!( + argand_site_registry::release::verify_signed( + &root.path().join("generation"), + &root.path().join("allowed"), + "fixture" + ) + .is_err() + ); + Ok(()) +} + +#[test] +fn unsigned_sqlite_sidecars_and_generation_symlinks_are_rejected() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let generation = common::build(&db, root.path(), "sealed")?; + let scratch = root.path().join("scratch.sqlite"); + fs::copy(root.path().join("sealed/registry.sqlite"), &scratch)?; + let attacker = rusqlite::Connection::open(&scratch)?; + attacker.execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;")?; + attacker.execute( + "UPDATE properties SET url='https://unsigned.example.org/' WHERE url='https://facebook.com/'", + [], + )?; + fs::copy( + scratch.with_extension("sqlite-wal"), + root.path().join("sealed/registry.sqlite-wal"), + )?; + let error = Registry::open(&root.path().join("sealed"), &generation.identity) + .err() + .ok_or_else(|| anyhow::anyhow!("unsigned sidecar was accepted"))?; + assert!(error.to_string().contains("unexpected generation entry")); + drop(attacker); + + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::symlink; + let link = root.path().join("generation-link"); + symlink(root.path().join("sealed"), &link)?; + assert!(Registry::open(&link, &generation.identity).is_err()); + } + Ok(()) +} #[test] fn removed_entity_websites_retire_the_previous_selection() -> anyhow::Result<()> { @@ -70,8 +182,9 @@ fn import_order_does_not_change_generation_and_psl_refresh_preserves_review() -> assert_eq!(first.identity, second.identity); common::approve(&b, &second, "FB", "https://facebook.com/", "primary", "")?; let input = root.path().join("psl"); - fs::write(&input, common::PSL)?; - let mut manifest = common::manifest(Source::Psl, Format::PslText, common::PSL.as_bytes())?; + let comment_only = format!("{}// Comment-only refresh\n", common::PSL); + fs::write(&input, &comment_only)?; + let mut manifest = common::manifest(Source::Psl, Format::PslText, comment_only.as_bytes())?; manifest.retrieved_at += chrono::Duration::days(1); store::import(&mut b, &manifest, &input)?; let refreshed = common::build(&b, root.path(), "refresh")?; @@ -80,6 +193,77 @@ fn import_order_does_not_change_generation_and_psl_refresh_preserves_review() -> .resolve("FB", None, None, common::timestamp()?)? .is_some() ); + assert_eq!( + second.lookup("FB", 1)?.candidates[0].fingerprint, + refreshed.lookup("FB", 1)?.candidates[0].fingerprint + ); + Ok(()) +} + +#[test] +fn psl_semantic_change_invalidates_affected_review_only() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let mut db = common::fixture(root.path())?; + let original = common::build(&db, root.path(), "original")?; + common::approve( + &db, + &original, + "Atlas", + "https://atlas.example.co.uk/", + "regional", + "GB", + )?; + let original_fingerprint = original + .lookup("Atlas", 10)? + .candidates + .into_iter() + .find(|candidate| candidate.url == "https://atlas.example.co.uk/") + .ok_or_else(|| anyhow::anyhow!("fixture regional property missing"))? + .fingerprint; + + let changed = common::PSL.replace("co.uk\n", ""); + let input = root.path().join("changed-psl"); + fs::write(&input, &changed)?; + let mut manifest = common::manifest(Source::Psl, Format::PslText, changed.as_bytes())?; + manifest.retrieved_at += chrono::Duration::days(1); + store::import(&mut db, &manifest, &input)?; + let rebuilt = common::build(&db, root.path(), "changed")?; + let changed_candidate = rebuilt + .lookup("Atlas", 10)? + .candidates + .into_iter() + .find(|candidate| candidate.url == "https://atlas.example.co.uk/") + .ok_or_else(|| anyhow::anyhow!("changed property missing"))?; + assert_ne!(original_fingerprint, changed_candidate.fingerprint); + assert_eq!( + changed_candidate.web_property["domain"]["registrable_domain"], + "co.uk" + ); + assert!( + rebuilt + .resolve("Atlas", None, Some("GB"), common::timestamp()?)? + .is_none() + ); + Ok(()) +} + +#[test] +fn typed_diff_reports_review_only_changes() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let before = common::build(&db, root.path(), "before")?; + common::approve(&db, &before, "FB", "https://facebook.com/", "primary", "")?; + let after = common::build(&db, root.path(), "after")?; + let mut output = Vec::new(); + argand_site_registry::release::diff(&before, &after, &mut output)?; + let rows = String::from_utf8(output)? + .lines() + .map(serde_json::from_str::) + .collect::, _>>()?; + assert!(rows.iter().any(|row| { + row["type"] == "change" && row["subject"] == "review" && row["change"] == "added" + })); + assert_eq!(rows.last().and_then(|row| row["changes"].as_u64()), Some(1)); Ok(()) } diff --git a/crates/argand-site-registry/tests/observation.rs b/crates/argand-site-registry/tests/observation.rs new file mode 100644 index 0000000..e1d04d5 --- /dev/null +++ b/crates/argand-site-registry/tests/observation.rs @@ -0,0 +1,60 @@ +// By Nic Weyand! +//! Future crawler evidence remains normalized, source-bound and non-authoritative. +#[allow(dead_code)] +mod common; + +use argand_site_registry::{ + normalize::Normalizer, + observation::{Observation, ObservationKind}, +}; + +fn fixture(relation: ObservationKind) -> anyhow::Result { + Ok(Observation { + relation, + from_url: "HTTPS://Example.COM:443/start/../region".into(), + to_url: "https://example.co.uk/shop".into(), + source: "synthetic-crawler".into(), + source_identifier: "capture:1".into(), + license: "CC0-1.0".into(), + license_url: "https://creativecommons.org/publicdomain/zero/1.0/".into(), + retrieved_at: common::timestamp()?, + content_sha256: "a".repeat(64), + selector: "HTTP Location header".into(), + confidence: 9000, + }) +} + +#[test] +fn observation_normalization_is_deterministic_and_retains_rights() -> anyhow::Result<()> { + let normalizer = Normalizer::new(common::PSL.as_bytes(), "fixture-psl".into())?; + let observation = fixture(ObservationKind::Redirect { status: 308 })?; + let first = observation.normalize(&normalizer)?; + let second = observation.normalize(&normalizer)?; + assert_eq!(first.fingerprint, second.fingerprint); + assert_eq!(first.from.url, "https://example.com/region"); + assert_eq!(first.to.domain.registrable_domain, "example.co.uk"); + assert_eq!(first.source_identifier, "capture:1"); + assert_eq!(first.license, "CC0-1.0"); + Ok(()) +} + +#[test] +fn observation_validation_rejects_false_redirects_and_bad_scopes() -> anyhow::Result<()> { + let normalizer = Normalizer::new(common::PSL.as_bytes(), "fixture-psl".into())?; + assert!( + fixture(ObservationKind::Redirect { status: 200 })? + .normalize(&normalizer) + .is_err() + ); + assert!( + fixture(ObservationKind::CountrySelector { + country: "usa".into() + })? + .normalize(&normalizer) + .is_err() + ); + let mut bad_rights = fixture(ObservationKind::Canonical)?; + bad_rights.license_url = "file:///tmp/claim".into(); + assert!(bad_rights.normalize(&normalizer).is_err()); + Ok(()) +} diff --git a/crates/argand-site-registry/tests/registry.rs b/crates/argand-site-registry/tests/registry.rs index 32740a6..e6d6757 100644 --- a/crates/argand-site-registry/tests/registry.rs +++ b/crates/argand-site-registry/tests/registry.rs @@ -5,7 +5,7 @@ use anyhow::{Context, ensure}; use argand_site_registry::{ model::{Compression, Format, Source}, normalize::{Normalizer, name_key}, - query::Registry, + query::{Registry, ResolutionStatus}, review::{self, Review}, store, }; @@ -94,6 +94,35 @@ fn aliases_deduplication_provenance_and_separate_popularity() -> anyhow::Result< assert_eq!(r.receipt.properties, 4); // Curlie Facebook shares the same property. assert_eq!(r.receipt.entities, 3); // Curlie is not automatically the Wikidata entity. assert!(r.resolve("Facebook", None, None, timestamp()?)?.is_none()); + let unresolved = r.resolve_explained("Facebook", None, None, timestamp()?)?; + assert_eq!(unresolved.status, ResolutionStatus::NoActiveReview); + assert_eq!(unresolved.counts.missing_review, 1); + let reverse = r.lookup_web("facebook.com", 10)?; + assert_eq!(reverse.total_properties, 1); + assert_eq!(reverse.total_edges, 2); + assert!( + reverse + .matches + .iter() + .any(|item| item.candidate.canonical_name == "Facebook") + ); + let popularity = r.popularity("https://facebook.com/", 10)?; + assert_eq!(popularity.total, 2); + assert_eq!( + popularity + .observations + .iter() + .map(|item| item.source.as_str()) + .collect::>(), + std::collections::BTreeSet::from(["crux", "majestic"]) + ); + let entity = r + .entity_by_id(&c.entity_id, 10)? + .context("entity lookup missing")?; + assert_eq!(entity.entity.canonical_name, "Facebook"); + let category = r.category("42", 10)?; + assert_eq!(category.total_members, 1); + assert!(!serde_json::to_string(&category)?.contains("Synthetic category description")); let export = dir.path().join("export.jsonl"); argand_site_registry::release::export(&r, &export, false)?; let text = std::fs::read_to_string(&export)?; @@ -150,6 +179,17 @@ fn regional_review_expiry_and_revocation() -> anyhow::Result<()> { )? .is_none() ); + assert_eq!( + approved + .resolve_explained( + "Atlas", + None, + None, + timestamp()? + chrono::Duration::days(8) + )? + .status, + ResolutionStatus::NoActiveReview + ); let fingerprint = approved .lookup("Atlas", 10)? .candidates @@ -201,6 +241,16 @@ fn ambiguity_survives_limits_and_same_named_domains_do_not_merge() -> anyhow::Re assert_eq!(result.total_edges, 4); assert!(result.truncated); assert!(r.resolve("Atlas", None, None, timestamp()?)?.is_none()); + assert_eq!( + r.resolve_explained("Atlas", None, None, timestamp()?)? + .status, + ResolutionStatus::AmbiguousIdentity + ); + assert_eq!( + r.resolve_explained("absent", None, None, timestamp()?)? + .status, + ResolutionStatus::NoNameMatch + ); Ok(()) } diff --git a/docs/CONSUMERS.md b/docs/CONSUMERS.md index 8afc168..54ba604 100644 --- a/docs/CONSUMERS.md +++ b/docs/CONSUMERS.md @@ -4,10 +4,11 @@ Use `argand_site_registry::query::Registry::open(generation, trusted_pin)` once per immutable generation and reuse the reader. `lookup(query, limit)` returns evidence -and complete ambiguity counts; `resolve(query, locale, country, now)` returns an -optional reviewed candidate. Check the compiled example and API docs for exact -types. `selection_context` binds the full alternative set for downstream query -review. Preserve the returned provenance, scopes, counts and attribution. +and complete ambiguity counts; `resolve_explained` returns a reviewed candidate or +a typed abstention reason with counts. Exact reverse views cover entity IDs, +URLs/domains, popularity and Curlie categories. Check the compiled example and API +docs for exact types. `selection_context` binds the full alternative set for +downstream query review. Preserve returned provenance, scopes, counts and attribution. For local integration, point a Cargo dependency at `crates/argand-site-registry` inside an extracted standalone source tree. Once an @@ -17,7 +18,8 @@ Both crates remain in this workspace; the atomic helper is a relative dependency ## CLI and other languages -`lookup`, `resolve`, `verify` and the other commands emit JSON. The Python example +`lookup`, `resolve`, `entity`, `lookup-web`, `popularity`, `category`, `stats`, +`evaluate`, `verify` and the other commands emit JSON. The Python example passes arguments directly to the native executable, preserving query text and the entire response. A nonzero exit is an error. `destination: null` is a successful abstention, not a request to pick the first lookup candidate. Render source names @@ -37,8 +39,10 @@ omits descriptions and includes fact provenance plus an attribution envelope. It is an assertion export, not a self-contained signed list of admitted routes. Raw SQL inspection is useful for audit; it does not implement resolution policy. -Code version 0.1.0 is an initial interface. Schema/rule contracts are versioned -independently in receipts. Unsupported contracts fail closed. Pin source releases, +Code version 0.2.0 uses schema version 2 and `argand.site-rules/v2`. +It adds authenticated reviewer proofs, exact immutable SQLite opening, typed diffs +and audit/evaluation APIs. Schema/rule contracts remain versioned independently in +receipts. Unsupported contracts fail closed. Pin source releases, compile consumers and replay fixed fixtures before upgrades. Preserve import and review history; never mutate complete generations to migrate them. diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md new file mode 100644 index 0000000..8c10673 --- /dev/null +++ b/docs/EVALUATION.md @@ -0,0 +1,35 @@ +# Resolver evaluation + +`evaluate` replays authored judgments through the same native resolver used by +consumers. It opens one externally pinned immutable generation, streams up to the +configured case limit and reports correctness plus native p50/p95 query latency. +Pass an explicit `--at` time when the report must be replayable across approval +expiry boundaries; the chosen clock is included in the report. +The input is bounded to 16 MiB, each line to 64 KiB and case IDs must be unique. + +Each nonempty JSONL line has this form: + +```json +{"id":"facebook-primary","query":"facebook","locale":null,"country":null,"expected_status":"resolved","expected_entity_id":"argand:entity:SOURCE:ID","expected_url":"https://www.facebook.com/"} +``` + +`locale`, `country`, `expected_entity_id` and `expected_url` are optional. Status +is required and is one of `resolved`, `no_name_match`, `ambiguous_identity`, +`safety_limit_exceeded`, `no_eligible_destination`, `no_active_review`, +`region_mismatch` or `ambiguous_destination`. Expected URLs must be copied from a +pinned registry, including normalization such as a trailing slash. + +```bash +argand-site-registry evaluate --generation /data/registry/reviewed \ + --pin "$REVIEWED_PIN" --cases /data/evaluation/navigation.jsonl \ + --maximum-cases 10000 --at 2026-09-13T00:00:00Z \ + > /data/evaluation/report.json +jq -e '.failed == 0 and .passed == .total' /data/evaluation/report.json +``` + +Keep the corpus version and digest with the report. Include canonical names, +aliases, Unicode normalization, multiple scripts, every served country/locale, +unknown names, deceptive lookalikes, ambiguous entities, expired/revoked reviews +and tied destinations. Use synthetic or authorized query material; do not commit +private user logs. Latency values are process and hardware measurements. Compare +them only with an equivalent environment and sufficient sample size. diff --git a/docs/INDEX.md b/docs/INDEX.md index 59f353f..5c743d1 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -6,6 +6,8 @@ - [Source licenses](../crates/argand-site-registry/LICENSE_SOURCES.md): exact terms and attribution. - [Consumers](CONSUMERS.md): Rust, Python/CLI, data distribution and Argand transition. - [Trust](TRUST.md): enforced checks and publisher/consumer responsibilities. +- [Publishing](PUBLISHING.md): reviewer keys, candidate acceptance and activation. +- [Evaluation](EVALUATION.md): bounded JSONL judgments and result interpretation. - [Releasing](RELEASING.md): CI, source signing and archive verification. - [Validation](VALIDATION.md): independent builds and native acceptance evidence. - [Contributing](../CONTRIBUTING.md), [governance](../GOVERNANCE.md), diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md new file mode 100644 index 0000000..1d5ecae --- /dev/null +++ b/docs/PUBLISHING.md @@ -0,0 +1,75 @@ +# Dataset publisher runbook + +This runbook creates a reviewable candidate and a signed activation. Source update, +review and release keys are separate authorities. Scheduled jobs may acquire, +import and build candidates; they do not approve, sign or activate destinations. + +## Trust roots and local state + +Keep cache, mutable store, immutable generations, reviewer keys, release keys and +consumer trust files outside the checkout. An OpenSSH reviewer trust file contains +one accepted principal and public key per line: + +```text +operator@example.org ssh-ed25519 REVIEWER_PUBLIC_KEY +``` + +Distribute the release publisher public key to consumers through an independent +authenticated channel. Do not put private keys, production trust files or source +datasets in Git or CI. The isolated source CI runner has none of these files. + +## Candidate acceptance + +Run `update` or the explicit download/import/build commands from the operator guide. +For every candidate generation: + +1. Verify its externally recorded receipt pin with `verify`. +2. Run `stats` and compare source snapshots, selected sources, facts, rejections, + reviews, database bytes and upcoming expiry with the prior accepted generation. +3. Run `diff` against the prior pin. Investigate every source-selection, identity, + name, property, edge, popularity, review and equivalence change. +4. Use `lookup`, `entity`, `lookup-web`, `popularity` and `category` to inspect exact + source evidence and conflicts. Popularity never proves ownership. +5. Replay the maintained evaluation corpus with `evaluate`; require zero judgment + mismatches and compare latency with a documented hardware/process baseline. + +## Authenticated decisions + +Create a bounded review JSON from the exact candidate fingerprint and evidence. +Sign its exact bytes and append it through the CLI: + +```bash +ssh-keygen -Y sign -n argand-site-registry-review \ + -f /secure/reviewer-key review.json +argand-site-registry review --database /data/registry/import.sqlite \ + --generation /data/registry/candidate --pin "$CANDIDATE_PIN" \ + --decision review.json --signature review.json.sig \ + --allowed-reviewers /secure/reviewer-allowed-signers \ + --identity operator@example.org +``` + +Identity equivalence decisions use the same signed JSON and reviewer namespace. +Rebuild after appending decisions, then repeat the complete diff and evaluation. +The release command re-verifies every retained reviewer signature against the +current reviewer trust file. Missing, altered or no-longer-trusted proofs stop it. + +## Sign and activate + +```bash +argand-site-registry sign --generation /data/registry/reviewed \ + --pin "$REVIEWED_PIN" --key /secure/release-key \ + --allowed-reviewers /secure/reviewer-allowed-signers +argand-site-registry activate --generation /data/registry/reviewed \ + --current /data/registry/current.json \ + --allowed-signers /secure/release-allowed-signers \ + --identity registry-publisher +``` + +Record the source commit, candidate and accepted receipt pins, typed diff, evaluation +report, reviewer trust-file digest, release signer identity and activation receipt +in an immutable operator log. Activation refuses a rollback that drops a distributed +revocation. Deliver a new current pointer/pin to every consumer and bound their caches. + +For an incident, append a signed revocation, rebuild with full history, inspect, +evaluate, sign and activate. Preserve the suspect source bytes, generation and proofs. +Follow [SECURITY.md](../SECURITY.md) for private reporting and key compromise. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index dbe2e17..eca95a5 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -14,7 +14,8 @@ define the build behavior. A lockfile pins dependencies, not the host compiler. The Forgejo workflow uses the documented [workflow and context syntax](https://forgejo.org/docs/latest/user/actions/reference/). Register `site-registry-isolated` only on a disposable, repository-scoped runner with the above tools, two build jobs and at least 4 GiB memory. Use a pinned, -reviewed runner image. Do not mount production directories, share signing keys, +reviewed runner image; the exact local build is in [ci/Dockerfile](../ci/Dockerfile). +Do not mount production directories, share signing keys, or use Argand's host runners. Follow Forgejo's [runner security guidance](https://forgejo.org/docs/latest/user/actions/security/). The label is a deployment requirement, not a provisioned runner supplied by this repository. The workflow runs on trusted main pushes or manual dispatch, fetches @@ -85,6 +86,8 @@ receipt; the initial release tool packages source only. Source releases contain no provider datasets or real approvals. Dataset publishers follow the operator guide: import, inspect, review, build, diff, sign and activate. +Follow the [publisher runbook](PUBLISHING.md); release signing requires the external +reviewer trust file and re-verifies every stored decision signature. The dataset namespace `argand-site-registry` is distinct from the source namespace above. Keep `LICENSE_SOURCES.md` and `ATTRIBUTION.json` with the database and receipt. The weekly update example creates candidates. It never approves, renews, signs or diff --git a/docs/TRUST.md b/docs/TRUST.md index 28fef8f..e73a2df 100644 --- a/docs/TRUST.md +++ b/docs/TRUST.md @@ -12,18 +12,22 @@ Destination and identity decisions bind exact evidence fingerprints, including names, assertions and normalization context. Reviews expire within 90 days. Changed evidence invalidates earlier approvals. Resolution abstains on ambiguity, ties, missing approval or ineligible claims; regional scopes must explicitly -match. The local writer owns the append-only review log. +match. Reviewer decisions are signed under a dedicated SSH namespace and the local +writer retains their exact decision/signature bytes in the append-only review log. Generations bind the database, license document and attribution to a completion receipt. Consumers provide a trusted hash or verify an external publisher key. -Activation checks signatures and refuses rollback that loses distributed -revocations. Updates build candidates and cannot approve, sign or activate them. +Release signing re-verifies every stored decision against an external reviewer +trust file. Activation checks publisher signatures, rejects structurally incomplete +review proofs and refuses rollback that loses distributed revocations. Updates +build candidates and cannot approve, sign or activate them. ## What a publisher must establish The reviewer name and evidence locator in a decision are operator assertions. -The CLI validates their structure and evidence binding; it does not independently -authenticate the reviewer, retrieve their evidence or prove website ownership. +The CLI authenticates exact decision bytes to an allowed SSH signer and validates +their structure and evidence binding; it does not retrieve the cited evidence or +prove website ownership. Protect the writer database and signing key with separate operating permissions. Restrict who can author decisions and require human review before release signing. @@ -37,8 +41,9 @@ to review. Confidence values are assertion scores, not calibrated probabilities. Choose expiry based on volatility, within the enforced maximum. Do not renew blindly on a timer. Expired approval should lead to abstention until evidence is reviewed. Disclose editorial conflicts and use an independent reviewer for a -disputed claim when possible. The initial implementation is a local single-writer -tool; it does not provide authenticated reviewer accounts or an enforced quorum. +disputed claim when possible. The implementation is a local single-writer tool +with externally authenticated reviewer keys; it does not provide accounts or an +enforced quorum. ## What consumers must preserve diff --git a/scripts/source_release.py b/scripts/source_release.py index 2d7ea3f..c21a7e4 100644 --- a/scripts/source_release.py +++ b/scripts/source_release.py @@ -57,7 +57,7 @@ def safe_path(name): raise ValueError("unsafe archive path") allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml", ".service", ".timer"} - if path.suffix not in allowed and path.name != ".gitignore" and name not in ( + if path.suffix not in allowed and path.name not in (".gitignore", "Dockerfile") and name not in ( "LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore"): raise ValueError(f"file is outside the source release allowlist: {name}")