release: implement site registry v0.5
All checks were successful
Standalone registry checks / check (push) Successful in 5m58s
All checks were successful
Standalone registry checks / check (push) Successful in 5m58s
This commit is contained in:
parent
967d7ab45f
commit
557ba7cd69
SSH key fingerprint:
SHA256:2te+ycJIQON/Wo/dH6+ZkFSQ4HnHWpetV2azx9E65dQ
40 changed files with 3331 additions and 158 deletions
55
Cargo.lock
generated
55
Cargo.lock
generated
|
|
@ -78,14 +78,14 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
|||
|
||||
[[package]]
|
||||
name = "argand-atomic"
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "argand-site-registry"
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"argand-atomic",
|
||||
|
|
@ -97,6 +97,7 @@ dependencies = [
|
|||
"flate2",
|
||||
"http",
|
||||
"libc",
|
||||
"md-5",
|
||||
"publicsuffix",
|
||||
"reqwest",
|
||||
"rusqlite",
|
||||
|
|
@ -110,6 +111,7 @@ dependencies = [
|
|||
"toml",
|
||||
"unicode-normalization",
|
||||
"url",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -527,6 +529,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
|||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1089,6 +1092,16 @@ dependencies = [
|
|||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
|
|
@ -2156,6 +2169,12 @@ version = "0.2.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
|
|
@ -2615,8 +2634,40 @@ dependencies = [
|
|||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "8.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"typed-path",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ resolver = "3"
|
|||
members = ["crates/argand-atomic", "crates/argand-site-registry"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
authors = ["Nic Weyand"]
|
||||
edition = "2024"
|
||||
license = "AGPL-3.0-or-later"
|
||||
|
|
@ -21,11 +21,13 @@ reqwest = { version = "0.13.2", default-features = false, features = ["json", "q
|
|||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.149"
|
||||
sha2 = "0.10.9"
|
||||
md-5 = "0.10.6"
|
||||
tempfile = "3.27.0"
|
||||
tokio = { version = "1.52.1", features = ["full"] }
|
||||
toml = "1.0.7"
|
||||
unicode-normalization = "0.1.25"
|
||||
url = { version = "2.5.8", features = ["serde"] }
|
||||
zip = { version = "=8.6.0", default-features = false, features = ["deflate"] }
|
||||
|
||||
[workspace.lints.rust]
|
||||
unsafe_code = "forbid"
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ Run the strict 0.4 trust and observer acceptance flow:
|
|||
cargo test -p argand-site-registry --test v04 --locked --offline -- --nocapture
|
||||
```
|
||||
|
||||
The first fixture imports synthetic Wikidata, Majestic Million, CrUX, Curlie,
|
||||
The first fixture imports synthetic Wikidata, ROR, Majestic Million, CrUX, Curlie,
|
||||
and Public Suffix List inputs. The second proves two-reviewer name and edge votes,
|
||||
policy epochs, stale evidence, observation replay, drift, sticky revocation,
|
||||
publisher separation, signed activation, and resistance to signature tampering.
|
||||
|
|
@ -157,6 +157,7 @@ CLI and preserves its JSON contract.
|
|||
| Source | Consumed evidence | Data license |
|
||||
| --- | --- | --- |
|
||||
| Wikidata | IDs, labels, aliases, P856 statements, selected locale/country metadata | CC0 1.0 Universal |
|
||||
| Research Organization Registry (ROR) | organization IDs, names, aliases, websites, domains, status, type, locations and external IDs | CC0 1.0 Universal; GeoNames location lineage is attributed separately |
|
||||
| Majestic Million | source-specific domain rank and supplied metrics | CC BY 3.0 Unported |
|
||||
| Chrome UX Report | origin popularity bucket, month, optional audience country | CC BY 4.0 International |
|
||||
| Curlie | site titles, categories, descriptions retained for audit | CC BY 3.0 Unported |
|
||||
|
|
@ -181,6 +182,8 @@ review, attribution rules, and tests.
|
|||
supersession; conflicts fail closed.
|
||||
- Active export contains selected, nonrejected facts. Audit export also retains
|
||||
superseded and rejected evidence.
|
||||
- Compact generations keep query projections local and bind complete historical
|
||||
records/facts through verified, content-addressed external audit bundles.
|
||||
- Votes bind assertion, evidence-bundle, policy, reviewer, scope, and expiry.
|
||||
- Revocations remain sticky until every member of a fresh quorum explicitly
|
||||
supersedes them.
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ clap.workspace = true
|
|||
csv = "1.4.0"
|
||||
flate2.workspace = true
|
||||
libc.workspace = true
|
||||
md-5.workspace = true
|
||||
publicsuffix = "=2.3.0"
|
||||
reqwest.workspace = true
|
||||
rusqlite = { version = "=0.40.2", features = ["bundled"] }
|
||||
|
|
@ -30,6 +31,7 @@ tokio.workspace = true
|
|||
toml.workspace = true
|
||||
unicode-normalization.workspace = true
|
||||
url.workspace = true
|
||||
zip.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
http.workspace = true
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ listing is evidence of an assertion, not a guarantee of ownership or safety.
|
|||
| Source | Exact data license and evidence | Distribution and consumed fields |
|
||||
| --- | --- | --- |
|
||||
| Wikidata | [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/), [`CC0-1.0`](https://www.wikidata.org/wiki/Wikidata:Licensing) | [JSON dumps](https://www.wikidata.org/wiki/Wikidata:Database_download), [entity JSON](https://www.wikidata.org/wiki/Special:EntityData/Q355.json). Entity ID, revision, all labels/aliases, full P856 statements including ranks, qualifiers and references; P17, P159, P407, P1001 and country/language code mappings P297/P218/P219/P220. Relevant raw entity records remain available for audit. |
|
||||
| Research Organization Registry (ROR) | [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/), [`CC0-1.0`](https://ror.readme.io/docs/data-dump) | [Official versioned data dump](https://ror.readme.io/docs/data-dump) distributed from ROR's Zenodo community. Schema 2.1 organization ID, names and aliases, website links, declared domains, status, organization types, locations/countries, relationships and external IDs. The importer requires the official JSON-and-CSV ZIP shape and retains the JSON record for audit. ROR documents GeoNames as upstream for location metadata; GeoNames attribution and CC BY 3.0 terms are preserved. |
|
||||
| Majestic Million | [CC BY 3.0 Unported](https://creativecommons.org/licenses/by/3.0/), [`CC-BY-3.0`](https://majestic.com/reports/majestic-million) | [Official CSV](https://downloads.majestic.com/majestic_million.csv). Domain/IDN/TLD, global/TLD ranks, referring subnet/IP counts and their previous values. Ranks remain source-specific signals; no entity ownership is inferred. |
|
||||
| Chrome UX Report (CrUX), Google | [CC BY 4.0 International](https://creativecommons.org/licenses/by/4.0/), [`CC-BY-4.0`](https://developer.chrome.com/docs/crux/methodology) | [Monthly BigQuery dataset](https://developer.chrome.com/docs/crux/bigquery/): `origin`, `experimental.popularity.rank`, observation month, optional audience-country dataset code. The adapter produces `origin,rank,yyyymm,country_code` CSV. Rank is a coarse bucket, not a precise visit count. Audience country is not website jurisdiction. No API key or OAuth token is retained. |
|
||||
| Curlie | [CC BY 3.0 Unported](https://creativecommons.org/licenses/by/3.0/), [`CC-BY-3.0`](https://curlie.org/docs/en/license.html), including the attribution placement prescribed on that page | [Format documentation](https://curlie.org/docs/en/rdf.html), [official download redirect](https://curlie.org/directory-dl), currently [Passau-hosted archive](https://share.innkube.fim.uni-passau.de/curlie-rdf/curlie-rdf-all.tar.gz). Despite its RDF name, the current archive contains **literal TSV**. Content: URL, title, description, category ID. Structure: category ID, full category path, entry count, description, latitude, longitude. Archive notices are retained. |
|
||||
|
|
@ -20,6 +21,13 @@ listing is evidence of an assertion, not a guarantee of ownership or safety.
|
|||
* **Wikidata:** CC0 imposes no attribution condition. Keep Wikidata IDs, source
|
||||
links and revision evidence for traceability. CC0 structured data does not
|
||||
extend to unrelated Wikipedia prose, images or linked websites.
|
||||
* **ROR:** CC0 imposes no attribution condition on the ROR dump. Keep ROR IDs,
|
||||
release/Zenodo record identity, provider checksum evidence and schema version
|
||||
for traceability. ROR location fields declare GeoNames as upstream; credit
|
||||
[GeoNames](https://www.geonames.org/), link its
|
||||
[CC BY 3.0 license](https://creativecommons.org/licenses/by/3.0/), retain notices
|
||||
and identify Argand's projection when distributing those fields. A ROR website
|
||||
or declared domain is an assertion and never grants route approval.
|
||||
* **Majestic:** credit “Majestic Million, Majestic”, link the source and CC BY 3.0,
|
||||
retain supplied notices, and identify Argand's changes. The current distribution
|
||||
page governs this import; older blog posts describe different historic terms.
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ argand-site-registry --help
|
|||
cargo test -p argand-site-registry --all-targets --locked --offline
|
||||
```
|
||||
|
||||
The native CLI test imports small source-shaped fixtures for **all five sources**,
|
||||
The native CLI tests import small source-shaped fixtures for **all six sources**,
|
||||
repeats the imports, resolves aliases, signs and activates an approved generation,
|
||||
revokes the destination, and rejects rollback past the revocation. Synthetic
|
||||
fixtures are authored in Rust test code; no provider datasets or signing keys
|
||||
|
|
@ -105,11 +105,52 @@ ambiguous or overlapping typed coverage.
|
|||
For a full Wikidata dump, select a real dump URL from the official
|
||||
[download index](https://dumps.wikimedia.org/wikidatawiki/entities/), then use
|
||||
`--format wikidata-dump --compression gzip` (or `bzip2`) and `--scope full`.
|
||||
The parser handles the documented one-entity-per-line JSON array and concatenated
|
||||
compressed streams. Do not use truthy RDF: it loses statement evidence. Full
|
||||
The parser streams the documented top-level JSON array without assuming that an
|
||||
entity is physically one line, and handles concatenated compressed streams. It
|
||||
rejects a record above the authenticated 16 MiB default; after inspecting the
|
||||
exact object, a publisher can bind a larger limit with `--maximum-record-bytes`
|
||||
on `download` or `manifest`, up to the 256 MiB audit-safe ceiling. Do not use
|
||||
truthy RDF: it loses statement evidence. Full
|
||||
dumps need substantial disk space and a long sequential scan even though memory
|
||||
is bounded. A small entity selection is useful on limited hardware.
|
||||
|
||||
Wikidata's current Add/Change artifacts are XML history/stub streams. Wikidata's
|
||||
own download guidance warns that JSON embedded in XML dumps is unstable, so this
|
||||
release refuses that format. For bounded incremental refresh, acquire official
|
||||
Wikibase API `wbgetentities` JSON, declare typed `delta` coverage against the
|
||||
exact selected-entity base and apply consecutive sequences. A changed entity with
|
||||
no P856 statements emits an empty record, which becomes a website-evidence
|
||||
tombstone in the audit export. Reconcile API deltas against a later full JSON dump
|
||||
before replacing broad production coverage.
|
||||
|
||||
### ROR
|
||||
|
||||
ROR publishes versioned CC0 dumps through Zenodo. The following dated example is
|
||||
the exact asset inspected for version 0.5; check the [current ROR release page](https://ror.readme.io/docs/data-dump)
|
||||
before acquiring a newer snapshot and use the checksum published by that Zenodo
|
||||
record.
|
||||
|
||||
```bash
|
||||
argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \
|
||||
--source ror --format ror-zip \
|
||||
--url https://zenodo.org/api/records/22099990/files/v2.12-2026-08-25-ror-data.zip/content \
|
||||
--snapshot v2.12-2026-08-25 --scope full --maximum-bytes 100000000 \
|
||||
--coverage "$ARGAND_SITE_DATA/full-coverage.json" \
|
||||
--provider-checksum md5:ce8807691455d4ada3216c31408e9e1a \
|
||||
--provider-checksum-url https://zenodo.org/api/records/22099990 \
|
||||
> "$ARGAND_SITE_DATA/ror-download.json"
|
||||
argand-site-registry import --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||
--input "$(jq -r .input "$ARGAND_SITE_DATA/ror-download.json")" \
|
||||
--manifest "$(jq -r .manifest "$ARGAND_SITE_DATA/ror-download.json")"
|
||||
```
|
||||
|
||||
The adapter accepts the official schema 2.1 JSON plus CSV ZIP, consumes only the
|
||||
documented fields, and fails on extra members or schema drift. ROR entities remain
|
||||
separate from Wikidata unless reviewed equivalence evidence joins exact stable
|
||||
IDs. Inactive and withdrawn organizations remain audit evidence but their website
|
||||
edges are ineligible for resolution. GeoNames is declared as upstream lineage for
|
||||
location metadata and is included in attribution output.
|
||||
|
||||
To reuse an already acquired file, retain its **original** retrieval time,
|
||||
source URL and snapshot/revision. First verify its acquisition receipt, then:
|
||||
|
||||
|
|
@ -198,6 +239,7 @@ candidate and inspect its deterministic work queue:
|
|||
```bash
|
||||
argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||
--output "$ARGAND_SITE_DATA/candidate" \
|
||||
--audit-store "$ARGAND_SITE_DATA/audit-objects" \
|
||||
--reviewer-trust /secure/reviewer-allowed-signers \
|
||||
> "$ARGAND_SITE_DATA/candidate.json"
|
||||
export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/candidate.json")"
|
||||
|
|
@ -309,7 +351,24 @@ rather than admitted-route lists.
|
|||
argand-site-registry export --generation "$ARGAND_SITE_DATA/reviewed" \
|
||||
--pin "$REVIEWED_PIN" --output "$ARGAND_SITE_DATA/active.jsonl"
|
||||
argand-site-registry export-audit --generation "$ARGAND_SITE_DATA/reviewed" \
|
||||
--pin "$REVIEWED_PIN" --output "$ARGAND_SITE_DATA/audit.jsonl"
|
||||
--pin "$REVIEWED_PIN" --audit-store "$ARGAND_SITE_DATA/audit-objects" \
|
||||
--output "$ARGAND_SITE_DATA/audit.jsonl"
|
||||
argand-site-registry verify-audit --generation "$ARGAND_SITE_DATA/reviewed" \
|
||||
--pin "$REVIEWED_PIN" --audit-store "$ARGAND_SITE_DATA/audit-objects"
|
||||
argand-site-registry audit-checkpoint \
|
||||
--generation "$ARGAND_SITE_DATA/reviewed" --pin "$REVIEWED_PIN" \
|
||||
--audit-store "$ARGAND_SITE_DATA/audit-objects" \
|
||||
--recorded-at 2026-09-13T00:00:00Z \
|
||||
--output "$ARGAND_SITE_DATA/audit-retention.json"
|
||||
argand-site-registry sign-audit-checkpoint \
|
||||
--input "$ARGAND_SITE_DATA/audit-retention.json" \
|
||||
--signature "$ARGAND_SITE_DATA/audit-retention.json.sig" \
|
||||
--key /secure/audit-retention-key
|
||||
argand-site-registry verify-audit-checkpoint \
|
||||
--input "$ARGAND_SITE_DATA/audit-retention.json" \
|
||||
--signature "$ARGAND_SITE_DATA/audit-retention.json.sig" \
|
||||
--allowed-signers /secure/audit-retention-allowed-signers \
|
||||
--identity registry-retention
|
||||
argand-site-registry sign --generation "$ARGAND_SITE_DATA/reviewed" \
|
||||
--pin "$REVIEWED_PIN" --key /secure/publisher-key \
|
||||
--allowed-reviewers /secure/reviewer-allowed-signers \
|
||||
|
|
@ -349,7 +408,8 @@ time bounds. No scheduled command approves, renews, signs or activates.
|
|||
|
||||
## Storage and operating limits
|
||||
|
||||
Migrations 001 through 005 own writer schema 5. Source manifests v2 declare typed
|
||||
Migrations 001 through 005 own writer schema 5. Source manifests v3 authenticate
|
||||
integrity checks, per-record bounds and declared lineage in addition to typed
|
||||
full, partition, or delta coverage. Deltas name an exact base, consecutive sequence
|
||||
and superseded object. Ambiguous coverage, overlap, cycles, gaps, cross-provider
|
||||
supersession and duplicate native records fail the build. V1 manifests remain
|
||||
|
|
@ -358,7 +418,12 @@ isolated by provider and scope for compatibility.
|
|||
Each generation contains `registry.sqlite`, `LICENSE_SOURCES.md`,
|
||||
`ATTRIBUTION.json`, and `COMPLETE.json`, plus an optional publisher signature. The
|
||||
receipt binds database bytes, source selection, policy, reviewer trust, licenses,
|
||||
attribution, and decision-time contract. Keep source objects, writer state,
|
||||
attribution, and decision-time contract. A compact v3 receipt also binds every
|
||||
cold audit object by digest, length, counts, source, coverage and attribution.
|
||||
The runtime database omits superseded raw history after the bundle is durable;
|
||||
`verify-audit` must pass wherever that history is retained. The retention
|
||||
checkpoint authorizes no deletion and uses the separate
|
||||
`argand-site-registry-audit-retention` SSH namespace. Keep source objects, writer state,
|
||||
generations, pins, trust files and signatures for recovery.
|
||||
|
||||
Imports are streaming, transactional, resumable and idempotent. Defaults bound
|
||||
|
|
|
|||
|
|
@ -7,10 +7,13 @@ use serde_json::json;
|
|||
use std::io::{BufRead, Read};
|
||||
pub(crate) mod csv_sources;
|
||||
mod curlie;
|
||||
mod ror;
|
||||
mod wikidata;
|
||||
|
||||
/// Maximum decompressed size of one entity, row, or metadata document.
|
||||
pub const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024;
|
||||
/// Largest explicitly authenticated record limit supported by audit bundles.
|
||||
pub const MAX_CUSTOM_RECORD_BYTES: usize = 256 * 1024 * 1024;
|
||||
|
||||
/// Transactional sink; an emitted record and all its facts share one checkpoint.
|
||||
pub trait RecordSink {
|
||||
|
|
@ -32,14 +35,29 @@ pub trait SourceAdapter {
|
|||
|
||||
/// Chooses the explicit source format adapter.
|
||||
#[must_use]
|
||||
pub fn adapter(format: Format) -> Box<dyn SourceAdapter> {
|
||||
pub fn adapter(
|
||||
format: Format,
|
||||
maximum_record_bytes: usize,
|
||||
coverage_delta: bool,
|
||||
) -> Box<dyn SourceAdapter> {
|
||||
match format {
|
||||
Format::WikidataDump => Box::new(wikidata::Wikidata { dump: true }),
|
||||
Format::WikidataEntities => Box::new(wikidata::Wikidata { dump: false }),
|
||||
Format::WikidataDump => Box::new(wikidata::Wikidata {
|
||||
dump: true,
|
||||
maximum_record_bytes,
|
||||
retire_empty: coverage_delta,
|
||||
}),
|
||||
Format::WikidataEntities => Box::new(wikidata::Wikidata {
|
||||
dump: false,
|
||||
maximum_record_bytes,
|
||||
retire_empty: true,
|
||||
}),
|
||||
Format::MajesticCsv => Box::new(csv_sources::CsvSource { crux: false }),
|
||||
Format::CruxCsv => Box::new(csv_sources::CsvSource { crux: true }),
|
||||
Format::CurlieTarGz => Box::new(curlie::Curlie),
|
||||
Format::PslText => Box::new(PslAdapter),
|
||||
Format::RorZip => Box::new(ror::Ror {
|
||||
maximum_record_bytes,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
290
crates/argand-site-registry/src/adapters/ror.rs
Normal file
290
crates/argand-site-registry/src/adapters/ror.rs
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
// By Nic Weyand!
|
||||
//! Official ROR schema 2.1 JSON from a versioned release ZIP.
|
||||
|
||||
use super::{RecordSink, SourceAdapter};
|
||||
use crate::model::{Fact, Record, Source, entity_id};
|
||||
use anyhow::{Context, ensure};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
io::{BufRead, BufReader, Read},
|
||||
};
|
||||
|
||||
pub(super) struct Ror {
|
||||
pub maximum_record_bytes: usize,
|
||||
}
|
||||
|
||||
impl SourceAdapter for Ror {
|
||||
#[allow(clippy::case_sensitive_file_extension_comparisons)] // Provider member names are exact.
|
||||
fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> {
|
||||
let mut stream = input;
|
||||
let mut json_name = None;
|
||||
let mut csv_name = None;
|
||||
let mut entries = 0_u8;
|
||||
loop {
|
||||
let Some(mut entry) = zip::read::read_zipfile_from_stream(&mut stream)? else {
|
||||
break;
|
||||
};
|
||||
entries = entries.checked_add(1).context("too many ROR ZIP members")?;
|
||||
ensure!(
|
||||
entries <= 2,
|
||||
"ROR ZIP must contain exactly JSON and CSV files"
|
||||
);
|
||||
ensure!(
|
||||
entry.is_file() && !entry.encrypted(),
|
||||
"ROR ZIP members must be unencrypted regular files"
|
||||
);
|
||||
ensure!(
|
||||
matches!(
|
||||
entry.compression(),
|
||||
zip::CompressionMethod::Deflated | zip::CompressionMethod::Stored
|
||||
),
|
||||
"unsupported ROR ZIP compression"
|
||||
);
|
||||
let name = entry.name().to_owned();
|
||||
ensure!(
|
||||
!name.contains(['/', '\\'])
|
||||
&& name.len() <= 128
|
||||
&& name.starts_with('v')
|
||||
&& name.contains("-ror-data."),
|
||||
"unexpected ROR ZIP member name"
|
||||
);
|
||||
if name.ends_with(".json") {
|
||||
ensure!(
|
||||
json_name.replace(name).is_none(),
|
||||
"duplicate ROR JSON member"
|
||||
);
|
||||
ensure!(
|
||||
entry.size() <= 2 * 1024 * 1024 * 1024,
|
||||
"ROR JSON member exceeds 2 GiB"
|
||||
);
|
||||
let mut limited = (&mut entry).take(2 * 1024 * 1024 * 1024 + 1);
|
||||
let mut reader = BufReader::new(&mut limited);
|
||||
crate::json::for_each_array(&mut reader, self.maximum_record_bytes, |record| {
|
||||
sink.emit(project(record)?)
|
||||
})?;
|
||||
drop(reader);
|
||||
ensure!(limited.limit() > 0, "ROR JSON member exceeds 2 GiB");
|
||||
} else if name.ends_with(".csv") {
|
||||
ensure!(csv_name.replace(name).is_none(), "duplicate ROR CSV member");
|
||||
ensure!(
|
||||
entry.size() <= 1024 * 1024 * 1024,
|
||||
"ROR CSV member exceeds 1 GiB"
|
||||
);
|
||||
let copied = std::io::copy(
|
||||
&mut (&mut entry).take(1024 * 1024 * 1024 + 1),
|
||||
&mut std::io::sink(),
|
||||
)?;
|
||||
ensure!(copied <= 1024 * 1024 * 1024, "ROR CSV member exceeds 1 GiB");
|
||||
} else {
|
||||
anyhow::bail!("unrecognized ROR ZIP member");
|
||||
}
|
||||
}
|
||||
let json_name = json_name.context("ROR ZIP has no JSON member")?;
|
||||
let csv_name = csv_name.context("ROR ZIP has no CSV member")?;
|
||||
ensure!(entries == 2, "ROR ZIP must contain exactly two members");
|
||||
ensure!(
|
||||
json_name.strip_suffix(".json") == csv_name.strip_suffix(".csv"),
|
||||
"ROR JSON and CSV release names differ"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)] // Keep the exact reviewed schema mapping together.
|
||||
fn project(raw: Value) -> anyhow::Result<Record> {
|
||||
let object = raw.as_object().context("ROR record must be an object")?;
|
||||
let expected = BTreeSet::from([
|
||||
"admin",
|
||||
"domains",
|
||||
"established",
|
||||
"external_ids",
|
||||
"id",
|
||||
"links",
|
||||
"locations",
|
||||
"names",
|
||||
"relationships",
|
||||
"status",
|
||||
"types",
|
||||
]);
|
||||
ensure!(
|
||||
object.keys().map(String::as_str).collect::<BTreeSet<_>>() == expected,
|
||||
"ROR schema differs from inspected schema 2.1"
|
||||
);
|
||||
let id = raw["id"].as_str().context("ROR ID must be text")?;
|
||||
let parsed = url::Url::parse(id)?;
|
||||
ensure!(
|
||||
parsed.scheme() == "https"
|
||||
&& parsed.host_str() == Some("ror.org")
|
||||
&& parsed.query().is_none()
|
||||
&& parsed.fragment().is_none()
|
||||
&& parsed.path().len() == 10
|
||||
&& parsed.path().as_bytes()[0] == b'/'
|
||||
&& parsed.path()[1..]
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()),
|
||||
"invalid ROR ID"
|
||||
);
|
||||
validate_admin(&raw["admin"])?;
|
||||
let status = raw["status"].as_str().context("ROR status must be text")?;
|
||||
ensure!(
|
||||
matches!(status, "active" | "inactive" | "withdrawn"),
|
||||
"unknown ROR status"
|
||||
);
|
||||
let subject = entity_id(Source::Ror, id);
|
||||
let mut facts = Vec::new();
|
||||
let names = raw["names"]
|
||||
.as_array()
|
||||
.context("ROR names must be an array")?;
|
||||
ensure!(!names.is_empty(), "ROR record has no names");
|
||||
for (index, name) in names.iter().enumerate() {
|
||||
let text = name["value"]
|
||||
.as_str()
|
||||
.context("ROR name value must be text")?;
|
||||
let language = match &name["lang"] {
|
||||
Value::Null => "und",
|
||||
Value::String(value) => value,
|
||||
_ => anyhow::bail!("ROR name language must be text or null"),
|
||||
};
|
||||
let types = strings(&name["types"], "ROR name types")?;
|
||||
ensure!(
|
||||
!types.is_empty()
|
||||
&& types.iter().all(|value| matches!(
|
||||
value.as_str(),
|
||||
"acronym" | "alias" | "label" | "ror_display"
|
||||
)),
|
||||
"unknown ROR name type"
|
||||
);
|
||||
let kind = if types
|
||||
.iter()
|
||||
.any(|value| matches!(value.as_str(), "label" | "ror_display"))
|
||||
{
|
||||
"label"
|
||||
} else {
|
||||
"alias"
|
||||
};
|
||||
facts.push(Fact {
|
||||
subject: subject.clone(),
|
||||
predicate: "name".into(),
|
||||
value: json!({"text":text,"language":language,"kind":kind,"native_id":id,"ror_types":types}),
|
||||
selector: format!("/names/{index}"),
|
||||
confidence: 7500,
|
||||
});
|
||||
}
|
||||
for (index, link) in raw["links"]
|
||||
.as_array()
|
||||
.context("ROR links must be an array")?
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let kind = link["type"]
|
||||
.as_str()
|
||||
.context("ROR link type must be text")?;
|
||||
let value = link["value"]
|
||||
.as_str()
|
||||
.context("ROR link value must be text")?;
|
||||
ensure!(
|
||||
matches!(kind, "website" | "wikipedia"),
|
||||
"unknown ROR link type"
|
||||
);
|
||||
facts.push(Fact {
|
||||
subject: subject.clone(),
|
||||
predicate: if kind == "website" { "website" } else { "external_id" }.into(),
|
||||
value: if kind == "website" {
|
||||
json!({"url":value,"native_id":id,"active":status=="active","status":status,"statement":{"source":"ror_schema_2.1","link_type":"website"}})
|
||||
} else {
|
||||
json!({"type":"wikipedia","value":value,"native_id":id})
|
||||
},
|
||||
selector: format!("/links/{index}"),
|
||||
confidence: 7500,
|
||||
});
|
||||
}
|
||||
for (index, domain) in strings(&raw["domains"], "ROR domains")?.iter().enumerate() {
|
||||
facts.push(Fact {
|
||||
subject: subject.clone(),
|
||||
predicate: "declared_domain".into(),
|
||||
value: json!({"domain":domain,"native_id":id,"status":status}),
|
||||
selector: format!("/domains/{index}"),
|
||||
confidence: 7000,
|
||||
});
|
||||
}
|
||||
for (index, external) in raw["external_ids"]
|
||||
.as_array()
|
||||
.context("ROR external IDs must be an array")?
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let kind = external["type"]
|
||||
.as_str()
|
||||
.context("ROR external ID type must be text")?;
|
||||
ensure!(
|
||||
matches!(kind, "fundref" | "grid" | "isni" | "wikidata"),
|
||||
"unknown ROR external ID type"
|
||||
);
|
||||
let _ = strings(&external["all"], "ROR external ID values")?;
|
||||
ensure!(
|
||||
external["preferred"].is_null() || external["preferred"].is_string(),
|
||||
"invalid ROR preferred external ID"
|
||||
);
|
||||
facts.push(Fact {
|
||||
subject: subject.clone(),
|
||||
predicate: "external_id".into(),
|
||||
value: external.clone(),
|
||||
selector: format!("/external_ids/{index}"),
|
||||
confidence: 8000,
|
||||
});
|
||||
}
|
||||
let mut countries = BTreeSet::new();
|
||||
for location in raw["locations"]
|
||||
.as_array()
|
||||
.context("ROR locations must be an array")?
|
||||
{
|
||||
let code = location
|
||||
.pointer("/geonames_details/country_code")
|
||||
.and_then(Value::as_str)
|
||||
.context("ROR location country code missing")?;
|
||||
ensure!(
|
||||
code.len() == 2 && code.bytes().all(|byte| byte.is_ascii_uppercase()),
|
||||
"invalid ROR country code"
|
||||
);
|
||||
countries.insert(code);
|
||||
}
|
||||
facts.push(Fact {
|
||||
subject: subject.clone(), predicate: "entity_metadata".into(),
|
||||
value: json!({"status":status,"types":strings(&raw["types"], "ROR organization types")?,"countries":countries,"established":raw["established"],"admin":raw["admin"]}),
|
||||
selector: String::new(), confidence: 7500,
|
||||
});
|
||||
Ok(Record {
|
||||
native_id: id.into(),
|
||||
raw,
|
||||
facts,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_admin(admin: &Value) -> anyhow::Result<()> {
|
||||
for key in ["created", "last_modified"] {
|
||||
ensure!(admin[key]["date"].is_string(), "ROR admin date missing");
|
||||
let version = admin[key]["schema_version"]
|
||||
.as_str()
|
||||
.context("ROR admin schema version missing")?;
|
||||
ensure!(
|
||||
matches!(version, "1.0" | "2.0" | "2.1"),
|
||||
"unsupported ROR schema version"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn strings(value: &Value, field: &str) -> anyhow::Result<Vec<String>> {
|
||||
value
|
||||
.as_array()
|
||||
.with_context(|| format!("{field} must be an array"))?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
value
|
||||
.as_str()
|
||||
.map(str::to_owned)
|
||||
.with_context(|| format!("{field} must contain text"))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
// By Nic Weyand!
|
||||
//! Wikibase JSON, retaining complete statement qualifiers, references, and rank.
|
||||
|
||||
use super::{MAX_RECORD_BYTES, RecordSink, SourceAdapter, bounded_line};
|
||||
use super::{RecordSink, SourceAdapter};
|
||||
use crate::model::{Fact, Record, Source, entity_id};
|
||||
use anyhow::{Context, ensure};
|
||||
use serde_json::{Value, json};
|
||||
|
|
@ -9,6 +9,8 @@ use std::io::{BufRead, Read};
|
|||
|
||||
pub(super) struct Wikidata {
|
||||
pub dump: bool,
|
||||
pub maximum_record_bytes: usize,
|
||||
pub retire_empty: bool,
|
||||
}
|
||||
|
||||
impl SourceAdapter for Wikidata {
|
||||
|
|
@ -16,11 +18,11 @@ impl SourceAdapter for Wikidata {
|
|||
if !self.dump {
|
||||
let mut raw = Vec::new();
|
||||
input
|
||||
.take((MAX_RECORD_BYTES + 1) as u64)
|
||||
.take((self.maximum_record_bytes + 1) as u64)
|
||||
.read_to_end(&mut raw)?;
|
||||
ensure!(
|
||||
raw.len() <= MAX_RECORD_BYTES,
|
||||
"entity response exceeds 16 MiB; use dump format"
|
||||
raw.len() <= self.maximum_record_bytes,
|
||||
"entity response exceeds authenticated record bound"
|
||||
);
|
||||
let value = crate::json::parse(&raw)?;
|
||||
for (key, entity) in value["entities"]
|
||||
|
|
@ -35,33 +37,14 @@ impl SourceAdapter for Wikidata {
|
|||
}
|
||||
return Ok(());
|
||||
}
|
||||
let mut line = String::new();
|
||||
bounded_line(input, &mut line)?;
|
||||
ensure!(line.trim() == "[", "dump must begin with [");
|
||||
let mut seen = false;
|
||||
let mut comma = false;
|
||||
loop {
|
||||
ensure!(
|
||||
bounded_line(input, &mut line)? > 0,
|
||||
"truncated Wikidata dump"
|
||||
);
|
||||
let row = line.trim();
|
||||
if row.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if row == "]" {
|
||||
ensure!(!comma && seen, "empty dump or trailing comma");
|
||||
while bounded_line(input, &mut line)? > 0 {
|
||||
ensure!(line.trim().is_empty(), "data after dump");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(!seen || comma, "missing entity separator");
|
||||
comma = row.ends_with(',');
|
||||
let entity = crate::json::parse(row.strip_suffix(',').unwrap_or(row).as_bytes())?;
|
||||
project(&entity, sink, false)?;
|
||||
crate::json::for_each_array(input, self.maximum_record_bytes, |entity| {
|
||||
project(&entity, sink, self.retire_empty)?;
|
||||
seen = true;
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
ensure!(seen, "empty Wikidata dump");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
734
crates/argand-site-registry/src/audit.rs
Normal file
734
crates/argand-site-registry/src/audit.rs
Normal file
|
|
@ -0,0 +1,734 @@
|
|||
// By Nic Weyand!
|
||||
//! Content-addressed cold audit bundles for source records and facts.
|
||||
|
||||
use anyhow::{Context, ensure};
|
||||
use rusqlite::Connection;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{BufRead, BufReader, BufWriter, Read, Seek, SeekFrom, Write},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
const BUNDLE_SCHEMA: &str = "argand.site-audit-bundle/v2";
|
||||
const MAXIMUM_LINE_BYTES: usize = 512 * 1024 * 1024;
|
||||
const MAXIMUM_HEADER_BYTES: usize = 1024 * 1024;
|
||||
const MAXIMUM_CHECKPOINT_BYTES: usize = 16 * 1024 * 1024;
|
||||
/// OpenSSH namespace for operator-authenticated retention checkpoints.
|
||||
pub const RETENTION_SIGNATURE_NAMESPACE: &str = "argand-site-registry-audit-retention";
|
||||
|
||||
/// Hash-bound external audit object referenced by a compact generation.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BundleReference {
|
||||
/// Audit bundle format.
|
||||
pub schema: String,
|
||||
/// Exact source-manifest identity represented by this bundle.
|
||||
pub source_id: String,
|
||||
/// SHA-256 of the complete bundle bytes and its mirror-independent locator.
|
||||
pub object_sha256: String,
|
||||
/// Exact bundle byte length.
|
||||
pub bytes: u64,
|
||||
/// Indexed source records in the bundle.
|
||||
pub records: u64,
|
||||
/// Indexed facts in the bundle.
|
||||
pub facts: u64,
|
||||
/// Whether this source participates in the selected coverage graph.
|
||||
pub selected: bool,
|
||||
/// Attribution document digest copied into the bundle header.
|
||||
pub attribution_sha256: String,
|
||||
/// Coverage projection digest under which selection states were calculated.
|
||||
pub coverage_sha256: String,
|
||||
}
|
||||
|
||||
impl BundleReference {
|
||||
/// Stable filename derived only from authenticated content.
|
||||
#[must_use]
|
||||
pub fn filename(&self) -> String {
|
||||
format!("{}.audit.jsonl", self.object_sha256)
|
||||
}
|
||||
|
||||
/// Checks reference syntax before touching an external store.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects unsupported schemas, invalid hashes, empty bundles, or impossible counts.
|
||||
pub fn validate(&self) -> anyhow::Result<()> {
|
||||
ensure!(
|
||||
self.schema == BUNDLE_SCHEMA,
|
||||
"unsupported audit bundle schema"
|
||||
);
|
||||
ensure!(
|
||||
crate::model::valid_digest(&self.source_id)
|
||||
&& crate::model::valid_digest(&self.object_sha256)
|
||||
&& crate::model::valid_digest(&self.attribution_sha256)
|
||||
&& crate::model::valid_digest(&self.coverage_sha256),
|
||||
"invalid audit bundle digest"
|
||||
);
|
||||
ensure!(self.bytes > 0 && self.records > 0, "empty audit bundle");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Aggregate verification result for one generation's complete cold history.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct VerificationReport {
|
||||
/// Report format.
|
||||
pub schema: String,
|
||||
/// Generation receipt pin.
|
||||
pub registry: String,
|
||||
/// Bundles successfully verified.
|
||||
pub bundles: u64,
|
||||
/// Complete verified bytes.
|
||||
pub bytes: u64,
|
||||
/// Complete verified record count.
|
||||
pub records: u64,
|
||||
/// Complete verified fact count.
|
||||
pub facts: u64,
|
||||
}
|
||||
|
||||
/// Signed-report payload proving which cold objects must remain available.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RetentionCheckpoint {
|
||||
/// Report format.
|
||||
pub schema: String,
|
||||
/// Exact generation receipt pin whose history is retained.
|
||||
pub registry: String,
|
||||
/// Operator-selected report time.
|
||||
pub recorded_at: chrono::DateTime<chrono::Utc>,
|
||||
/// All authenticated objects required by this generation.
|
||||
pub retain: Vec<BundleReference>,
|
||||
/// Explicitly false: a checkpoint never authorizes deletion.
|
||||
pub deletion_authorized: bool,
|
||||
}
|
||||
|
||||
impl RetentionCheckpoint {
|
||||
fn validate(&self) -> anyhow::Result<()> {
|
||||
ensure!(
|
||||
self.schema == "argand.site-audit-retention/v1",
|
||||
"unsupported audit retention checkpoint"
|
||||
);
|
||||
ensure!(
|
||||
crate::model::valid_digest(&self.registry)
|
||||
&& !self.retain.is_empty()
|
||||
&& !self.deletion_authorized,
|
||||
"invalid audit retention checkpoint"
|
||||
);
|
||||
let mut sources = std::collections::BTreeSet::new();
|
||||
for reference in &self.retain {
|
||||
reference.validate()?;
|
||||
ensure!(
|
||||
sources.insert(&reference.source_id),
|
||||
"duplicate source in retention checkpoint"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes or reuses one deterministic bundle for every complete source snapshot.
|
||||
pub(crate) fn package_all(
|
||||
db: &Connection,
|
||||
store: &Path,
|
||||
attribution_sha256: &str,
|
||||
coverage_sha256: &str,
|
||||
) -> anyhow::Result<Vec<BundleReference>> {
|
||||
std::fs::create_dir_all(store)?;
|
||||
ensure!(
|
||||
std::fs::symlink_metadata(store)?.is_dir(),
|
||||
"audit store must be a real directory"
|
||||
);
|
||||
let mut statement = db.prepare(
|
||||
"SELECT s.id,s.manifest,EXISTS(SELECT 1 FROM selected_sources a WHERE a.id=s.id) \
|
||||
FROM sources s WHERE s.complete=1 ORDER BY s.id",
|
||||
)?;
|
||||
let sources = statement
|
||||
.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, bool>(2)?,
|
||||
))
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let mut references = Vec::with_capacity(sources.len());
|
||||
for (source_id, manifest_json, selected) in sources {
|
||||
let manifest: crate::model::SourceManifest = serde_json::from_str(&manifest_json)?;
|
||||
ensure!(
|
||||
manifest.id()? == source_id,
|
||||
"stored source identity mismatch"
|
||||
);
|
||||
let (records, facts) = source_counts(db, &source_id)?;
|
||||
if let Some(reference) = find_existing(
|
||||
store,
|
||||
&source_id,
|
||||
selected,
|
||||
attribution_sha256,
|
||||
coverage_sha256,
|
||||
records,
|
||||
facts,
|
||||
)? {
|
||||
references.push(reference);
|
||||
continue;
|
||||
}
|
||||
let temporary = tempfile::NamedTempFile::new_in(store)?;
|
||||
let mut file = temporary.reopen()?;
|
||||
let written = write_bundle(
|
||||
db,
|
||||
&mut file,
|
||||
&source_id,
|
||||
&manifest,
|
||||
selected,
|
||||
attribution_sha256,
|
||||
coverage_sha256,
|
||||
records,
|
||||
facts,
|
||||
)?;
|
||||
ensure!(written == (records, facts), "audit source counts changed");
|
||||
file.sync_all()?;
|
||||
let bytes = file.metadata()?.len();
|
||||
drop(file);
|
||||
let object_sha256 = crate::file_digest(temporary.path())?;
|
||||
let reference = BundleReference {
|
||||
schema: BUNDLE_SCHEMA.into(),
|
||||
source_id,
|
||||
object_sha256,
|
||||
bytes,
|
||||
records,
|
||||
facts,
|
||||
selected,
|
||||
attribution_sha256: attribution_sha256.into(),
|
||||
coverage_sha256: coverage_sha256.into(),
|
||||
};
|
||||
reference.validate()?;
|
||||
let destination = store.join(reference.filename());
|
||||
match temporary.persist_noclobber(&destination) {
|
||||
Ok(_) => File::open(store)?.sync_all()?,
|
||||
Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
verify_one(&reference, store)?;
|
||||
}
|
||||
Err(error) => return Err(error.error.into()),
|
||||
}
|
||||
verify_one(&reference, store)?;
|
||||
references.push(reference);
|
||||
}
|
||||
ensure!(!references.is_empty(), "no complete sources to package");
|
||||
Ok(references)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)] // Every authenticated header coordinate is explicit.
|
||||
fn write_bundle(
|
||||
db: &Connection,
|
||||
file: &mut File,
|
||||
source_id: &str,
|
||||
manifest: &crate::model::SourceManifest,
|
||||
selected: bool,
|
||||
attribution_sha256: &str,
|
||||
coverage_sha256: &str,
|
||||
expected_records: u64,
|
||||
expected_facts: u64,
|
||||
) -> anyhow::Result<(u64, u64)> {
|
||||
let mut writer = BufWriter::new(file);
|
||||
line(
|
||||
&mut writer,
|
||||
&json!({"type":"manifest","schema":BUNDLE_SCHEMA,"source_id":source_id,"source":manifest,"selected":selected,"rules":crate::store::RULE_VERSION,"attribution_sha256":attribution_sha256,"coverage_sha256":coverage_sha256,"records":expected_records,"facts":expected_facts}),
|
||||
)?;
|
||||
let mut records = 0_u64;
|
||||
let mut record_query = db.prepare(
|
||||
"SELECT r.ordinal,r.native_id,r.raw_json,(SELECT count(*) FROM facts f WHERE f.source_id=r.source_id AND f.ordinal=r.ordinal) FROM records r WHERE r.source_id=?1 ORDER BY r.ordinal",
|
||||
)?;
|
||||
let mut rows = record_query.query([source_id])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
records += 1;
|
||||
line(
|
||||
&mut writer,
|
||||
&json!({"type":"record","source_id":source_id,"ordinal":crate::store::unsigned(row,0)?,"native_id":row.get::<_,String>(1)?,"raw":serde_json::from_str::<Value>(&row.get::<_,String>(2)?)?,"facts":crate::store::unsigned(row,3)?}),
|
||||
)?;
|
||||
}
|
||||
drop(rows);
|
||||
drop(record_query);
|
||||
let mut facts = 0_u64;
|
||||
let mut fact_query = db.prepare(
|
||||
"SELECT f.id,f.ordinal,f.subject,f.predicate,f.value,f.selector,f.confidence,\
|
||||
CASE WHEN j.fact IS NOT NULL THEN 'rejected' WHEN a.source_id IS NULL THEN 'superseded' ELSE 'active' END,j.reason \
|
||||
,r.native_id FROM facts f JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal \
|
||||
LEFT JOIN active_records a ON a.source_id=f.source_id AND a.ordinal=f.ordinal \
|
||||
LEFT JOIN rejected j ON j.fact=f.id WHERE f.source_id=?1 ORDER BY f.id",
|
||||
)?;
|
||||
let mut rows = fact_query.query([source_id])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
facts += 1;
|
||||
line(
|
||||
&mut writer,
|
||||
&json!({"type":"fact","source_id":source_id,"id":row.get::<_,String>(0)?,"ordinal":crate::store::unsigned(row,1)?,"subject":row.get::<_,String>(2)?,"predicate":row.get::<_,String>(3)?,"value":serde_json::from_str::<Value>(&row.get::<_,String>(4)?)?,"selector":row.get::<_,String>(5)?,"confidence":row.get::<_,u16>(6)?,"selection_state":row.get::<_,String>(7)?,"rejection_reason":row.get::<_,Option<String>>(8)?,"native_id":row.get::<_,String>(9)?}),
|
||||
)?;
|
||||
}
|
||||
writer.flush()?;
|
||||
Ok((records, facts))
|
||||
}
|
||||
|
||||
fn source_counts(db: &Connection, source_id: &str) -> anyhow::Result<(u64, u64)> {
|
||||
Ok((
|
||||
db.query_row(
|
||||
"SELECT count(*) FROM records WHERE source_id=?1",
|
||||
[source_id],
|
||||
|row| crate::store::unsigned(row, 0),
|
||||
)?,
|
||||
db.query_row(
|
||||
"SELECT count(*) FROM facts WHERE source_id=?1",
|
||||
[source_id],
|
||||
|row| crate::store::unsigned(row, 0),
|
||||
)?,
|
||||
))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)] // All bundle identity coordinates are checked explicitly.
|
||||
fn find_existing(
|
||||
store: &Path,
|
||||
source_id: &str,
|
||||
selected: bool,
|
||||
attribution_sha256: &str,
|
||||
coverage_sha256: &str,
|
||||
records: u64,
|
||||
facts: u64,
|
||||
) -> anyhow::Result<Option<BundleReference>> {
|
||||
let mut entries = 0_u64;
|
||||
for entry in std::fs::read_dir(store)? {
|
||||
let entry = entry?;
|
||||
entries += 1;
|
||||
ensure!(entries <= 100_000, "audit store entry limit exceeded");
|
||||
let name = entry
|
||||
.file_name()
|
||||
.into_string()
|
||||
.map_err(|_| anyhow::anyhow!("non-UTF8 audit store entry"))?;
|
||||
let Some(object_sha256) = name.strip_suffix(".audit.jsonl") else {
|
||||
continue;
|
||||
};
|
||||
if !crate::model::valid_digest(object_sha256) {
|
||||
continue;
|
||||
}
|
||||
ensure!(
|
||||
entry.file_type()?.is_file(),
|
||||
"audit object must be a regular file"
|
||||
);
|
||||
let file = crate::generation::open_no_follow(&entry.path())?;
|
||||
let bytes = file.metadata()?.len();
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut buffer = Vec::new();
|
||||
let header = read_line_with_limit(&mut reader, &mut buffer, MAXIMUM_HEADER_BYTES)?
|
||||
.context("audit bundle is empty")?;
|
||||
if header["type"] != "manifest"
|
||||
|| header["schema"] != BUNDLE_SCHEMA
|
||||
|| header["source_id"] != source_id
|
||||
|| header["selected"] != selected
|
||||
|| header["attribution_sha256"] != attribution_sha256
|
||||
|| header["coverage_sha256"] != coverage_sha256
|
||||
|| header["records"] != records
|
||||
|| header["facts"] != facts
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let reference = BundleReference {
|
||||
schema: BUNDLE_SCHEMA.into(),
|
||||
source_id: source_id.into(),
|
||||
object_sha256: object_sha256.into(),
|
||||
bytes,
|
||||
records,
|
||||
facts,
|
||||
selected,
|
||||
attribution_sha256: attribution_sha256.into(),
|
||||
coverage_sha256: coverage_sha256.into(),
|
||||
};
|
||||
verify_one(&reference, store)?;
|
||||
return Ok(Some(reference));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn line(output: &mut impl Write, value: &Value) -> anyhow::Result<()> {
|
||||
serde_json::to_writer(&mut *output, value)?; // atomic-writes: allow caller supplies only unpublished or atomic output
|
||||
output.write_all(b"\n")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verifies every external bundle referenced by an already pinned generation.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects missing/substituted/truncated objects, malformed records, and attribution,
|
||||
/// coverage, source, count, or length mismatches.
|
||||
pub fn verify(
|
||||
registry: &crate::query::Registry,
|
||||
store: &Path,
|
||||
) -> anyhow::Result<VerificationReport> {
|
||||
ensure!(
|
||||
registry.receipt.runtime_layout == "compact-v1",
|
||||
"generation does not use external audit bundles"
|
||||
);
|
||||
let mut report = VerificationReport {
|
||||
schema: "argand.site-audit-verification/v1".into(),
|
||||
registry: registry.identity.clone(),
|
||||
bundles: 0,
|
||||
bytes: 0,
|
||||
records: 0,
|
||||
facts: 0,
|
||||
};
|
||||
for reference in ®istry.receipt.audit_bundles {
|
||||
verify_one(reference, store)?;
|
||||
report.bundles += 1;
|
||||
report.bytes = report
|
||||
.bytes
|
||||
.checked_add(reference.bytes)
|
||||
.context("audit byte count overflow")?;
|
||||
report.records = report
|
||||
.records
|
||||
.checked_add(reference.records)
|
||||
.context("audit record count overflow")?;
|
||||
report.facts = report
|
||||
.facts
|
||||
.checked_add(reference.facts)
|
||||
.context("audit fact count overflow")?;
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// Verifies the live store and writes a no-delete retention checkpoint.
|
||||
///
|
||||
/// The exact output can then be signed with [`sign_retention`]. No deletion
|
||||
/// operation is provided, so this tool cannot remove the only authenticated copy.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects invalid bundles, an existing output, or malformed report time.
|
||||
pub fn checkpoint(
|
||||
registry: &crate::query::Registry,
|
||||
store: &Path,
|
||||
recorded_at: chrono::DateTime<chrono::Utc>,
|
||||
output: &Path,
|
||||
) -> anyhow::Result<RetentionCheckpoint> {
|
||||
verify(registry, store)?;
|
||||
let checkpoint = RetentionCheckpoint {
|
||||
schema: "argand.site-audit-retention/v1".into(),
|
||||
registry: registry.identity.clone(),
|
||||
recorded_at,
|
||||
retain: registry.receipt.audit_bundles.clone(),
|
||||
deletion_authorized: false,
|
||||
};
|
||||
checkpoint.validate()?;
|
||||
argand_atomic::create_durable(output, &serde_json::to_vec_pretty(&checkpoint)?)?;
|
||||
Ok(checkpoint)
|
||||
}
|
||||
|
||||
/// Signs an exact, validated retention checkpoint with a distinct SSH namespace.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects malformed or oversized input, an invalid key, or an existing signature.
|
||||
pub fn sign_retention(input: &Path, signature: &Path, key: &Path) -> anyhow::Result<()> {
|
||||
let input = crate::ssh::sealed_input(input, MAXIMUM_CHECKPOINT_BYTES)?;
|
||||
let checkpoint: RetentionCheckpoint =
|
||||
serde_json::from_value(crate::json::parse(&input.bytes)?)?;
|
||||
checkpoint.validate()?;
|
||||
crate::ssh::sign(RETENTION_SIGNATURE_NAMESPACE, &input.bytes, key, signature)
|
||||
}
|
||||
|
||||
/// Verifies an exact retention checkpoint against an independent SSH trust root.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects malformed input, invalid report data, or an untrusted signature.
|
||||
pub fn verify_retention(
|
||||
input: &Path,
|
||||
signature: &Path,
|
||||
allowed_signers: &Path,
|
||||
identity: &str,
|
||||
) -> anyhow::Result<RetentionCheckpoint> {
|
||||
ensure!(
|
||||
!identity.trim().is_empty() && identity.len() <= 256,
|
||||
"retention signer identity is required and bounded"
|
||||
);
|
||||
let input = crate::ssh::sealed_input(input, MAXIMUM_CHECKPOINT_BYTES)?;
|
||||
let signature = crate::ssh::sealed_input(signature, 64 * 1024)?;
|
||||
let allowed = crate::ssh::sealed_input(allowed_signers, 1024 * 1024)?;
|
||||
let checkpoint: RetentionCheckpoint =
|
||||
serde_json::from_value(crate::json::parse(&input.bytes)?)?;
|
||||
checkpoint.validate()?;
|
||||
crate::ssh::verify(
|
||||
RETENTION_SIGNATURE_NAMESPACE,
|
||||
&input.bytes,
|
||||
&signature.bytes,
|
||||
&allowed.bytes,
|
||||
identity,
|
||||
)?;
|
||||
Ok(checkpoint)
|
||||
}
|
||||
|
||||
fn verify_one(reference: &BundleReference, store: &Path) -> anyhow::Result<()> {
|
||||
reference.validate()?;
|
||||
let path = store.join(reference.filename());
|
||||
let mut file = crate::generation::open_no_follow(&path)
|
||||
.with_context(|| format!("open audit bundle {}", path.display()))?;
|
||||
ensure!(
|
||||
file.metadata()?.is_file(),
|
||||
"audit bundle is not a regular file"
|
||||
);
|
||||
ensure!(
|
||||
file.metadata()?.len() == reference.bytes,
|
||||
"audit bundle length mismatch"
|
||||
);
|
||||
ensure!(
|
||||
reader_digest(&mut file)? == reference.object_sha256,
|
||||
"audit bundle digest mismatch"
|
||||
);
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut buffer = Vec::new();
|
||||
let header = read_line_with_limit(&mut reader, &mut buffer, MAXIMUM_HEADER_BYTES)?
|
||||
.context("audit bundle is empty")?;
|
||||
ensure!(
|
||||
header["type"] == "manifest"
|
||||
&& header["schema"] == BUNDLE_SCHEMA
|
||||
&& header["source_id"] == reference.source_id
|
||||
&& header["selected"] == reference.selected
|
||||
&& header["attribution_sha256"] == reference.attribution_sha256
|
||||
&& header["coverage_sha256"] == reference.coverage_sha256
|
||||
&& header["records"] == reference.records
|
||||
&& header["facts"] == reference.facts,
|
||||
"audit bundle header mismatch"
|
||||
);
|
||||
let manifest: crate::model::SourceManifest = serde_json::from_value(header["source"].clone())?;
|
||||
manifest.validate()?;
|
||||
ensure!(
|
||||
manifest.id()? == reference.source_id,
|
||||
"audit manifest identity mismatch"
|
||||
);
|
||||
let mut records = 0_u64;
|
||||
let mut facts = 0_u64;
|
||||
let mut in_facts = false;
|
||||
while let Some(value) = read_line(&mut reader, &mut buffer)? {
|
||||
ensure!(
|
||||
value["source_id"] == reference.source_id,
|
||||
"cross-source audit entry"
|
||||
);
|
||||
match value["type"].as_str() {
|
||||
Some("record") if !in_facts => {
|
||||
records += 1;
|
||||
ensure!(
|
||||
value["ordinal"].as_u64().is_some()
|
||||
&& value["native_id"].is_string()
|
||||
&& value["facts"].as_u64().is_some(),
|
||||
"invalid audit record index"
|
||||
);
|
||||
}
|
||||
Some("fact") => {
|
||||
in_facts = true;
|
||||
facts += 1;
|
||||
ensure!(
|
||||
value["ordinal"].as_u64().is_some()
|
||||
&& value["id"].as_str().is_some_and(crate::model::valid_digest)
|
||||
&& value["native_id"].is_string()
|
||||
&& matches!(
|
||||
value["selection_state"].as_str(),
|
||||
Some("active" | "superseded" | "rejected")
|
||||
),
|
||||
"invalid audit fact index"
|
||||
);
|
||||
}
|
||||
_ => anyhow::bail!("invalid or out-of-order audit entry"),
|
||||
}
|
||||
}
|
||||
ensure!(
|
||||
records == reference.records && facts == reference.facts,
|
||||
"audit bundle count mismatch"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_line(reader: &mut impl BufRead, buffer: &mut Vec<u8>) -> anyhow::Result<Option<Value>> {
|
||||
read_line_with_limit(reader, buffer, MAXIMUM_LINE_BYTES)
|
||||
}
|
||||
|
||||
fn read_line_with_limit(
|
||||
reader: &mut impl BufRead,
|
||||
buffer: &mut Vec<u8>,
|
||||
maximum_bytes: usize,
|
||||
) -> anyhow::Result<Option<Value>> {
|
||||
buffer.clear();
|
||||
loop {
|
||||
let (consumed, complete) = {
|
||||
let available = reader.fill_buf()?;
|
||||
if available.is_empty() {
|
||||
ensure!(buffer.is_empty(), "truncated audit bundle line");
|
||||
return Ok(None);
|
||||
}
|
||||
let consumed = available
|
||||
.iter()
|
||||
.position(|byte| *byte == b'\n')
|
||||
.map_or(available.len(), |position| position + 1);
|
||||
let length = buffer
|
||||
.len()
|
||||
.checked_add(consumed)
|
||||
.context("audit line length overflow")?;
|
||||
ensure!(length <= maximum_bytes, "audit line exceeds bound");
|
||||
buffer.extend_from_slice(&available[..consumed]);
|
||||
(consumed, available[consumed - 1] == b'\n')
|
||||
};
|
||||
reader.consume(consumed);
|
||||
if complete {
|
||||
break;
|
||||
}
|
||||
}
|
||||
buffer.pop();
|
||||
Ok(Some(crate::json::parse(buffer)?))
|
||||
}
|
||||
|
||||
fn reader_digest(reader: &mut impl Read) -> anyhow::Result<String> {
|
||||
let mut sha256 = Sha256::new();
|
||||
let mut buffer = vec![0_u8; 1024 * 1024];
|
||||
loop {
|
||||
let count = reader.read(&mut buffer)?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
sha256.update(&buffer[..count]);
|
||||
}
|
||||
Ok(format!("{:x}", sha256.finalize()))
|
||||
}
|
||||
|
||||
pub(crate) fn compact_runtime(db: &Connection) -> anyhow::Result<()> {
|
||||
// SQLite otherwise performs an unindexed reverse-FK scan for every bulk
|
||||
// deletion. The exact keep-set is constructed first and the caller requires
|
||||
// a complete foreign_key_check before publishing the receipt.
|
||||
db.execute_batch("PRAGMA foreign_keys=OFF")?;
|
||||
let result = (|| {
|
||||
db.execute_batch("BEGIN IMMEDIATE")?;
|
||||
db.execute(
|
||||
"DELETE FROM facts WHERE NOT EXISTS(SELECT 1 FROM active_records a WHERE a.source_id=facts.source_id AND a.ordinal=facts.ordinal)",
|
||||
[],
|
||||
)?;
|
||||
db.execute(
|
||||
"DELETE FROM records WHERE NOT EXISTS(SELECT 1 FROM active_records a WHERE a.source_id=records.source_id AND a.ordinal=records.ordinal)",
|
||||
[],
|
||||
)?;
|
||||
// Runtime lookup needs only projection-bearing facts. The complete source
|
||||
// rows, unprojected metadata and original raw JSON are already sealed in
|
||||
// the bundle references written before this transaction.
|
||||
db.execute_batch(
|
||||
"CREATE TEMP TABLE runtime_facts(id TEXT PRIMARY KEY) WITHOUT ROWID;
|
||||
INSERT OR IGNORE INTO runtime_facts SELECT fact FROM names;
|
||||
INSERT OR IGNORE INTO runtime_facts SELECT fact FROM popularity;
|
||||
INSERT OR IGNORE INTO runtime_facts SELECT fact FROM rejected;
|
||||
INSERT OR IGNORE INTO runtime_facts SELECT item.value FROM edges,json_each(edges.facts) item;",
|
||||
)?;
|
||||
db.execute(
|
||||
"DELETE FROM facts WHERE NOT EXISTS(SELECT 1 FROM runtime_facts r WHERE r.id=facts.id)",
|
||||
[],
|
||||
)?;
|
||||
db.execute(
|
||||
"DELETE FROM active_records WHERE NOT EXISTS(SELECT 1 FROM facts f WHERE f.source_id=active_records.source_id AND f.ordinal=active_records.ordinal)",
|
||||
[],
|
||||
)?;
|
||||
db.execute(
|
||||
"DELETE FROM records WHERE NOT EXISTS(SELECT 1 FROM facts f WHERE f.source_id=records.source_id AND f.ordinal=records.ordinal)",
|
||||
[],
|
||||
)?;
|
||||
db.execute("UPDATE records SET raw_json='null'", [])?;
|
||||
db.execute_batch("DROP TABLE runtime_facts;")?;
|
||||
db.execute(
|
||||
"DELETE FROM sources WHERE NOT EXISTS(SELECT 1 FROM selected_sources a WHERE a.id=sources.id)",
|
||||
[],
|
||||
)?;
|
||||
db.execute_batch("COMMIT")?;
|
||||
Ok::<(), anyhow::Error>(())
|
||||
})();
|
||||
let rollback = if result.is_err() && !db.is_autocommit() {
|
||||
db.execute_batch("ROLLBACK")
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
let restore = db.execute_batch("PRAGMA foreign_keys=ON");
|
||||
rollback.context("roll back failed runtime compaction")?;
|
||||
restore.context("restore foreign keys after runtime compaction")?;
|
||||
result?;
|
||||
db.execute_batch("VACUUM; ANALYZE;")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the external path for a reference without trusting a mirror name.
|
||||
#[must_use]
|
||||
pub fn path(store: &Path, reference: &BundleReference) -> PathBuf {
|
||||
store.join(reference.filename())
|
||||
}
|
||||
|
||||
/// Streams a verified compact history as an attribution-bearing audit export.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects an existing output, any invalid external bundle, or malformed evidence.
|
||||
pub fn export(
|
||||
registry: &crate::query::Registry,
|
||||
store: &Path,
|
||||
output: &Path,
|
||||
include_descriptions: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
verify(registry, store)?;
|
||||
argand_atomic::create_durable_with(output, |file| {
|
||||
export_inner(registry, store, file, include_descriptions).map_err(std::io::Error::other)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn export_inner(
|
||||
registry: &crate::query::Registry,
|
||||
store: &Path,
|
||||
file: &mut File,
|
||||
include_descriptions: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut writer = BufWriter::new(file);
|
||||
line(
|
||||
&mut writer,
|
||||
&json!({"schema":"argand.site-export/v3","registry":registry.identity,"mode":"audit","rules":registry.receipt.rules,"coverage":{"schema":"argand.site-coverage-selection/v1","sha256":registry.receipt.coverage_sha256},"audit_bundles":registry.receipt.audit_bundles,"attribution_sha256":registry.receipt.attribution_sha256,"attribution":crate::release::attribution(),"descriptions_included":include_descriptions}),
|
||||
)?;
|
||||
for reference in ®istry.receipt.audit_bundles {
|
||||
let file = crate::generation::open_no_follow(&path(store, reference))?;
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut buffer = Vec::new();
|
||||
let header = read_line_with_limit(&mut reader, &mut buffer, MAXIMUM_HEADER_BYTES)?
|
||||
.context("audit bundle is empty")?;
|
||||
let manifest = header["source"].clone();
|
||||
while let Some(mut value) = read_line(&mut reader, &mut buffer)? {
|
||||
match value["type"].as_str() {
|
||||
Some("record") if value["facts"] == 0 => {
|
||||
let replaces = manifest
|
||||
.pointer("/coverage/supersedes")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!([]));
|
||||
line(
|
||||
&mut writer,
|
||||
&json!({"type":"tombstone","selection_state":"tombstoned","source":manifest,"source_identifier":value["native_id"],"source_snapshot_id":reference.source_id,"replaces":replaces}),
|
||||
)?;
|
||||
}
|
||||
Some("record") => {}
|
||||
Some("fact") => {
|
||||
let predicate = value["predicate"]
|
||||
.as_str()
|
||||
.context("audit predicate missing")?
|
||||
.to_owned();
|
||||
if !include_descriptions && predicate == "description" {
|
||||
continue;
|
||||
}
|
||||
if !include_descriptions
|
||||
&& predicate == "category"
|
||||
&& let Some(object) = value["value"].as_object_mut()
|
||||
{
|
||||
object.remove("description");
|
||||
}
|
||||
line(
|
||||
&mut writer,
|
||||
&json!({"type":"assertion","selection_state":value["selection_state"],"rejection_reason":value["rejection_reason"],"replacement_source_ids":[],"id":value["id"],"subject":value["subject"],"predicate":predicate,"value":value["value"],"selector":value["selector"],"confidence":value["confidence"],"source":manifest,"source_identifier":value["native_id"],"source_snapshot_id":reference.source_id,"description_redacted":!include_descriptions && predicate=="category"}),
|
||||
)?;
|
||||
}
|
||||
_ => anyhow::bail!("invalid audit bundle entry during export"),
|
||||
}
|
||||
}
|
||||
}
|
||||
writer.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -35,7 +35,7 @@ CREATE TABLE review_policy(singleton INTEGER PRIMARY KEY CHECK(singleton=1),id T
|
|||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Receipt {
|
||||
/// `argand.site-registry/v2`.
|
||||
/// `argand.site-registry/v2` with embedded history or compact v3.
|
||||
pub schema: String,
|
||||
/// Parser/derivation contract.
|
||||
pub rules: String,
|
||||
|
|
@ -64,6 +64,12 @@ pub struct Receipt {
|
|||
/// Trusted writer-acceptance and bounded-expiry contract.
|
||||
#[serde(default)]
|
||||
pub decision_time_policy: String,
|
||||
/// Empty for v2 embedded history or `compact-v1` for external audit bundles.
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub runtime_layout: String,
|
||||
/// Complete content-addressed cold history for compact generations.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub audit_bundles: Vec<crate::audit::BundleReference>,
|
||||
/// Distinct entity count.
|
||||
pub entities: u64,
|
||||
/// Strict URL identity count.
|
||||
|
|
@ -104,6 +110,31 @@ pub fn build_with_policy_and_trust(
|
|||
output: &Path,
|
||||
policy: &crate::policy::ReviewPolicy,
|
||||
reviewer_trust: Option<&Path>,
|
||||
) -> anyhow::Result<Receipt> {
|
||||
build_inner(db, output, policy, reviewer_trust, None)
|
||||
}
|
||||
|
||||
/// Builds a compact runtime generation backed by a content-addressed cold audit store.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects invalid policy/trust, unsafe audit storage, corrupt evidence, existing
|
||||
/// destinations, and any failure to verify the complete cold history.
|
||||
pub fn build_compact_with_policy_and_trust(
|
||||
db: &Connection,
|
||||
output: &Path,
|
||||
policy: &crate::policy::ReviewPolicy,
|
||||
reviewer_trust: Option<&Path>,
|
||||
audit_store: &Path,
|
||||
) -> anyhow::Result<Receipt> {
|
||||
build_inner(db, output, policy, reviewer_trust, Some(audit_store))
|
||||
}
|
||||
|
||||
fn build_inner(
|
||||
db: &Connection,
|
||||
output: &Path,
|
||||
policy: &crate::policy::ReviewPolicy,
|
||||
reviewer_trust: Option<&Path>,
|
||||
audit_store: Option<&Path>,
|
||||
) -> anyhow::Result<Receipt> {
|
||||
policy.validate()?;
|
||||
let reviewer_trust_sha256 = reviewer_trust
|
||||
|
|
@ -135,6 +166,17 @@ pub fn build_with_policy_and_trust(
|
|||
project_names(&snapshot)?;
|
||||
project_facts(&snapshot, &normalizer)?;
|
||||
snapshot.execute_batch("COMMIT; ANALYZE;")?;
|
||||
let coverage_sha256 = crate::coverage::projection_digest(&snapshot)?;
|
||||
let attribution_sha256 =
|
||||
crate::digest(&serde_json::to_vec_pretty(&crate::release::attribution())?);
|
||||
let audit_bundles = if let Some(store) = audit_store {
|
||||
let bundles =
|
||||
crate::audit::package_all(&snapshot, store, &attribution_sha256, &coverage_sha256)?;
|
||||
crate::audit::compact_runtime(&snapshot)?;
|
||||
bundles
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let check: String = snapshot.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "registry integrity failed");
|
||||
let foreign_count: u64 =
|
||||
|
|
@ -149,15 +191,17 @@ pub fn build_with_policy_and_trust(
|
|||
.map(|s| Ok(serde_json::from_str(&s?)?))
|
||||
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||
drop(statement);
|
||||
let coverage_sha256 = crate::coverage::projection_digest(&snapshot)?;
|
||||
let mut receipt = Receipt {
|
||||
schema: "argand.site-registry/v2".into(),
|
||||
schema: if audit_store.is_some() {
|
||||
"argand.site-registry/v3"
|
||||
} else {
|
||||
"argand.site-registry/v2"
|
||||
}
|
||||
.into(),
|
||||
rules: store::RULE_VERSION.into(),
|
||||
database_sha256: String::new(),
|
||||
licenses_sha256: crate::digest(crate::release::LICENSES.as_bytes()),
|
||||
attribution_sha256: crate::digest(&serde_json::to_vec_pretty(
|
||||
&crate::release::attribution(),
|
||||
)?),
|
||||
attribution_sha256,
|
||||
psl_source: psl_id,
|
||||
sources,
|
||||
review_policy: policy.clone(),
|
||||
|
|
@ -165,6 +209,12 @@ pub fn build_with_policy_and_trust(
|
|||
reviewer_trust_sha256,
|
||||
coverage_sha256,
|
||||
decision_time_policy: "argand.site-decision-time/v1".into(),
|
||||
runtime_layout: if audit_store.is_some() {
|
||||
"compact-v1".into()
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
audit_bundles,
|
||||
entities: count(&snapshot, "entities")?,
|
||||
properties: count(&snapshot, "properties")?,
|
||||
edges: count(&snapshot, "edges")?,
|
||||
|
|
@ -445,7 +495,7 @@ fn project_edge(
|
|||
serde_json::to_string(&property)?
|
||||
],
|
||||
)?;
|
||||
let relation = if source == "wikidata" {
|
||||
let relation = if matches!(source, "wikidata" | "ror") {
|
||||
"asserted_official"
|
||||
} else {
|
||||
"directory_listing"
|
||||
|
|
@ -469,7 +519,8 @@ fn project_edge(
|
|||
// End-dated assertions stay as historical evidence, never current destinations.
|
||||
// Future/partial starts require the operator to inspect the retained qualifiers.
|
||||
let eligible = value.pointer("/statement/rank").and_then(Value::as_str) != Some("deprecated")
|
||||
&& value.pointer("/statement/qualifiers/P582").is_none();
|
||||
&& value.pointer("/statement/qualifiers/P582").is_none()
|
||||
&& value.get("active").and_then(Value::as_bool) != Some(false);
|
||||
db.execute("INSERT INTO edges VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(fingerprint) DO UPDATE SET facts=json_insert(edges.facts,'$[#]',?8)",params![fingerprint,entity,property.id,relation,serde_json::to_string(&vec![id])?,serde_json::to_string(&evidence)?,eligible,id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -141,9 +141,18 @@ enum Command {
|
|||
scope: String,
|
||||
#[arg(long)]
|
||||
maximum_bytes: u64,
|
||||
/// Bind a non-default per-record parsing limit into the source manifest.
|
||||
#[arg(long)]
|
||||
maximum_record_bytes: Option<u64>,
|
||||
/// JSON file containing a typed coverage declaration.
|
||||
#[arg(long)]
|
||||
coverage: Option<PathBuf>,
|
||||
/// Optional provider checksum formatted as `sha256:...` or `md5:...`.
|
||||
#[arg(long, requires = "provider_checksum_url")]
|
||||
provider_checksum: Option<String>,
|
||||
/// Authoritative provider record or checksum-file URL.
|
||||
#[arg(long, requires = "provider_checksum")]
|
||||
provider_checksum_url: Option<String>,
|
||||
},
|
||||
/// Download paginated `CrUX` data using an explicit billing configuration JSON.
|
||||
CruxDownload {
|
||||
|
|
@ -175,6 +184,9 @@ enum Command {
|
|||
/// JSON file containing a typed coverage declaration.
|
||||
#[arg(long)]
|
||||
coverage: Option<PathBuf>,
|
||||
/// Bind a non-default per-record parsing limit into the source manifest.
|
||||
#[arg(long)]
|
||||
maximum_record_bytes: Option<u64>,
|
||||
},
|
||||
/// Import a complete pinned source, resuming committed record batches.
|
||||
Import {
|
||||
|
|
@ -203,6 +215,9 @@ enum Command {
|
|||
/// Exact SSH allowed-signers trust root; required by strict policies.
|
||||
#[arg(long)]
|
||||
reviewer_trust: Option<PathBuf>,
|
||||
/// External content-addressed audit store; enables compact v3 generations.
|
||||
#[arg(long)]
|
||||
audit_store: Option<PathBuf>,
|
||||
},
|
||||
/// Audit an exact name or alias; includes ambiguity counts and attribution.
|
||||
Lookup {
|
||||
|
|
@ -537,6 +552,51 @@ enum Command {
|
|||
output: PathBuf,
|
||||
#[arg(long)]
|
||||
include_descriptions: bool,
|
||||
/// External audit store required by compact v3 generations.
|
||||
#[arg(long)]
|
||||
audit_store: Option<PathBuf>,
|
||||
},
|
||||
/// Stream-verify all cold evidence referenced by a compact generation.
|
||||
VerifyAudit {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
audit_store: PathBuf,
|
||||
},
|
||||
/// Verify cold objects and write a no-delete retention checkpoint.
|
||||
AuditCheckpoint {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
audit_store: PathBuf,
|
||||
#[arg(long)]
|
||||
recorded_at: chrono::DateTime<chrono::Utc>,
|
||||
#[arg(long)]
|
||||
output: PathBuf,
|
||||
},
|
||||
/// Sign a validated audit retention checkpoint.
|
||||
SignAuditCheckpoint {
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
#[arg(long)]
|
||||
signature: PathBuf,
|
||||
#[arg(long)]
|
||||
key: PathBuf,
|
||||
},
|
||||
/// Verify a signed audit retention checkpoint against an independent trust root.
|
||||
VerifyAuditCheckpoint {
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
#[arg(long)]
|
||||
signature: PathBuf,
|
||||
#[arg(long)]
|
||||
allowed_signers: PathBuf,
|
||||
#[arg(long)]
|
||||
identity: String,
|
||||
},
|
||||
/// Export cumulative emergency revocation state at an explicit time.
|
||||
ExportRevocations {
|
||||
|
|
@ -662,6 +722,7 @@ pub(super) async fn run() -> anyhow::Result<()> {
|
|||
| Command::Observations { .. }
|
||||
| Command::ObservationLookup { .. }
|
||||
| Command::Evaluate { .. }
|
||||
| Command::VerifyAudit { .. }
|
||||
| Command::Resolve { .. }) => inspect(command)?,
|
||||
Command::Import {
|
||||
database,
|
||||
|
|
@ -827,18 +888,30 @@ pub(super) async fn run() -> anyhow::Result<()> {
|
|||
output,
|
||||
policy,
|
||||
reviewer_trust,
|
||||
audit_store,
|
||||
} => {
|
||||
ensure!(database.is_file(), "import database does not exist");
|
||||
let policy = policy
|
||||
.map(|path| registry::read_json(&path))
|
||||
.transpose()?
|
||||
.unwrap_or_else(registry::policy::ReviewPolicy::reference);
|
||||
let receipt = registry::build::build_with_policy_and_trust(
|
||||
®istry::store::open(&database)?,
|
||||
let database = registry::store::open(&database)?;
|
||||
let receipt = if let Some(audit_store) = audit_store {
|
||||
registry::build::build_compact_with_policy_and_trust(
|
||||
&database,
|
||||
&output,
|
||||
&policy,
|
||||
reviewer_trust.as_deref(),
|
||||
)?;
|
||||
&audit_store,
|
||||
)?
|
||||
} else {
|
||||
registry::build::build_with_policy_and_trust(
|
||||
&database,
|
||||
&output,
|
||||
&policy,
|
||||
reviewer_trust.as_deref(),
|
||||
)?
|
||||
};
|
||||
serde_json::json!({"receipt":receipt,"pin":registry::file_digest(&output.join("COMPLETE.json"))?,"generation":output})
|
||||
}
|
||||
Command::Review {
|
||||
|
|
@ -893,14 +966,47 @@ pub(super) async fn run() -> anyhow::Result<()> {
|
|||
pin,
|
||||
output,
|
||||
include_descriptions,
|
||||
audit_store,
|
||||
} => {
|
||||
registry::release::export_audit(
|
||||
&Registry::open(&generation, &pin)?,
|
||||
&output,
|
||||
include_descriptions,
|
||||
)?;
|
||||
let registry = Registry::open(&generation, &pin)?;
|
||||
if let Some(audit_store) = audit_store {
|
||||
registry::audit::export(®istry, &audit_store, &output, include_descriptions)?;
|
||||
} else {
|
||||
registry::release::export_audit(®istry, &output, include_descriptions)?;
|
||||
}
|
||||
serde_json::json!({"audit_export":output})
|
||||
}
|
||||
Command::AuditCheckpoint {
|
||||
generation,
|
||||
pin,
|
||||
audit_store,
|
||||
recorded_at,
|
||||
output,
|
||||
} => serde_json::to_value(registry::audit::checkpoint(
|
||||
&Registry::open(&generation, &pin)?,
|
||||
&audit_store,
|
||||
recorded_at,
|
||||
&output,
|
||||
)?)?,
|
||||
Command::SignAuditCheckpoint {
|
||||
input,
|
||||
signature,
|
||||
key,
|
||||
} => {
|
||||
registry::audit::sign_retention(&input, &signature, &key)?;
|
||||
serde_json::json!({"checkpoint":input,"signature":signature,"signature_namespace":registry::audit::RETENTION_SIGNATURE_NAMESPACE})
|
||||
}
|
||||
Command::VerifyAuditCheckpoint {
|
||||
input,
|
||||
signature,
|
||||
allowed_signers,
|
||||
identity,
|
||||
} => serde_json::to_value(registry::audit::verify_retention(
|
||||
&input,
|
||||
&signature,
|
||||
&allowed_signers,
|
||||
&identity,
|
||||
)?)?,
|
||||
Command::ExportRevocations {
|
||||
generation,
|
||||
pin,
|
||||
|
|
@ -1226,6 +1332,14 @@ fn inspect(command: Command) -> anyhow::Result<serde_json::Value> {
|
|||
maximum_cases,
|
||||
at.unwrap_or_else(chrono::Utc::now),
|
||||
)?)?,
|
||||
Command::VerifyAudit {
|
||||
generation,
|
||||
pin,
|
||||
audit_store,
|
||||
} => serde_json::to_value(registry::audit::verify(
|
||||
&Registry::open(&generation, &pin)?,
|
||||
&audit_store,
|
||||
)?)?,
|
||||
Command::Resolve {
|
||||
generation,
|
||||
pin,
|
||||
|
|
@ -1389,7 +1503,10 @@ async fn acquire(command: Command) -> anyhow::Result<serde_json::Value> {
|
|||
snapshot,
|
||||
scope,
|
||||
maximum_bytes,
|
||||
maximum_record_bytes,
|
||||
coverage,
|
||||
provider_checksum,
|
||||
provider_checksum_url,
|
||||
} => serde_json::to_value(
|
||||
registry::download::download(
|
||||
&cache,
|
||||
|
|
@ -1401,9 +1518,12 @@ async fn acquire(command: Command) -> anyhow::Result<serde_json::Value> {
|
|||
snapshot,
|
||||
scope,
|
||||
maximum_bytes,
|
||||
maximum_record_bytes,
|
||||
coverage: coverage
|
||||
.map(|path| registry::read_json(&path))
|
||||
.transpose()?,
|
||||
provider_checksum,
|
||||
provider_checksum_url,
|
||||
},
|
||||
)
|
||||
.await?,
|
||||
|
|
@ -1422,17 +1542,14 @@ async fn acquire(command: Command) -> anyhow::Result<serde_json::Value> {
|
|||
scope,
|
||||
retrieved_at,
|
||||
coverage,
|
||||
maximum_record_bytes,
|
||||
} => {
|
||||
let coverage = coverage
|
||||
.map(|path| registry::read_json(&path))
|
||||
.transpose()?;
|
||||
let sha256 = registry::file_digest(&input)?;
|
||||
let manifest = SourceManifest {
|
||||
schema: if coverage.is_some() {
|
||||
"argand.site-source/v2"
|
||||
} else {
|
||||
"argand.site-source/v1"
|
||||
}
|
||||
.into(),
|
||||
schema: "argand.site-source/v3".into(),
|
||||
source,
|
||||
format,
|
||||
compression,
|
||||
|
|
@ -1443,8 +1560,16 @@ async fn acquire(command: Command) -> anyhow::Result<serde_json::Value> {
|
|||
retrieved_at,
|
||||
license: source.license().into(),
|
||||
license_url: source.license_url().into(),
|
||||
sha256: registry::file_digest(&input)?,
|
||||
sha256: sha256.clone(),
|
||||
bytes: input.metadata()?.len(),
|
||||
maximum_record_bytes,
|
||||
integrity: vec![registry::model::IntegrityProof {
|
||||
method: "content_digest".into(),
|
||||
algorithm: "sha256".into(),
|
||||
value: sha256,
|
||||
evidence_url: None,
|
||||
}],
|
||||
lineage: Some(registry::model::SourceLineage::direct(source)),
|
||||
};
|
||||
manifest.validate()?;
|
||||
argand_atomic::create_durable(&output, &serde_json::to_vec_pretty(&manifest)?)?;
|
||||
|
|
|
|||
|
|
@ -416,6 +416,9 @@ mod tests {
|
|||
.context("test timestamp")?,
|
||||
sha256: crate::digest(b"input"),
|
||||
bytes: 5,
|
||||
maximum_record_bytes: None,
|
||||
integrity: Vec::new(),
|
||||
lineage: None,
|
||||
};
|
||||
Ok(Snapshot {
|
||||
id,
|
||||
|
|
|
|||
|
|
@ -82,12 +82,7 @@ pub async fn download(cache: &Path, request: &CruxDownload) -> anyhow::Result<Ca
|
|||
let sha256 = crate::file_digest(&part)?;
|
||||
let input = root.join(&sha256);
|
||||
let manifest = SourceManifest {
|
||||
schema: if request.coverage.is_some() {
|
||||
"argand.site-source/v2"
|
||||
} else {
|
||||
"argand.site-source/v1"
|
||||
}
|
||||
.into(),
|
||||
schema: "argand.site-source/v3".into(),
|
||||
source: Source::Crux,
|
||||
format: Format::CruxCsv,
|
||||
compression: Compression::None,
|
||||
|
|
@ -106,8 +101,16 @@ pub async fn download(cache: &Path, request: &CruxDownload) -> anyhow::Result<Ca
|
|||
license: Source::Crux.license().into(),
|
||||
license_url: Source::Crux.license_url().into(),
|
||||
retrieved_at: Utc::now(),
|
||||
sha256,
|
||||
sha256: sha256.clone(),
|
||||
bytes: part.metadata()?.len(),
|
||||
maximum_record_bytes: None,
|
||||
integrity: vec![crate::model::IntegrityProof {
|
||||
method: "content_digest".into(),
|
||||
algorithm: "sha256".into(),
|
||||
value: sha256,
|
||||
evidence_url: None,
|
||||
}],
|
||||
lineage: Some(crate::model::SourceLineage::direct(Source::Crux)),
|
||||
};
|
||||
fs::rename(part, &input)?;
|
||||
File::open(&root)?.sync_all()?;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// By Nic Weyand!
|
||||
//! Allowlisted source acquisition with validator-bound range resume.
|
||||
|
||||
use crate::model::{Compression, Format, Source, SourceManifest};
|
||||
use crate::model::{Compression, Format, IntegrityProof, Source, SourceLineage, SourceManifest};
|
||||
use anyhow::{Context, ensure};
|
||||
use chrono::Utc;
|
||||
use reqwest::{Client, StatusCode, header};
|
||||
|
|
@ -32,9 +32,18 @@ pub struct Download {
|
|||
pub scope: String,
|
||||
/// Maximum downloaded object bytes.
|
||||
pub maximum_bytes: u64,
|
||||
/// Authenticated maximum bytes materialized for one source record.
|
||||
#[serde(default)]
|
||||
pub maximum_record_bytes: Option<u64>,
|
||||
/// Explicit full/partition/delta coverage; absent preserves legacy scope semantics.
|
||||
#[serde(default)]
|
||||
pub coverage: Option<crate::model::SourceCoverage>,
|
||||
/// Optional provider-published checksum, formatted as `algorithm:hex`.
|
||||
#[serde(default)]
|
||||
pub provider_checksum: Option<String>,
|
||||
/// Exact provider record or checksum file supplying the expected checksum.
|
||||
#[serde(default)]
|
||||
pub provider_checksum_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Result points to immutable cached bytes and their manifest.
|
||||
|
|
@ -92,6 +101,7 @@ pub fn validate_source_url(source: Source, input: &str) -> anyhow::Result<()> {
|
|||
&& path == "/curlie-rdf/curlie-rdf-all.tar.gz")
|
||||
}
|
||||
Source::Psl => host == "publicsuffix.org" && path == "/list/public_suffix_list.dat",
|
||||
Source::Ror => ror_url(host, path),
|
||||
};
|
||||
ensure!(allowed, "unreviewed source endpoint: {host}{path}");
|
||||
Ok(())
|
||||
|
|
@ -120,6 +130,7 @@ pub async fn download(cache: &Path, request: &Download) -> anyhow::Result<Cached
|
|||
);
|
||||
validate_source_url(request.source, &request.url)?;
|
||||
ensure!(request.maximum_bytes > 0, "maximum bytes must be positive");
|
||||
let requested_proof = requested_checksum_proof(request)?;
|
||||
let key = crate::digest(&serde_json::to_vec(request)?);
|
||||
let dir = cache.join(request.source.key()).join(key);
|
||||
fs::create_dir_all(&dir)?;
|
||||
|
|
@ -144,6 +155,12 @@ pub async fn download(cache: &Path, request: &Download) -> anyhow::Result<Cached
|
|||
&& manifest.bytes <= request.maximum_bytes,
|
||||
"cached source declaration differs from request"
|
||||
);
|
||||
ensure!(
|
||||
requested_proof
|
||||
.as_ref()
|
||||
.is_none_or(|proof| manifest.integrity.contains(proof)),
|
||||
"cached provider checksum differs from request"
|
||||
);
|
||||
let input = dir.join(&manifest.sha256);
|
||||
ensure!(
|
||||
crate::file_digest(&input)? == manifest.sha256
|
||||
|
|
@ -180,15 +197,35 @@ pub async fn download(cache: &Path, request: &Download) -> anyhow::Result<Cached
|
|||
if let Some(error) = last_error {
|
||||
return Err(error);
|
||||
}
|
||||
let sha256 = crate::file_digest(&part)?;
|
||||
seal_download(&dir, &part, &complete, request, requested_proof)
|
||||
}
|
||||
|
||||
fn seal_download(
|
||||
dir: &Path,
|
||||
part: &Path,
|
||||
complete: &Path,
|
||||
request: &Download,
|
||||
requested_proof: Option<IntegrityProof>,
|
||||
) -> anyhow::Result<CachedSource> {
|
||||
let sha256 = crate::file_digest(part)?;
|
||||
let bytes = part.metadata()?.len();
|
||||
let manifest = SourceManifest {
|
||||
schema: if request.coverage.is_some() {
|
||||
"argand.site-source/v2"
|
||||
} else {
|
||||
"argand.site-source/v1"
|
||||
let mut integrity = vec![IntegrityProof {
|
||||
method: "content_digest".into(),
|
||||
algorithm: "sha256".into(),
|
||||
value: sha256.clone(),
|
||||
evidence_url: None,
|
||||
}];
|
||||
if let Some(proof) = requested_proof {
|
||||
let actual = match proof.algorithm.as_str() {
|
||||
"sha256" => sha256.clone(),
|
||||
"md5" => md5_digest(part)?,
|
||||
_ => anyhow::bail!("unsupported provider checksum algorithm"),
|
||||
};
|
||||
ensure!(actual == proof.value, "provider checksum mismatch");
|
||||
integrity.push(proof);
|
||||
}
|
||||
.into(),
|
||||
let manifest = SourceManifest {
|
||||
schema: "argand.site-source/v3".into(),
|
||||
source: request.source,
|
||||
format: request.format,
|
||||
compression: request.compression,
|
||||
|
|
@ -201,18 +238,117 @@ pub async fn download(cache: &Path, request: &Download) -> anyhow::Result<Cached
|
|||
retrieved_at: Utc::now(),
|
||||
sha256: sha256.clone(),
|
||||
bytes,
|
||||
maximum_record_bytes: request.maximum_record_bytes,
|
||||
integrity,
|
||||
lineage: Some(SourceLineage::direct(request.source)),
|
||||
};
|
||||
manifest.validate()?;
|
||||
let input = dir.join(sha256);
|
||||
fs::rename(&part, &input)?;
|
||||
File::open(&dir)?.sync_all()?;
|
||||
argand_atomic::create_durable(&complete, &serde_json::to_vec_pretty(&manifest)?)?;
|
||||
fs::rename(part, &input)?;
|
||||
File::open(dir)?.sync_all()?;
|
||||
argand_atomic::create_durable(complete, &serde_json::to_vec_pretty(&manifest)?)?;
|
||||
Ok(CachedSource {
|
||||
input,
|
||||
manifest: complete,
|
||||
manifest: complete.into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn ror_url(host: &str, path: &str) -> bool {
|
||||
let components = path.trim_start_matches('/').split('/').collect::<Vec<_>>();
|
||||
host == "zenodo.org"
|
||||
&& components.len() == 6
|
||||
&& components[0] == "api"
|
||||
&& components[1] == "records"
|
||||
&& !components[2].is_empty()
|
||||
&& components[2].bytes().all(|byte| byte.is_ascii_digit())
|
||||
&& components[3] == "files"
|
||||
&& components[4].starts_with('v')
|
||||
&& components[4].ends_with("-ror-data.zip")
|
||||
&& components[5] == "content"
|
||||
}
|
||||
|
||||
fn requested_checksum_proof(request: &Download) -> anyhow::Result<Option<IntegrityProof>> {
|
||||
match (&request.provider_checksum, &request.provider_checksum_url) {
|
||||
(None, None) => Ok(None),
|
||||
(Some(expected), Some(evidence_url)) => {
|
||||
let (algorithm, value) = expected
|
||||
.split_once(':')
|
||||
.context("provider checksum must be algorithm:hex")?;
|
||||
let proof = IntegrityProof {
|
||||
method: "provider_checksum".into(),
|
||||
algorithm: algorithm.into(),
|
||||
value: value.into(),
|
||||
evidence_url: Some(evidence_url.clone()),
|
||||
};
|
||||
proof.validate()?;
|
||||
validate_integrity_evidence(request.source, &request.url, &proof)?;
|
||||
Ok(Some(proof))
|
||||
}
|
||||
(Some(_), None) => anyhow::bail!("provider checksum needs its evidence URL"),
|
||||
(None, Some(_)) => anyhow::bail!("provider checksum URL needs a checksum"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_integrity_evidence(
|
||||
source: Source,
|
||||
source_url: &str,
|
||||
proof: &IntegrityProof,
|
||||
) -> anyhow::Result<()> {
|
||||
if proof.method == "content_digest" {
|
||||
return Ok(());
|
||||
}
|
||||
let source_url = url::Url::parse(source_url)?;
|
||||
let evidence = url::Url::parse(
|
||||
proof
|
||||
.evidence_url
|
||||
.as_deref()
|
||||
.context("provider checksum needs its evidence URL")?,
|
||||
)?;
|
||||
let allowed = match source {
|
||||
Source::Ror => {
|
||||
let source_parts = source_url
|
||||
.path()
|
||||
.trim_start_matches('/')
|
||||
.split('/')
|
||||
.collect::<Vec<_>>();
|
||||
let evidence_parts = evidence
|
||||
.path()
|
||||
.trim_start_matches('/')
|
||||
.split('/')
|
||||
.collect::<Vec<_>>();
|
||||
evidence.host_str() == Some("zenodo.org")
|
||||
&& source_parts.len() == 6
|
||||
&& evidence_parts.as_slice() == ["api", "records", source_parts[2]].as_slice()
|
||||
}
|
||||
Source::Wikidata => {
|
||||
let source_parent = source_url.path().rsplit_once('/').map(|item| item.0);
|
||||
evidence.host_str() == Some("dumps.wikimedia.org")
|
||||
&& source_url.host_str() == evidence.host_str()
|
||||
&& evidence.path().rsplit_once('/').map(|item| item.0) == source_parent
|
||||
&& evidence.path().ends_with("/sha256sums.txt")
|
||||
}
|
||||
Source::Majestic | Source::Crux | Source::Curlie | Source::Psl => false,
|
||||
};
|
||||
ensure!(allowed, "unreviewed provider checksum evidence endpoint");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn md5_digest(path: &Path) -> anyhow::Result<String> {
|
||||
use md5::{Digest, Md5};
|
||||
use std::io::Read as _;
|
||||
let mut input = crate::generation::open_no_follow(path)?;
|
||||
let mut hash = Md5::new();
|
||||
let mut buffer = [0_u8; 8192];
|
||||
loop {
|
||||
let count = input.read(&mut buffer)?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
hash.update(&buffer[..count]);
|
||||
}
|
||||
Ok(format!("{:x}", hash.finalize()))
|
||||
}
|
||||
|
||||
async fn transfer(
|
||||
client: &Client,
|
||||
request: &Download,
|
||||
|
|
@ -437,4 +573,33 @@ mod tests {
|
|||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_checksums_need_source_bound_evidence() -> anyhow::Result<()> {
|
||||
let source =
|
||||
"https://zenodo.org/api/records/22099990/files/v2.12-2026-08-25-ror-data.zip/content";
|
||||
let proof = IntegrityProof {
|
||||
method: "provider_checksum".into(),
|
||||
algorithm: "md5".into(),
|
||||
value: "ce8807691455d4ada3216c31408e9e1a".into(),
|
||||
evidence_url: Some("https://zenodo.org/api/records/22099990".into()),
|
||||
};
|
||||
validate_integrity_evidence(Source::Ror, source, &proof)?;
|
||||
for url in [
|
||||
"https://example.org/api/records/22099990",
|
||||
"https://zenodo.org/api/records/999",
|
||||
"https://zenodo.org/api/records/22099990?changed=true",
|
||||
] {
|
||||
let mut bad = proof.clone();
|
||||
bad.evidence_url = Some(url.into());
|
||||
assert!(
|
||||
bad.validate().is_err()
|
||||
|| validate_integrity_evidence(Source::Ror, source, &bad).is_err()
|
||||
);
|
||||
}
|
||||
let mut unsupported = proof;
|
||||
unsupported.method = "provider_signature".into();
|
||||
assert!(unsupported.validate().is_err());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
// By Nic Weyand!
|
||||
//! Bounded, replayable destination-resolution evaluation.
|
||||
|
||||
use crate::{ResolutionStatus, query::Registry};
|
||||
use crate::{
|
||||
ResolutionStatus,
|
||||
model::{Source, SourceLineage},
|
||||
query::Registry,
|
||||
};
|
||||
use anyhow::{Context, ensure};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
|
|
@ -65,6 +69,8 @@ pub struct Report {
|
|||
pub registry: String,
|
||||
/// Explicit resolver clock used for every case.
|
||||
pub evaluated_at: chrono::DateTime<chrono::Utc>,
|
||||
/// Active source snapshots and their declared dependency groups.
|
||||
pub source_lineage: Vec<EvaluationLineage>,
|
||||
/// Parsed cases.
|
||||
pub total: u64,
|
||||
/// Cases matching every asserted field.
|
||||
|
|
@ -79,6 +85,19 @@ pub struct Report {
|
|||
pub failures: Vec<Failure>,
|
||||
}
|
||||
|
||||
/// Source dependency coordinates bound into an evaluation report.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct EvaluationLineage {
|
||||
/// Content identity of the exact source declaration.
|
||||
pub source_snapshot_id: String,
|
||||
/// Direct source adapter/provider class.
|
||||
pub source: Source,
|
||||
/// Provider-native snapshot coordinate.
|
||||
pub snapshot: String,
|
||||
/// Declared dependencies, or null for a legacy source with unknown lineage.
|
||||
pub lineage: Option<SourceLineage>,
|
||||
}
|
||||
|
||||
/// Evaluates newline-delimited cases against one already verified registry.
|
||||
///
|
||||
/// # Errors
|
||||
|
|
@ -151,10 +170,24 @@ pub fn run(
|
|||
ensure!(total > 0, "evaluation contains no cases");
|
||||
durations.sort_unstable();
|
||||
let failed = u64::try_from(failures.len())?;
|
||||
let source_lineage = registry
|
||||
.receipt
|
||||
.sources
|
||||
.iter()
|
||||
.map(|source| {
|
||||
Ok(EvaluationLineage {
|
||||
source_snapshot_id: source.id()?,
|
||||
source: source.source,
|
||||
snapshot: source.snapshot.clone(),
|
||||
lineage: source.lineage.clone(),
|
||||
})
|
||||
})
|
||||
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||
Ok(Report {
|
||||
schema: "argand.site-evaluation/v1".into(),
|
||||
schema: "argand.site-evaluation/v2".into(),
|
||||
registry: registry.identity.clone(),
|
||||
evaluated_at,
|
||||
source_lineage,
|
||||
total,
|
||||
passed: total - failed,
|
||||
failed,
|
||||
|
|
|
|||
|
|
@ -51,14 +51,17 @@ impl Registry {
|
|||
&& crate::digest(&attribution) == receipt.attribution_sha256,
|
||||
"registry license or attribution digest mismatch"
|
||||
);
|
||||
let current = receipt.schema == "argand.site-registry/v2"
|
||||
&& crate::store::supported_rule_version(&receipt.rules)
|
||||
let current = matches!(
|
||||
receipt.schema.as_str(),
|
||||
"argand.site-registry/v2" | "argand.site-registry/v3"
|
||||
) && crate::store::supported_rule_version(&receipt.rules)
|
||||
&& crate::model::valid_digest(&receipt.review_policy_sha256)
|
||||
&& receipt.review_policy.id()? == receipt.review_policy_sha256
|
||||
&& crate::model::valid_digest(&receipt.coverage_sha256)
|
||||
&& receipt.decision_time_policy == "argand.site-decision-time/v1"
|
||||
&& (receipt.review_policy.allow_legacy_reviews
|
||||
|| crate::model::valid_digest(&receipt.reviewer_trust_sha256));
|
||||
|| crate::model::valid_digest(&receipt.reviewer_trust_sha256))
|
||||
&& validate_audit_contract(&receipt).is_ok();
|
||||
let legacy = allow_legacy_rules
|
||||
&& receipt.schema == "argand.site-registry/v1"
|
||||
&& crate::store::legacy_rule_version(&receipt.rules);
|
||||
|
|
@ -71,6 +74,7 @@ impl Registry {
|
|||
crate::coverage::projection_digest(&db)? == receipt.coverage_sha256,
|
||||
"selected coverage graph differs from receipt"
|
||||
);
|
||||
validate_source_selection(&db, &receipt)?;
|
||||
}
|
||||
Ok(Self {
|
||||
db,
|
||||
|
|
@ -81,6 +85,74 @@ impl Registry {
|
|||
}
|
||||
}
|
||||
|
||||
fn validate_audit_contract(receipt: &Receipt) -> anyhow::Result<()> {
|
||||
let mut declared_sources = BTreeSet::new();
|
||||
for source in &receipt.sources {
|
||||
source.validate()?;
|
||||
ensure!(
|
||||
declared_sources.insert(source.id()?),
|
||||
"duplicate receipt source declaration"
|
||||
);
|
||||
}
|
||||
ensure!(
|
||||
receipt.sources.iter().any(|source| {
|
||||
source.source == crate::model::Source::Psl
|
||||
&& source.id().ok().as_ref() == Some(&receipt.psl_source)
|
||||
}),
|
||||
"receipt PSL is not the active PSL source"
|
||||
);
|
||||
if receipt.schema == "argand.site-registry/v2" {
|
||||
ensure!(
|
||||
receipt.runtime_layout.is_empty() && receipt.audit_bundles.is_empty(),
|
||||
"v2 registry cannot reference external audit history"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
receipt.runtime_layout == "compact-v1" && !receipt.audit_bundles.is_empty(),
|
||||
"v3 registry needs compact audit references"
|
||||
);
|
||||
let mut sources = BTreeSet::new();
|
||||
let mut selected_sources = BTreeSet::new();
|
||||
for reference in &receipt.audit_bundles {
|
||||
reference.validate()?;
|
||||
ensure!(
|
||||
reference.attribution_sha256 == receipt.attribution_sha256
|
||||
&& reference.coverage_sha256 == receipt.coverage_sha256,
|
||||
"audit reference is bound to another generation contract"
|
||||
);
|
||||
ensure!(
|
||||
sources.insert(&reference.source_id),
|
||||
"duplicate audit source reference"
|
||||
);
|
||||
if reference.selected {
|
||||
selected_sources.insert(reference.source_id.clone());
|
||||
}
|
||||
}
|
||||
ensure!(
|
||||
selected_sources == declared_sources,
|
||||
"selected audit references differ from active receipt sources"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_source_selection(db: &Connection, receipt: &Receipt) -> anyhow::Result<()> {
|
||||
let mut statement = db.prepare("SELECT id FROM selected_sources ORDER BY id")?;
|
||||
let stored = statement
|
||||
.query_map([], |row| row.get::<_, String>(0))?
|
||||
.collect::<Result<BTreeSet<_>, _>>()?;
|
||||
let declared = receipt
|
||||
.sources
|
||||
.iter()
|
||||
.map(crate::model::SourceManifest::id)
|
||||
.collect::<anyhow::Result<BTreeSet<_>>>()?;
|
||||
ensure!(
|
||||
stored == declared,
|
||||
"receipt sources differ from selected database sources"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn verify_layout(path: &Path) -> anyhow::Result<()> {
|
||||
ensure!(
|
||||
std::fs::symlink_metadata(path)?.is_dir(),
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
use serde::de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor};
|
||||
use serde_json::{Map, Number, Value};
|
||||
use std::fmt;
|
||||
use std::{cell::Cell, io::Read, rc::Rc};
|
||||
|
||||
struct Unique(Value);
|
||||
|
||||
|
|
@ -61,6 +62,71 @@ pub(crate) fn parse(bytes: &[u8]) -> anyhow::Result<Value> {
|
|||
Ok(serde_json::from_slice::<Unique>(bytes)?.0)
|
||||
}
|
||||
|
||||
/// Streams a top-level JSON array while bounding each materialized record.
|
||||
pub(crate) fn for_each_array(
|
||||
reader: &mut dyn Read,
|
||||
maximum_record_bytes: usize,
|
||||
mut callback: impl FnMut(Value) -> anyhow::Result<()>,
|
||||
) -> anyhow::Result<()> {
|
||||
use serde::de::Error as _;
|
||||
struct Bound<'a> {
|
||||
inner: &'a mut dyn Read,
|
||||
used: Rc<Cell<usize>>,
|
||||
maximum: usize,
|
||||
}
|
||||
impl Read for Bound<'_> {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||
let remaining = self.maximum.saturating_sub(self.used.get());
|
||||
if remaining == 0 {
|
||||
return Err(std::io::Error::other(
|
||||
"JSON record exceeds configured bound",
|
||||
));
|
||||
}
|
||||
let length = buffer.len().min(remaining);
|
||||
let count = self.inner.read(&mut buffer[..length])?;
|
||||
self.used.set(self.used.get().saturating_add(count));
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
struct Array<'a, F> {
|
||||
callback: &'a mut F,
|
||||
used: Rc<Cell<usize>>,
|
||||
}
|
||||
impl<'de, F> Visitor<'de> for Array<'_, F>
|
||||
where
|
||||
F: FnMut(Value) -> anyhow::Result<()>,
|
||||
{
|
||||
type Value = ();
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("a JSON array of uniquely keyed records")
|
||||
}
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut sequence: A) -> Result<(), A::Error> {
|
||||
while let Some(Unique(value)) = sequence.next_element()? {
|
||||
(self.callback)(value).map_err(A::Error::custom)?;
|
||||
self.used.set(0);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
anyhow::ensure!(
|
||||
maximum_record_bytes > 0,
|
||||
"JSON record bound must be positive"
|
||||
);
|
||||
let used = Rc::new(Cell::new(0));
|
||||
let bound = Bound {
|
||||
inner: reader,
|
||||
used: Rc::clone(&used),
|
||||
maximum: maximum_record_bytes,
|
||||
};
|
||||
let mut deserializer = serde_json::Deserializer::from_reader(bound);
|
||||
deserializer.deserialize_seq(Array {
|
||||
callback: &mut callback,
|
||||
used,
|
||||
})?;
|
||||
deserializer.end()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
|
|
@ -73,4 +139,19 @@ mod tests {
|
|||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streamed_arrays_reject_duplicates_and_oversized_records() -> anyhow::Result<()> {
|
||||
let mut values = Vec::new();
|
||||
super::for_each_array(&mut &br#"[{"a":1},{"b":[2,3]}]"#[..], 1024, |value| {
|
||||
values.push(value);
|
||||
Ok(())
|
||||
})?;
|
||||
assert_eq!(values.len(), 2);
|
||||
assert!(super::for_each_array(&mut &br#"[{"a":1,"a":2}]"#[..], 1024, |_| Ok(())).is_err());
|
||||
assert!(
|
||||
super::for_each_array(&mut &br#"[{"long":"0123456789"}]"#[..], 8, |_| Ok(())).is_err()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
//! Source-separated website assertions and reviewed, immutable registry releases.
|
||||
|
||||
pub mod adapters;
|
||||
pub mod audit;
|
||||
pub mod build;
|
||||
pub mod bundle;
|
||||
pub mod catalog;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// By Nic Weyand!
|
||||
//! Source identities and source-native assertion envelopes.
|
||||
|
||||
use anyhow::ensure;
|
||||
use anyhow::{Context, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
|
@ -20,6 +20,8 @@ pub enum Source {
|
|||
Curlie,
|
||||
/// Public Suffix List, including PRIVATE rules.
|
||||
Psl,
|
||||
/// Research Organization Registry organization records.
|
||||
Ror,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
|
|
@ -32,13 +34,14 @@ impl Source {
|
|||
Self::Crux => "crux",
|
||||
Self::Curlie => "curlie",
|
||||
Self::Psl => "psl",
|
||||
Self::Ror => "ror",
|
||||
}
|
||||
}
|
||||
/// Exact SPDX data license.
|
||||
#[must_use]
|
||||
pub const fn license(self) -> &'static str {
|
||||
match self {
|
||||
Self::Wikidata => "CC0-1.0",
|
||||
Self::Wikidata | Self::Ror => "CC0-1.0",
|
||||
Self::Majestic | Self::Curlie => "CC-BY-3.0",
|
||||
Self::Crux => "CC-BY-4.0",
|
||||
Self::Psl => "MPL-2.0",
|
||||
|
|
@ -53,6 +56,7 @@ impl Source {
|
|||
Self::Crux => "https://developer.chrome.com/docs/crux/methodology",
|
||||
Self::Curlie => "https://curlie.org/docs/en/license.html",
|
||||
Self::Psl => "https://publicsuffix.org/list/public_suffix_list.dat",
|
||||
Self::Ror => "https://ror.readme.io/docs/data-dump",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -73,6 +77,132 @@ pub enum Format {
|
|||
CurlieTarGz,
|
||||
/// UTF-8 PSL text.
|
||||
PslText,
|
||||
/// Official ROR release ZIP containing schema 2.1 JSON and CSV.
|
||||
RorZip,
|
||||
}
|
||||
|
||||
/// How an immutable source object was checked before import.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct IntegrityProof {
|
||||
/// `content_digest` or `provider_checksum`.
|
||||
pub method: String,
|
||||
/// Lowercase digest algorithm, currently `sha256` or `md5`.
|
||||
pub algorithm: String,
|
||||
/// Lowercase hexadecimal digest.
|
||||
pub value: String,
|
||||
/// Provider page or checksum object that supplied the expected value.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub evidence_url: Option<String>,
|
||||
}
|
||||
|
||||
impl IntegrityProof {
|
||||
/// Validates bounded, explicit integrity evidence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects unknown methods/algorithms, malformed digests, and unsafe evidence URLs.
|
||||
pub fn validate(&self) -> anyhow::Result<()> {
|
||||
ensure!(
|
||||
matches!(self.method.as_str(), "content_digest" | "provider_checksum"),
|
||||
"unsupported integrity method"
|
||||
);
|
||||
let length = match self.algorithm.as_str() {
|
||||
"sha256" => 64,
|
||||
"md5" => 32,
|
||||
_ => anyhow::bail!("unsupported integrity algorithm"),
|
||||
};
|
||||
ensure!(
|
||||
self.value.len() == length
|
||||
&& self
|
||||
.value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)),
|
||||
"invalid integrity digest"
|
||||
);
|
||||
if self.method == "content_digest" {
|
||||
ensure!(
|
||||
self.algorithm == "sha256" && self.evidence_url.is_none(),
|
||||
"content digest is the local SHA-256"
|
||||
);
|
||||
} else {
|
||||
let url = url::Url::parse(
|
||||
self.evidence_url
|
||||
.as_deref()
|
||||
.context("provider integrity proof needs an evidence URL")?,
|
||||
)?;
|
||||
ensure!(
|
||||
url.scheme() == "https"
|
||||
&& url.username().is_empty()
|
||||
&& url.password().is_none()
|
||||
&& url.port().is_none()
|
||||
&& url.fragment().is_none()
|
||||
&& url.query().is_none(),
|
||||
"integrity evidence URL must be plain HTTPS"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Declared provenance path used to detect copied or correlated assertions.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SourceLineage {
|
||||
/// Provider directly distributing this object.
|
||||
pub direct_provider: String,
|
||||
/// Stable upstream dataset identifiers known to contribute consumed fields.
|
||||
#[serde(default)]
|
||||
pub upstream_datasets: Vec<String>,
|
||||
/// Ordered, declared transformations between upstream and this object.
|
||||
#[serde(default)]
|
||||
pub transformations: Vec<String>,
|
||||
/// Stable corroboration group, or `unknown` when independence is not established.
|
||||
pub independence_group: String,
|
||||
}
|
||||
|
||||
impl SourceLineage {
|
||||
/// Conservative direct-provider lineage for an official snapshot.
|
||||
#[must_use]
|
||||
pub fn direct(source: Source) -> Self {
|
||||
Self {
|
||||
direct_provider: source.key().into(),
|
||||
upstream_datasets: if source == Source::Ror {
|
||||
vec!["geonames".into()]
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
transformations: vec!["provider_snapshot".into()],
|
||||
independence_group: source.key().into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates bounded identifiers without claiming unknown independence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects empty, oversized, duplicate, or unsafe lineage coordinates.
|
||||
pub fn validate(&self) -> anyhow::Result<()> {
|
||||
fn item(value: &str) -> bool {
|
||||
!value.trim().is_empty() && value.len() <= 256 && !value.chars().any(char::is_control)
|
||||
}
|
||||
ensure!(
|
||||
item(&self.direct_provider)
|
||||
&& item(&self.independence_group)
|
||||
&& self.upstream_datasets.len() <= 64
|
||||
&& self.upstream_datasets.iter().all(|value| item(value))
|
||||
&& self.transformations.len() <= 64
|
||||
&& self.transformations.iter().all(|value| item(value)),
|
||||
"invalid source lineage"
|
||||
);
|
||||
let upstream = self
|
||||
.upstream_datasets
|
||||
.iter()
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
ensure!(
|
||||
upstream.len() == self.upstream_datasets.len(),
|
||||
"duplicate upstream lineage"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Compression of the downloaded object (Curlie tar.gz uses `None` here).
|
||||
|
|
@ -189,7 +319,7 @@ impl SourceCoverage {
|
|||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SourceManifest {
|
||||
/// `argand.site-source/v1` for legacy scope semantics or v2 for typed coverage.
|
||||
/// V1 legacy scope, v2 typed coverage, or v3 integrity/lineage coverage.
|
||||
pub schema: String,
|
||||
/// Approved provider.
|
||||
pub source: Source,
|
||||
|
|
@ -217,6 +347,15 @@ pub struct SourceManifest {
|
|||
pub sha256: String,
|
||||
/// Original compressed object length.
|
||||
pub bytes: u64,
|
||||
/// Authenticated per-record materialization ceiling; defaults to 16 MiB.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub maximum_record_bytes: Option<u64>,
|
||||
/// Checks applied to the exact source bytes.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub integrity: Vec<IntegrityProof>,
|
||||
/// Provider and upstream dependency declaration.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub lineage: Option<SourceLineage>,
|
||||
}
|
||||
|
||||
impl SourceManifest {
|
||||
|
|
@ -228,7 +367,9 @@ impl SourceManifest {
|
|||
ensure!(
|
||||
matches!(
|
||||
(self.schema.as_str(), self.coverage.as_ref()),
|
||||
("argand.site-source/v1", None) | ("argand.site-source/v2", Some(_))
|
||||
("argand.site-source/v1", None)
|
||||
| ("argand.site-source/v2", Some(_))
|
||||
| ("argand.site-source/v3", _)
|
||||
),
|
||||
"unsupported source schema or coverage declaration"
|
||||
);
|
||||
|
|
@ -240,6 +381,16 @@ impl SourceManifest {
|
|||
self.bytes > 0 && valid_digest(&self.sha256),
|
||||
"invalid source length or digest"
|
||||
);
|
||||
ensure!(
|
||||
self.maximum_record_bytes.is_none_or(|bytes| {
|
||||
(1..=crate::adapters::MAX_CUSTOM_RECORD_BYTES as u64).contains(&bytes)
|
||||
}),
|
||||
"source record bound must be 1 byte..256 MiB"
|
||||
);
|
||||
ensure!(
|
||||
self.schema == "argand.site-source/v3" || self.maximum_record_bytes.is_none(),
|
||||
"custom record bounds require source schema v3"
|
||||
);
|
||||
ensure!(
|
||||
!self.snapshot.is_empty()
|
||||
&& self.snapshot.len() <= 512
|
||||
|
|
@ -256,14 +407,47 @@ impl SourceManifest {
|
|||
| (Source::Crux, Format::CruxCsv)
|
||||
| (Source::Curlie, Format::CurlieTarGz)
|
||||
| (Source::Psl, Format::PslText)
|
||||
| (Source::Ror, Format::RorZip)
|
||||
);
|
||||
ensure!(valid, "source/format mismatch");
|
||||
if let Some(coverage) = &self.coverage {
|
||||
coverage.validate()?;
|
||||
}
|
||||
for proof in &self.integrity {
|
||||
proof.validate()?;
|
||||
crate::download::validate_integrity_evidence(self.source, &self.source_url, proof)?;
|
||||
}
|
||||
if self.schema == "argand.site-source/v3" {
|
||||
ensure!(
|
||||
self.integrity.iter().any(|proof| {
|
||||
proof.method == "content_digest"
|
||||
&& proof.algorithm == "sha256"
|
||||
&& proof.value == self.sha256
|
||||
}),
|
||||
"source v3 needs its exact content SHA-256 proof"
|
||||
);
|
||||
self.lineage
|
||||
.as_ref()
|
||||
.context("source v3 needs explicit lineage")?
|
||||
.validate()?;
|
||||
} else {
|
||||
ensure!(
|
||||
self.integrity.is_empty() && self.lineage.is_none(),
|
||||
"lineage and integrity metadata require source schema v3"
|
||||
);
|
||||
}
|
||||
crate::download::validate_source_url(self.source, &self.source_url)?;
|
||||
Ok(())
|
||||
}
|
||||
/// Exact per-record limit bound into this source declaration.
|
||||
#[must_use]
|
||||
pub fn record_bytes_limit(&self) -> usize {
|
||||
usize::try_from(
|
||||
self.maximum_record_bytes
|
||||
.unwrap_or(crate::adapters::MAX_RECORD_BYTES as u64),
|
||||
)
|
||||
.unwrap_or(crate::adapters::MAX_RECORD_BYTES)
|
||||
}
|
||||
/// Content identity of the declaration, including original retrieval time.
|
||||
///
|
||||
/// # Errors
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ pub fn attribution() -> Value {
|
|||
"crux":{"license":"CC-BY-4.0","credit":"Chrome UX Report, Google","url":"https://developer.chrome.com/docs/crux/","license_url":"https://creativecommons.org/licenses/by/4.0/"},
|
||||
"curlie":{"license":"CC-BY-3.0","credit":"With content from Curlie.org - the largest human-edited directory of the web. Contribute by submitting a website or becoming an editor.","url":"https://curlie.org/","license_url":"https://creativecommons.org/licenses/by/3.0/","public_display":"Use the prescribed HTML attribution on every page using Curlie content: https://curlie.org/docs/en/license.html"},
|
||||
"psl":{"license":"MPL-2.0","url":"https://publicsuffix.org/list/","license_url":"https://mozilla.org/MPL/2.0/"},
|
||||
"ror":{"license":"CC0-1.0","url":"https://ror.org/","license_url":"https://ror.readme.io/docs/data-dump","lineage_note":"ROR location metadata identifies GeoNames as an upstream CC BY 3.0 source","upstream_attribution":{"credit":"GeoNames","url":"https://www.geonames.org/","license_url":"https://creativecommons.org/licenses/by/3.0/"}},
|
||||
"argand_candidate_observer":{"license":"CC0-1.0","url":"https://git.argand.org/nicweyand/argand-site-registry","license_url":"https://creativecommons.org/publicdomain/zero/1.0/","scope":"locally authored observation metadata; captured page content is not redistributed"},
|
||||
"changes":"Argand normalizes and combines assertions; provider endorsement is not implied."})
|
||||
}
|
||||
|
|
@ -51,6 +52,10 @@ pub fn export_audit(
|
|||
output: &Path,
|
||||
include_descriptions: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
ensure!(
|
||||
registry.receipt.runtime_layout.is_empty(),
|
||||
"compact generation audit export requires its external audit store"
|
||||
);
|
||||
argand_atomic::create_durable_with(output, |file| {
|
||||
export_inner(registry, file, include_descriptions, ExportMode::Audit)
|
||||
.map_err(std::io::Error::other)
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ pub fn import(
|
|||
///
|
||||
/// # Errors
|
||||
/// Returns source integrity, resource, parser, filesystem, or database errors.
|
||||
#[allow(clippy::too_many_lines)] // Keep verification, checkpoints, and cleanup in one transaction lifecycle.
|
||||
pub fn import_with_limits(
|
||||
db: &mut Connection,
|
||||
manifest: &SourceManifest,
|
||||
|
|
@ -217,7 +218,15 @@ pub fn import_with_limits(
|
|||
limits,
|
||||
database_before,
|
||||
};
|
||||
let parsed = adapters::adapter(manifest.format).ingest(&mut reader, &mut sink);
|
||||
let parsed = adapters::adapter(
|
||||
manifest.format,
|
||||
manifest.record_bytes_limit(),
|
||||
manifest
|
||||
.coverage
|
||||
.as_ref()
|
||||
.is_some_and(|coverage| coverage.kind == crate::model::CoverageKind::Delta),
|
||||
)
|
||||
.ingest(&mut reader, &mut sink);
|
||||
parsed.and_then(|()| {
|
||||
std::io::copy(&mut reader, &mut std::io::sink())?;
|
||||
ensure!(
|
||||
|
|
@ -241,9 +250,7 @@ pub fn import_with_limits(
|
|||
match result {
|
||||
Ok(()) => db.execute_batch("COMMIT")?,
|
||||
Err(error) => {
|
||||
db.execute_batch("ROLLBACK")?;
|
||||
discard_incomplete_source(db, &id)?;
|
||||
db.pragma_update(None, "max_page_count", i64::try_from(original_max_pages)?)?;
|
||||
recover_failed_import(db, &id, original_max_pages)?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
|
|
@ -251,6 +258,22 @@ pub fn import_with_limits(
|
|||
Ok(id)
|
||||
}
|
||||
|
||||
fn recover_failed_import(db: &Connection, id: &str, original_max_pages: u64) -> anyhow::Result<()> {
|
||||
// SQLITE_FULL and some I/O errors can roll back the transaction themselves.
|
||||
// Cleanup and ceiling restoration must still run.
|
||||
let rollback = if db.is_autocommit() {
|
||||
Ok(())
|
||||
} else {
|
||||
db.execute_batch("ROLLBACK")
|
||||
};
|
||||
let cleanup = discard_incomplete_source(db, id);
|
||||
let restore = db.pragma_update(None, "max_page_count", i64::try_from(original_max_pages)?);
|
||||
rollback.context("roll back failed source import")?;
|
||||
cleanup.context("discard incomplete source after import failure")?;
|
||||
restore.context("restore database growth ceiling after import failure")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn discard_incomplete_source(db: &Connection, id: &str) -> anyhow::Result<()> {
|
||||
db.execute_batch("BEGIN IMMEDIATE")?;
|
||||
db.execute("DELETE FROM facts WHERE source_id=?1", [id])?;
|
||||
|
|
|
|||
|
|
@ -211,6 +211,7 @@ mod tests {
|
|||
snapshot: "{date}".into(),
|
||||
scope: "full".into(),
|
||||
maximum_bytes: 1,
|
||||
maximum_record_bytes: None,
|
||||
coverage: Some(SourceCoverage {
|
||||
collection: "default".into(),
|
||||
kind: CoverageKind::Partition,
|
||||
|
|
@ -219,6 +220,8 @@ mod tests {
|
|||
sequence: None,
|
||||
supersedes: Vec::new(),
|
||||
}),
|
||||
provider_checksum: None,
|
||||
provider_checksum_url: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,9 @@ pub fn manifest(source: Source, format: Format, bytes: &[u8]) -> anyhow::Result<
|
|||
Source::Majestic => "https://downloads.majestic.com/majestic_million.csv",
|
||||
Source::Crux => "https://developer.chrome.com/docs/crux/bigquery/",
|
||||
Source::Curlie => "https://curlie.org/directory-dl",
|
||||
Source::Ror => {
|
||||
"https://zenodo.org/api/records/22099990/files/v2.12-2026-08-25-ror-data.zip/content"
|
||||
}
|
||||
};
|
||||
Ok(SourceManifest {
|
||||
schema: "argand.site-source/v1".into(),
|
||||
|
|
@ -83,6 +86,9 @@ pub fn manifest(source: Source, format: Format, bytes: &[u8]) -> anyhow::Result<
|
|||
retrieved_at: timestamp()?,
|
||||
sha256: argand_site_registry::digest(bytes),
|
||||
bytes: u64::try_from(bytes.len())?,
|
||||
maximum_record_bytes: None,
|
||||
integrity: Vec::new(),
|
||||
lineage: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,6 +58,14 @@ fn multilingual_regional_and_deceptive_queries_are_replayed() -> anyhow::Result<
|
|||
assert_eq!(report.passed, 8);
|
||||
assert_eq!(report.failed, 0);
|
||||
assert_eq!(report.evaluated_at, common::timestamp()?);
|
||||
assert_eq!(report.schema, "argand.site-evaluation/v2");
|
||||
assert_eq!(report.source_lineage.len(), 5);
|
||||
assert!(
|
||||
report
|
||||
.source_lineage
|
||||
.iter()
|
||||
.all(|source| source.lineage.is_none())
|
||||
);
|
||||
assert!(report.failures.is_empty());
|
||||
|
||||
let incorrect = serde_json::to_string(&json!({
|
||||
|
|
|
|||
|
|
@ -205,6 +205,39 @@ fn import_limits_rollback_all_partial_source_rows() -> anyhow::Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_growth_ceiling_simulates_disk_full_and_rolls_back() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = store::open(&root.path().join("database-full.sqlite"))?;
|
||||
let mut data = String::from("origin,rank,yyyymm,country_code\n");
|
||||
for index in 0..20_000 {
|
||||
use std::fmt::Write as _;
|
||||
writeln!(data, "https://{index}.example.com/,1000,202608,")?;
|
||||
}
|
||||
let input = root.path().join("many-origins.csv");
|
||||
fs::write(&input, &data)?;
|
||||
let manifest = common::manifest(Source::Crux, Format::CruxCsv, data.as_bytes())?;
|
||||
assert!(
|
||||
store::import_with_limits(
|
||||
&mut db,
|
||||
&manifest,
|
||||
&input,
|
||||
ImportLimits {
|
||||
maximum_expanded_bytes: u64::try_from(data.len())? + 1,
|
||||
maximum_records: 25_000,
|
||||
maximum_database_growth_bytes: 64 * 1024,
|
||||
},
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
db.query_row("SELECT count(*) FROM sources", [], |row| row
|
||||
.get::<_, i64>(0))?,
|
||||
0
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retired_reviewer_keys_verify_history_only_with_a_validity_epoch() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
|
|
|
|||
580
crates/argand-site-registry/tests/v05.rs
Normal file
580
crates/argand-site-registry/tests/v05.rs
Normal file
|
|
@ -0,0 +1,580 @@
|
|||
// By Nic Weyand!
|
||||
//! v0.5 source-lineage and ROR schema acceptance proof.
|
||||
#[allow(dead_code)]
|
||||
mod common;
|
||||
|
||||
use anyhow::{Context, ensure};
|
||||
use argand_site_registry::{
|
||||
build, evaluation,
|
||||
model::{CoverageKind, Format, IntegrityProof, Source, SourceCoverage, SourceLineage},
|
||||
policy::ReviewPolicy,
|
||||
query::Registry,
|
||||
store,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
io::{Cursor, Write},
|
||||
process::{Command, Stdio},
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
fn record(id: &str, name: &str, status: &str, website: &str) -> Value {
|
||||
json!({
|
||||
"admin":{"created":{"date":"2020-01-01","schema_version":"1.0"},"last_modified":{"date":"2026-08-01","schema_version":"2.1"}},
|
||||
"domains":[url::Url::parse(website).ok().and_then(|url| url.host_str().map(str::to_owned)).unwrap_or_default()],
|
||||
"established":2020,
|
||||
"external_ids":[{"all":["Q42"],"preferred":"Q42","type":"wikidata"}],
|
||||
"id":format!("https://ror.org/{id}"),
|
||||
"links":[{"type":"website","value":website},{"type":"wikipedia","value":"https://en.wikipedia.org/wiki/Example"}],
|
||||
"locations":[{"geonames_id":2_643_743,"geonames_details":{"continent_code":"EU","continent_name":"Europe","country_code":"GB","country_name":"United Kingdom","country_subdivision_code":"ENG","country_subdivision_name":"England","lat":51.5,"lng":-0.1,"name":"London"}}],
|
||||
"names":[{"lang":"en","types":["ror_display","label"],"value":name},{"lang":null,"types":["acronym"],"value":"EU"}],
|
||||
"relationships":[],
|
||||
"status":status,
|
||||
"types":["education"]
|
||||
})
|
||||
}
|
||||
|
||||
fn ror_zip(records: &[Value], extra: Option<(&str, &[u8])>) -> anyhow::Result<Vec<u8>> {
|
||||
let mut archive = zip::ZipWriter::new(Cursor::new(Vec::new()));
|
||||
let options = zip::write::SimpleFileOptions::default()
|
||||
.compression_method(zip::CompressionMethod::Deflated);
|
||||
archive.start_file("v2.12-2026-08-25-ror-data.json", options)?;
|
||||
archive.write_all(&serde_json::to_vec_pretty(records)?)?;
|
||||
archive.start_file("v2.12-2026-08-25-ror-data.csv", options)?;
|
||||
archive.write_all(b"id,name\n")?;
|
||||
if let Some((name, bytes)) = extra {
|
||||
archive.start_file(name, options)?;
|
||||
archive.write_all(bytes)?;
|
||||
}
|
||||
Ok(archive.finish()?.into_inner())
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::too_many_lines)] // One source scenario covers every reviewed ROR mapping and hold.
|
||||
fn ror_schema_21_is_separate_ineligible_when_inactive_and_lineage_is_exported() -> anyhow::Result<()>
|
||||
{
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = store::open(&root.path().join("writer.sqlite"))?;
|
||||
common::import(
|
||||
&mut db,
|
||||
root.path(),
|
||||
Source::Psl,
|
||||
Format::PslText,
|
||||
common::PSL.as_bytes(),
|
||||
)?;
|
||||
let wikidata = json!({"entities":{"Q42":common::entity(
|
||||
"Q42",
|
||||
"Example University Wikidata",
|
||||
&[],
|
||||
&["https://wikidata-example.org/"]
|
||||
)}});
|
||||
common::import(
|
||||
&mut db,
|
||||
root.path(),
|
||||
Source::Wikidata,
|
||||
Format::WikidataEntities,
|
||||
&serde_json::to_vec(&wikidata)?,
|
||||
)?;
|
||||
let mut sparse = record(
|
||||
"fedcba210",
|
||||
"Conflicting Organization",
|
||||
"active",
|
||||
"https://example.org/",
|
||||
);
|
||||
sparse["domains"] = json!([]);
|
||||
sparse["established"] = Value::Null;
|
||||
sparse["external_ids"] = json!([]);
|
||||
sparse["locations"] = json!([]);
|
||||
sparse["names"] =
|
||||
json!([{"lang":"en","types":["ror_display"],"value":"Conflicting Organization"}]);
|
||||
sparse["links"] = json!([
|
||||
{"type":"website","value":"https://example.org/"},
|
||||
{"type":"website","value":"https://regional.example.org/"}
|
||||
]);
|
||||
let bytes = ror_zip(
|
||||
&[
|
||||
record(
|
||||
"012345678",
|
||||
"Example University",
|
||||
"active",
|
||||
"https://example.org/",
|
||||
),
|
||||
record(
|
||||
"abcdef012",
|
||||
"Former Institute",
|
||||
"inactive",
|
||||
"https://former.example.org/",
|
||||
),
|
||||
sparse,
|
||||
],
|
||||
None,
|
||||
)?;
|
||||
let mut manifest = common::manifest(Source::Ror, Format::RorZip, &bytes)?;
|
||||
manifest.schema = "argand.site-source/v3".into();
|
||||
manifest.integrity = vec![IntegrityProof {
|
||||
method: "content_digest".into(),
|
||||
algorithm: "sha256".into(),
|
||||
value: manifest.sha256.clone(),
|
||||
evidence_url: None,
|
||||
}];
|
||||
manifest.lineage = Some(SourceLineage::direct(Source::Ror));
|
||||
let input = root.path().join("ror.zip");
|
||||
std::fs::write(&input, &bytes)?;
|
||||
let first = store::import(&mut db, &manifest, &input)?;
|
||||
assert_eq!(first, store::import(&mut db, &manifest, &input)?);
|
||||
let registry = common::build(&db, root.path(), "generation")?;
|
||||
let active = registry.lookup("Example University", 20)?;
|
||||
assert_eq!(active.total_entities, 1);
|
||||
assert_eq!(active.candidates[0].relation, "asserted_official");
|
||||
assert!(active.candidates[0].eligible);
|
||||
assert_eq!(
|
||||
active.candidates[0].provenance[0]["source"]["lineage"]["upstream_datasets"][0],
|
||||
"geonames"
|
||||
);
|
||||
assert!(!registry.lookup("Former Institute", 20)?.candidates[0].eligible);
|
||||
assert_eq!(registry.lookup("EU", 20)?.total_entities, 2);
|
||||
assert_eq!(
|
||||
registry
|
||||
.lookup("Conflicting Organization", 20)?
|
||||
.candidates
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
let ror_id = argand_site_registry::model::entity_id(Source::Ror, "https://ror.org/012345678");
|
||||
let wikidata_id = argand_site_registry::model::entity_id(Source::Wikidata, "Q42");
|
||||
let equivalence = argand_site_registry::identity::propose(®istry, &ror_id, &wikidata_id)?;
|
||||
assert_eq!(equivalence.entities.len(), 2);
|
||||
let audit = root.path().join("audit.jsonl");
|
||||
argand_site_registry::release::export_audit(®istry, &audit, false)?;
|
||||
let audit = std::fs::read_to_string(audit)?;
|
||||
assert!(audit.contains("declared_domain"));
|
||||
assert!(audit.contains("geonames"));
|
||||
assert!(audit.contains("wikidata"));
|
||||
let report = evaluation::run(
|
||||
®istry,
|
||||
Cursor::new(
|
||||
r#"{"id":"ror","query":"Example University","expected_status":"no_active_name_review"}"#,
|
||||
),
|
||||
1,
|
||||
common::timestamp()?,
|
||||
)?;
|
||||
let lineage = report
|
||||
.source_lineage
|
||||
.iter()
|
||||
.find(|source| source.source == Source::Ror)
|
||||
.context("ROR evaluation lineage missing")?;
|
||||
assert_eq!(
|
||||
lineage
|
||||
.lineage
|
||||
.as_ref()
|
||||
.context("ROR lineage missing")?
|
||||
.upstream_datasets,
|
||||
vec!["geonames".to_owned()]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ror_zip_and_schema_drift_fail_before_completion() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = store::open(&root.path().join("writer.sqlite"))?;
|
||||
let mut changed = record(
|
||||
"012345678",
|
||||
"Changed Schema",
|
||||
"active",
|
||||
"https://example.org/",
|
||||
);
|
||||
changed
|
||||
.as_object_mut()
|
||||
.context("record object")?
|
||||
.insert("unreviewed_field".into(), json!(true));
|
||||
for bytes in [
|
||||
ror_zip(&[changed], None)?,
|
||||
ror_zip(
|
||||
&[record(
|
||||
"012345678",
|
||||
"Extra Member",
|
||||
"active",
|
||||
"https://example.org/",
|
||||
)],
|
||||
Some(("unexpected.txt", b"bad")),
|
||||
)?,
|
||||
] {
|
||||
let manifest = common::manifest(Source::Ror, Format::RorZip, &bytes)?;
|
||||
let input = root.path().join(format!("{}.zip", manifest.sha256));
|
||||
std::fs::write(&input, bytes)?;
|
||||
assert!(store::import(&mut db, &manifest, &input).is_err());
|
||||
let incomplete: bool = db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sources WHERE id=?1)",
|
||||
[manifest.id()?],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
ensure!(!incomplete, "failed ROR source remained in writer store");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::too_many_lines)] // One lifecycle proves package, compact, verify, export, sign, and tamper behavior.
|
||||
fn compact_generation_keeps_runtime_results_and_authenticates_cold_history() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = common::fixture(root.path())?;
|
||||
let before = common::build(&db, root.path(), "before-revision")?;
|
||||
|
||||
// A later legacy snapshot in the same source scope leaves the earlier source
|
||||
// available for audit while replacing it in the active projection.
|
||||
let mut replacement = json!({"entities":{"Q355":common::entity(
|
||||
"Q355",
|
||||
"Facebook",
|
||||
&["FB"],
|
||||
&["https://facebook.com/"]
|
||||
)}});
|
||||
replacement["entities"]["Q355"]["lastrevid"] = json!(2);
|
||||
let bytes = serde_json::to_vec(&replacement)?;
|
||||
let mut manifest = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?;
|
||||
manifest.retrieved_at += chrono::Duration::days(1);
|
||||
let input = root.path().join("wikidata-replacement.json");
|
||||
std::fs::write(&input, &bytes)?;
|
||||
store::import(&mut db, &manifest, &input)?;
|
||||
|
||||
let full = common::build(&db, root.path(), "full")?;
|
||||
assert_eq!(
|
||||
before.lookup("Facebook", 20)?.candidates[0].fingerprint,
|
||||
full.lookup("Facebook", 20)?.candidates[0].fingerprint
|
||||
);
|
||||
let mut revision_diff = Vec::new();
|
||||
argand_site_registry::release::diff(&before, &full, &mut revision_diff)?;
|
||||
let revision_diff = String::from_utf8(revision_diff)?;
|
||||
assert!(revision_diff.contains("revision"));
|
||||
assert!(revision_diff.contains("\"subject\":\"fact\""));
|
||||
let audit_store = root.path().join("audit-store");
|
||||
let compact_path = root.path().join("compact");
|
||||
let compact_receipt = build::build_compact_with_policy_and_trust(
|
||||
&db,
|
||||
&compact_path,
|
||||
&ReviewPolicy::legacy_compatible(),
|
||||
None,
|
||||
&audit_store,
|
||||
)?;
|
||||
let compact = Registry::open(
|
||||
&compact_path,
|
||||
&argand_site_registry::file_digest(&compact_path.join("COMPLETE.json"))?,
|
||||
)?;
|
||||
|
||||
assert_eq!(compact_receipt.runtime_layout, "compact-v1");
|
||||
assert_eq!(
|
||||
serde_json::to_value(full.lookup("Facebook", 20)?.candidates)?,
|
||||
serde_json::to_value(compact.lookup("Facebook", 20)?.candidates)?
|
||||
);
|
||||
assert!(
|
||||
std::fs::metadata(compact_path.join("registry.sqlite"))?.len()
|
||||
< std::fs::metadata(root.path().join("full/registry.sqlite"))?.len()
|
||||
);
|
||||
let verified = argand_site_registry::audit::verify(&compact, &audit_store)?;
|
||||
assert_eq!(
|
||||
usize::try_from(verified.bundles)?,
|
||||
compact_receipt.audit_bundles.len()
|
||||
);
|
||||
assert!(
|
||||
compact_receipt
|
||||
.audit_bundles
|
||||
.iter()
|
||||
.any(|item| !item.selected)
|
||||
);
|
||||
let checkpoint_path = root.path().join("retention.json");
|
||||
let checkpoint = argand_site_registry::audit::checkpoint(
|
||||
&compact,
|
||||
&audit_store,
|
||||
common::timestamp()?,
|
||||
&checkpoint_path,
|
||||
)?;
|
||||
assert!(!checkpoint.deletion_authorized);
|
||||
assert_eq!(checkpoint.retain, compact_receipt.audit_bundles);
|
||||
let key = root.path().join("retention-key");
|
||||
ensure!(
|
||||
Command::new("ssh-keygen")
|
||||
.args(["-q", "-t", "ed25519", "-N", "", "-f"])
|
||||
.arg(&key)
|
||||
.status()?
|
||||
.success(),
|
||||
"generate retention test key"
|
||||
);
|
||||
let allowed = root.path().join("retention-allowed-signers");
|
||||
std::fs::write(
|
||||
&allowed,
|
||||
format!(
|
||||
"retention {}",
|
||||
std::fs::read_to_string(key.with_extension("pub"))?
|
||||
),
|
||||
)?;
|
||||
let signature = root.path().join("retention.json.sig");
|
||||
argand_site_registry::audit::sign_retention(&checkpoint_path, &signature, &key)?;
|
||||
assert_eq!(
|
||||
argand_site_registry::audit::verify_retention(
|
||||
&checkpoint_path,
|
||||
&signature,
|
||||
&allowed,
|
||||
"retention"
|
||||
)?,
|
||||
checkpoint
|
||||
);
|
||||
|
||||
let export = root.path().join("cold-history.jsonl");
|
||||
argand_site_registry::audit::export(&compact, &audit_store, &export, false)?;
|
||||
let export = std::fs::read_to_string(export)?;
|
||||
assert!(export.contains("superseded"));
|
||||
assert!(export.contains("\"revision\":2"));
|
||||
assert!(
|
||||
argand_site_registry::release::export_audit(
|
||||
&compact,
|
||||
&root.path().join("must-not-exist.jsonl"),
|
||||
false
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
|
||||
// A second build reuses the exact immutable bundle objects.
|
||||
let second_path = root.path().join("compact-second");
|
||||
let second = build::build_compact_with_policy_and_trust(
|
||||
&db,
|
||||
&second_path,
|
||||
&ReviewPolicy::legacy_compatible(),
|
||||
None,
|
||||
&audit_store,
|
||||
)?;
|
||||
assert_eq!(compact_receipt.audit_bundles, second.audit_bundles);
|
||||
|
||||
let receipt_path = compact_path.join("COMPLETE.json");
|
||||
let original_receipt = std::fs::read(&receipt_path)?;
|
||||
let mut omitted: build::Receipt = serde_json::from_slice(&original_receipt)?;
|
||||
omitted
|
||||
.audit_bundles
|
||||
.retain(|reference| !reference.selected);
|
||||
std::fs::write(&receipt_path, serde_json::to_vec_pretty(&omitted)?)?;
|
||||
let omitted_pin = argand_site_registry::file_digest(&receipt_path)?;
|
||||
assert!(Registry::open(&compact_path, &omitted_pin).is_err());
|
||||
std::fs::write(&receipt_path, original_receipt)?;
|
||||
|
||||
let reference = &compact_receipt.audit_bundles[0];
|
||||
let object = argand_site_registry::audit::path(&audit_store, reference);
|
||||
let original = std::fs::read(&object)?;
|
||||
std::fs::write(&object, &original[..original.len() - 1])?;
|
||||
assert!(argand_site_registry::audit::verify(&compact, &audit_store).is_err());
|
||||
std::fs::write(&object, &original)?;
|
||||
assert!(argand_site_registry::audit::verify(&compact, &audit_store).is_ok());
|
||||
std::fs::remove_file(&object)?;
|
||||
assert!(argand_site_registry::audit::verify(&compact, &audit_store).is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wikidata_dump_streams_multiline_entities_with_an_authenticated_large_record_bound()
|
||||
-> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = store::open(&root.path().join("writer.sqlite"))?;
|
||||
let mut entity = common::entity("Q355", "Facebook", &["FB"], &["https://facebook.com/"]);
|
||||
entity["unconsumed_large_field"] = Value::String("x".repeat(17 * 1024 * 1024));
|
||||
let bytes = serde_json::to_vec_pretty(&vec![entity])?;
|
||||
let input = root.path().join("wikidata-large.json");
|
||||
std::fs::write(&input, &bytes)?;
|
||||
|
||||
let mut legacy = common::manifest(Source::Wikidata, Format::WikidataDump, &bytes)?;
|
||||
assert!(store::import(&mut db, &legacy, &input).is_err());
|
||||
legacy.schema = "argand.site-source/v3".into();
|
||||
legacy.maximum_record_bytes = Some(32 * 1024 * 1024);
|
||||
legacy.integrity = vec![IntegrityProof {
|
||||
method: "content_digest".into(),
|
||||
algorithm: "sha256".into(),
|
||||
value: legacy.sha256.clone(),
|
||||
evidence_url: None,
|
||||
}];
|
||||
legacy.lineage = Some(SourceLineage::direct(Source::Wikidata));
|
||||
let source_id = store::import(&mut db, &legacy, &input)?;
|
||||
let records: i64 = db.query_row(
|
||||
"SELECT count(*) FROM records WHERE source_id=?1",
|
||||
[source_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
assert_eq!(records, 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wikidata_json_delta_turns_removed_websites_into_auditable_tombstones() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = store::open(&root.path().join("writer.sqlite"))?;
|
||||
common::import(
|
||||
&mut db,
|
||||
root.path(),
|
||||
Source::Psl,
|
||||
Format::PslText,
|
||||
common::PSL.as_bytes(),
|
||||
)?;
|
||||
let base_bytes = serde_json::to_vec(&vec![common::entity(
|
||||
"Q355",
|
||||
"Facebook",
|
||||
&[],
|
||||
&["https://facebook.com/"],
|
||||
)])?;
|
||||
let mut base = common::manifest(Source::Wikidata, Format::WikidataDump, &base_bytes)?;
|
||||
base.schema = "argand.site-source/v3".into();
|
||||
base.coverage = Some(SourceCoverage {
|
||||
collection: "wikidata-entities".into(),
|
||||
kind: CoverageKind::Full,
|
||||
partition: None,
|
||||
base: None,
|
||||
sequence: None,
|
||||
supersedes: Vec::new(),
|
||||
});
|
||||
base.integrity = vec![IntegrityProof {
|
||||
method: "content_digest".into(),
|
||||
algorithm: "sha256".into(),
|
||||
value: base.sha256.clone(),
|
||||
evidence_url: None,
|
||||
}];
|
||||
base.lineage = Some(SourceLineage::direct(Source::Wikidata));
|
||||
let base_input = root.path().join("wikidata-base.json");
|
||||
std::fs::write(&base_input, &base_bytes)?;
|
||||
let base_id = store::import(&mut db, &base, &base_input)?;
|
||||
|
||||
let removed =
|
||||
json!({"id":"Q355","type":"item","lastrevid":2,"labels":{},"aliases":{},"claims":{}});
|
||||
let delta_bytes = serde_json::to_vec(&vec![removed])?;
|
||||
let mut delta = common::manifest(Source::Wikidata, Format::WikidataDump, &delta_bytes)?;
|
||||
delta.schema = "argand.site-source/v3".into();
|
||||
delta.retrieved_at += chrono::Duration::days(1);
|
||||
delta.coverage = Some(SourceCoverage {
|
||||
collection: "wikidata-entities".into(),
|
||||
kind: CoverageKind::Delta,
|
||||
partition: None,
|
||||
base: Some(base_id.clone()),
|
||||
sequence: Some(1),
|
||||
supersedes: vec![base_id],
|
||||
});
|
||||
delta.integrity = vec![IntegrityProof {
|
||||
method: "content_digest".into(),
|
||||
algorithm: "sha256".into(),
|
||||
value: delta.sha256.clone(),
|
||||
evidence_url: None,
|
||||
}];
|
||||
delta.lineage = Some(SourceLineage::direct(Source::Wikidata));
|
||||
let delta_input = root.path().join("wikidata-delta.json");
|
||||
std::fs::write(&delta_input, &delta_bytes)?;
|
||||
store::import(&mut db, &delta, &delta_input)?;
|
||||
|
||||
let generation = root.path().join("compact-delta");
|
||||
let audit_store = root.path().join("delta-audit");
|
||||
build::build_compact_with_policy_and_trust(
|
||||
&db,
|
||||
&generation,
|
||||
&ReviewPolicy::legacy_compatible(),
|
||||
None,
|
||||
&audit_store,
|
||||
)?;
|
||||
let registry = Registry::open(
|
||||
&generation,
|
||||
&argand_site_registry::file_digest(&generation.join("COMPLETE.json"))?,
|
||||
)?;
|
||||
assert_eq!(registry.lookup("Facebook", 20)?.total_entities, 0);
|
||||
let export = root.path().join("delta-audit.jsonl");
|
||||
argand_site_registry::audit::export(®istry, &audit_store, &export, false)?;
|
||||
let export = std::fs::read_to_string(export)?;
|
||||
assert!(export.contains("tombstone"));
|
||||
let replaced = &delta
|
||||
.coverage
|
||||
.as_ref()
|
||||
.context("delta coverage")?
|
||||
.supersedes[0];
|
||||
assert!(export.contains(replaced));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn killed_import_resumes_from_a_committed_checkpoint() -> anyhow::Result<()> {
|
||||
const RECORDS: usize = 30_000;
|
||||
let root = tempfile::tempdir()?;
|
||||
let database = root.path().join("writer.sqlite");
|
||||
drop(store::open(&database)?);
|
||||
let mut dump = String::from("[");
|
||||
for index in 0..RECORDS {
|
||||
if index > 0 {
|
||||
dump.push(',');
|
||||
}
|
||||
dump.push_str(&serde_json::to_string(&common::entity(
|
||||
&format!("Q{}", 10_000_000 + index),
|
||||
&format!("Checkpoint {index}"),
|
||||
&[],
|
||||
&["https://example.org/"],
|
||||
))?);
|
||||
}
|
||||
dump.push(']');
|
||||
let bytes = common::gzip(dump.as_bytes())?;
|
||||
let mut manifest = common::manifest(Source::Wikidata, Format::WikidataDump, &bytes)?;
|
||||
manifest.compression = argand_site_registry::model::Compression::Gzip;
|
||||
let source_id = manifest.id()?;
|
||||
let input = root.path().join("large.json.gz");
|
||||
let declaration = root.path().join("source.json");
|
||||
std::fs::write(&input, bytes)?;
|
||||
std::fs::write(&declaration, serde_json::to_vec_pretty(&manifest)?)?;
|
||||
|
||||
for target in [256_i64, 2_048] {
|
||||
let mut child = Command::new(env!("CARGO_BIN_EXE_argand-site-registry"))
|
||||
.args(["import", "--database"])
|
||||
.arg(&database)
|
||||
.arg("--input")
|
||||
.arg(&input)
|
||||
.arg("--manifest")
|
||||
.arg(&declaration)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()?;
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let checkpoint = loop {
|
||||
if child.try_wait()?.is_some() {
|
||||
anyhow::bail!("fixture import completed before checkpoint interruption");
|
||||
}
|
||||
let connection = rusqlite::Connection::open(&database)?;
|
||||
let observed: i64 = connection.query_row(
|
||||
"SELECT COALESCE(max(checkpoint),0) FROM sources WHERE id=?1",
|
||||
[&source_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if observed >= target {
|
||||
child.kill()?;
|
||||
child.wait()?;
|
||||
break observed;
|
||||
}
|
||||
ensure!(Instant::now() < deadline, "import checkpoint timed out");
|
||||
thread::sleep(Duration::from_millis(2));
|
||||
};
|
||||
assert!(checkpoint >= target);
|
||||
}
|
||||
let connection = rusqlite::Connection::open(&database)?;
|
||||
let partial: (i64, bool) = connection.query_row(
|
||||
"SELECT checkpoint,complete FROM sources WHERE id=?1",
|
||||
[&source_id],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
assert!(partial.0 >= 256 && !partial.1);
|
||||
drop(connection);
|
||||
|
||||
let status = Command::new(env!("CARGO_BIN_EXE_argand-site-registry"))
|
||||
.args(["import", "--database"])
|
||||
.arg(&database)
|
||||
.arg("--input")
|
||||
.arg(&input)
|
||||
.arg("--manifest")
|
||||
.arg(&declaration)
|
||||
.stdout(Stdio::null())
|
||||
.status()?;
|
||||
ensure!(status.success(), "checkpoint replay failed");
|
||||
let connection = store::open(&database)?;
|
||||
let completed: (i64, bool) = connection.query_row(
|
||||
"SELECT checkpoint,complete FROM sources WHERE id=?1",
|
||||
[&source_id],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
assert_eq!(completed, (i64::try_from(RECORDS)?, true));
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -11,7 +11,9 @@ flowchart LR
|
|||
I --> W[(Writer store)]
|
||||
O[Bounded candidate observer] --> B[Immutable observation batches]
|
||||
B --> W
|
||||
W --> G[Candidate generation]
|
||||
W --> A[Content-addressed cold audit bundles]
|
||||
W --> G[Compact candidate generation]
|
||||
A -. digest references .-> G
|
||||
G --> Q[Evidence bundles and review queue]
|
||||
Q --> V[Signed reviewer votes]
|
||||
V --> W
|
||||
|
|
@ -24,12 +26,19 @@ flowchart LR
|
|||
|
||||
## Data boundaries
|
||||
|
||||
`sources`, `records`, and `facts` preserve provider-native evidence. Typed
|
||||
The writer's `sources`, `records`, and `facts` preserve provider-native evidence. Typed
|
||||
coverage selects a coherent active set without deleting older snapshots.
|
||||
`selected_sources` and `active_records` record that derivation. Names, entities,
|
||||
web properties, edges, popularity, and rejected facts are deterministic
|
||||
projections.
|
||||
|
||||
A compact build packages every completed source into a content-addressed cold
|
||||
JSONL bundle before removing superseded records and unprojected fact history from
|
||||
the runtime copy. `COMPLETE.json` binds every bundle and the runtime database.
|
||||
Normal queries need only the compact generation. Audit export, verification and
|
||||
retention checkpoints stream the external bundle store and fail on absence,
|
||||
substitution, truncation, malformed order, or mismatched source/attribution.
|
||||
|
||||
Names, website edges, and entity equivalences have separate material
|
||||
fingerprints. Adding an alias therefore cannot inherit an approved destination,
|
||||
and changing unrelated entity metadata does not invalidate an unchanged website
|
||||
|
|
@ -72,7 +81,7 @@ unlinked snapshot before opening them.
|
|||
| `source`, `store`, adapters | Manifest validation and streaming source-specific import |
|
||||
| `coverage` | Full/partition/delta graph validation and active-record masking |
|
||||
| `normalize` | Deterministic URL, hostname, registrable-domain, suffix, and name normalization |
|
||||
| `build` | Canonical immutable generation and receipt creation |
|
||||
| `build`, `audit` | Canonical compact generation, cold history, verification and retention |
|
||||
| `bundle`, `policy`, `vote` | Review evidence, policy epochs, authenticated quorum, revocation |
|
||||
| `observer`, `observation`, `queue` | Candidate-only collection, replay/import, reverse lookup, drift and queues |
|
||||
| `query`, `resolution`, `identity`, `catalog` | Audit lookup, equivalence, resolution, statistics |
|
||||
|
|
|
|||
80
docs/BENCHMARKING.md
Normal file
80
docs/BENCHMARKING.md
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
# Benchmarking and recovery
|
||||
|
||||
`scripts/benchmark.py` runs one import, build, audit verification, or evaluation
|
||||
command without a shell and creates a new evidence directory. `REPORT.json`
|
||||
records exact source-manifest and source-object hashes, hardware, wall/CPU time,
|
||||
peak RSS, database/record/fact growth, hashes of stdout/stderr, optional measured
|
||||
artifacts, and a hash of the argument vector. Source bytes are never copied into
|
||||
the report. The command refuses to overwrite an existing directory.
|
||||
|
||||
Use three profiles consistently:
|
||||
|
||||
- `small`: synthetic or source-shaped fixtures used for correctness and rapid
|
||||
regression checks;
|
||||
- `medium`: a documented provider subset large enough to exercise checkpoints,
|
||||
bounds and projection behavior; and
|
||||
- `provider`: one complete current official provider object in its native format.
|
||||
|
||||
Example import measurement:
|
||||
|
||||
```bash
|
||||
python3 scripts/benchmark.py --profile provider \
|
||||
--output /data/benchmarks/ror-v2.12-import \
|
||||
--source-manifest /data/cache/ror/source.json \
|
||||
--database /data/ror-writer.sqlite --expanded-bytes 362619018 -- \
|
||||
argand-site-registry import --database /data/ror-writer.sqlite \
|
||||
--input /data/cache/ror/SHA256 --manifest /data/cache/ror/source.json
|
||||
```
|
||||
|
||||
For build measurements, label exact regular files with repeated
|
||||
`--measured-path NAME=PATH`. Run evaluation with at least 1,000 representative
|
||||
cases in one process so authenticated startup/copy time remains separate from
|
||||
the report's native per-query p50/p95. Store reports outside Git because commands
|
||||
and snapshots can reveal private operational structure. A report establishes one
|
||||
machine and object only; synthetic numbers are never provider-capacity evidence.
|
||||
|
||||
## Version 0.5 provider canary
|
||||
|
||||
The 2026-09-13 canary used official ROR release `v2.12-2026-08-25`, Zenodo record
|
||||
`22099990`. The compressed object was 36,246,232 bytes, SHA-256
|
||||
`5779c7baf71771fd8ea829201e7bd4343a3c68ff36c595f480b3a00292f78931`, and
|
||||
matched provider MD5 `ce8807691455d4ada3216c31408e9e1a`. Its JSON and CSV
|
||||
members expanded to 362,619,018 bytes.
|
||||
|
||||
The final streaming import produced 137,398 records and 914,439 facts in 62.30
|
||||
seconds (9.47 user, 13.24 system), with 25,032 KiB peak RSS and about 14,678
|
||||
facts/second. It grew the PSL-initialized writer by 828,682,240 bytes to
|
||||
829,505,536 bytes. Reimport replayed the complete pinned source in 0.025 seconds
|
||||
without changing any record, fact, or database byte count.
|
||||
|
||||
A compact build combining that ROR source with the current PSL projected 137,398
|
||||
entities, 131,591 properties and 133,398 edges. The pre-compaction database was
|
||||
1,345,097,728 bytes; the runtime database was 824,705,024 bytes, a 38.688%
|
||||
reduction. The initial compact build took 58.36 seconds with 45,140 KiB peak RSS;
|
||||
a second build reused the same bundles and produced byte-identical database and
|
||||
receipt bytes. The two cold bundles totaled 686,468,048 bytes and held 137,399
|
||||
records plus 914,440 facts. Warm-cache full streaming audit verification took
|
||||
3.03 seconds with 24,076 KiB peak RSS.
|
||||
|
||||
A 1,000-case safe-abstention evaluation passed 1,000/1,000 with native reusable
|
||||
reader latency of 72 microseconds p50 and 96 microseconds p95. The warm-cache full
|
||||
process took 0.83 seconds, including authentication and the private copy of the
|
||||
786.5 MiB SQLite file. Serving deployments must reuse the verified `Registry` and
|
||||
must measure cold startup on their own storage.
|
||||
|
||||
Hardware: Linux 6.18.51-1-lts x86_64, AMD Ryzen 9 7900X (12 cores, 24 logical
|
||||
CPUs), 61 GiB RAM, ext4. These numbers are a single local canary and not a public
|
||||
service capacity claim.
|
||||
|
||||
## Recovery matrix
|
||||
|
||||
The acceptance suite sends real process termination after two distinct committed
|
||||
checkpoints and proves the same import completes exactly once on restart. The
|
||||
provider canary also killed the final binary at a 2,304-record checkpoint via a
|
||||
bounded-runner fault, recovered its hot journal, and reached the exact clean-import
|
||||
record/fact totals. Other tests cover handled parser failure rollback, SQLite growth-cap
|
||||
(`SQLITE_FULL`) behavior, truncated/compressed inputs, mutated cache objects,
|
||||
duplicate records and JSON keys, archive-member drift, missing/truncated/substituted
|
||||
cold bundles, existing publication targets, failed atomic writes, and unchanged
|
||||
previous generations after update failure. Provider objects and benchmark output
|
||||
remain outside the repository.
|
||||
|
|
@ -34,8 +34,8 @@ source attribution and application-specific malware/content policy.
|
|||
|
||||
## Current contracts
|
||||
|
||||
Code version 0.4.0 uses writer schema 5 and `argand.site-rules/v4`.
|
||||
`COMPLETE.json` uses `argand.site-registry/v2` and binds:
|
||||
Code version 0.5.0 uses writer schema 5 and `argand.site-rules/v4`.
|
||||
New compact `COMPLETE.json` files use `argand.site-registry/v3` and bind:
|
||||
|
||||
- authenticated `registry.sqlite` bytes;
|
||||
- the active source coverage graph and exact PSL source;
|
||||
|
|
@ -43,12 +43,23 @@ Code version 0.4.0 uses writer schema 5 and `argand.site-rules/v4`.
|
|||
- decision-time policy;
|
||||
- source license and machine-readable attribution files; and
|
||||
- entity, property, edge and rejection counts.
|
||||
- every cold audit bundle's digest, size, counts, source, selection state,
|
||||
attribution digest and coverage digest.
|
||||
|
||||
Normal `export` uses `argand.site-export/v2`, contains selected active nonrejected
|
||||
facts, and marks every assertion `active`. `export-audit` uses the same schema with
|
||||
mode `audit` and includes superseded/rejected states and rejection reasons. Neither
|
||||
mode `audit` for v2 generations. Compact generations require their external audit
|
||||
store and emit `argand.site-export/v3` after verifying every referenced bundle.
|
||||
Both include superseded/rejected states and rejection reasons. Neither
|
||||
contains a list of resolver-approved routes.
|
||||
|
||||
Keep the audit store separate from the runtime deployment, replicate it by exact
|
||||
object digest, and run `verify-audit` before retention attestations or audit export.
|
||||
Runtime resolution verifies the compact generation receipt and database without
|
||||
opening cold history. A missing audit object therefore does not silently change a
|
||||
query, but it is an audit/retention failure and blocks any claim of complete
|
||||
provenance. Retention checkpoints never authorize deletion.
|
||||
|
||||
Votes use `argand.site-vote/v1` and the OpenSSH namespace
|
||||
`argand-site-registry-vote`. Consumers compile them under the exact receipt-bound
|
||||
policy and trusted query time. Unknown schema or rule versions fail closed.
|
||||
|
|
@ -77,7 +88,8 @@ resolution policy, expiry, revocation continuity or signature verification.
|
|||
|
||||
V1 source manifests remain readable as isolated legacy provider/scope streams.
|
||||
A deliberate legacy-compatible build policy can replay v0.3 reviews, but strict
|
||||
0.4 builds require votes and reviewer trust. Current readers accept v2 receipts;
|
||||
0.4 builds require votes and reviewer trust. Current readers accept v2 and compact
|
||||
v3 receipts;
|
||||
rollback checks may open pinned v1 receipts with rules v1-v3 only to compare
|
||||
revocation history.
|
||||
|
||||
|
|
@ -93,7 +105,7 @@ The last recorded downstream integration replaced Argand's embedded crate with
|
|||
signed v0.3.0 revision `ac8282093d8a815c6227cff86e1f40714d510bcd` at Argand
|
||||
commit `d9dfd1585ce21d9c4136bcc24fa01fe3bfb8ed6e`.
|
||||
|
||||
Version 0.4 is handed off as a signed standalone revision. Argand should update its
|
||||
Version 0.5 is handed off as a signed standalone revision. Argand should update its
|
||||
full Git `rev` in a separate coordinated source/build window, compare contract
|
||||
changes, and rerun navigation compiler, native resolver, API, abstention,
|
||||
revocation and clean-process gates. Changing the code dependency does not activate
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
# Resolver evaluation
|
||||
|
||||
`evaluate` streams authored JSONL judgments through the same pinned native reader
|
||||
used by consumers. It reports correctness and p50/p95 latency. Pass `--at` for a
|
||||
used by consumers. Its v2 report includes every active source snapshot and declared
|
||||
lineage (explicit `null` for legacy unknown lineage), plus correctness and p50/p95
|
||||
latency. Pass `--at` for a
|
||||
reproducible policy clock; approval expiry and future votes otherwise depend on
|
||||
current time. Inputs are bounded to 16 MiB, lines to 64 KiB, and case IDs must be
|
||||
unique.
|
||||
|
|
|
|||
|
|
@ -1,13 +1,19 @@
|
|||
# Versioned formats
|
||||
|
||||
Version 0.4 uses writer schema 5 and `argand.site-rules/v4`. Schema identifiers
|
||||
Version 0.5 uses writer schema 5 and `argand.site-rules/v4`. Schema identifiers
|
||||
are independent from the crate version. Unknown schemas and rules fail closed.
|
||||
|
||||
| Artifact | Current schema | Purpose |
|
||||
| --- | --- | --- |
|
||||
| Source manifest | `argand.site-source/v2` | Exact source object plus typed coverage |
|
||||
| Generation receipt | `argand.site-registry/v2` | Hash-bound immutable generation contract |
|
||||
| Source manifest | `argand.site-source/v3` | Exact source object, integrity proof, lineage, parser bound and typed coverage |
|
||||
| Generation receipt | `argand.site-registry/v3` | Compact runtime plus content-addressed audit references |
|
||||
| Active/audit JSONL | `argand.site-export/v2` | Source-bearing assertions with selection state |
|
||||
| Compact audit JSONL | `argand.site-export/v3` | Verified external history with bundle references and tombstones |
|
||||
| Cold audit bundle | `argand.site-audit-bundle/v2` | Source manifest and ordered record/fact history |
|
||||
| Audit verification | `argand.site-audit-verification/v1` | Streaming bundle verification totals |
|
||||
| Audit retention checkpoint | `argand.site-audit-retention/v1` | Signed no-delete retention set |
|
||||
| Benchmark report | `argand.site-benchmark/v1` | Source-pinned operation and resource evidence |
|
||||
| Evaluation report | `argand.site-evaluation/v2` | Resolver judgments, latency, and active source lineage |
|
||||
| Evidence bundle | `argand.site-evidence-bundle/v1` | Exact evidence signed by reviewer votes |
|
||||
| Vote | `argand.site-vote/v1` | Authenticated approve/revoke decision |
|
||||
| Review policy | `argand.site-policy/v1` | Threshold, groups, revocation, and separation rules |
|
||||
|
|
@ -18,12 +24,30 @@ are independent from the crate version. Unknown schemas and rules fail closed.
|
|||
| Active pointer | `argand.site-current/v2` | Signed generation pin plus revocation continuity |
|
||||
| Diff | `argand.site-diff/v4` | Typed change stream across generations |
|
||||
|
||||
Source manifest v2 coverage is one of `full`, `partition`, or `delta`. A delta
|
||||
Source manifest v3 adds explicit `integrity`, `lineage`, and an optional
|
||||
`maximum_record_bytes` parser ceiling to v2 typed coverage. Integrity always
|
||||
contains the locally computed SHA-256 and may also carry a source-bound provider
|
||||
checksum. Lineage names the direct provider, upstream datasets, transformations,
|
||||
and a conservative independence group. A missing legacy lineage serializes as
|
||||
unknown and never becomes evidence of independence.
|
||||
|
||||
Coverage is one of `full`, `partition`, or `delta`. A delta
|
||||
names its exact base, positive consecutive sequence, and superseded source IDs.
|
||||
A full source cannot compose with active partitions. Overlap, gaps, cycles,
|
||||
missing bases, cross-provider supersession, and mixed legacy/typed frontiers fail
|
||||
the build.
|
||||
|
||||
Generation v3 uses `runtime_layout: "compact-v1"`. Its local SQLite database keeps
|
||||
the selected source manifests, active facts and projections, review/policy state,
|
||||
and the evidence needed by runtime queries. Each complete source history is first
|
||||
written to an external JSONL object. The receipt binds its SHA-256, length, source,
|
||||
record/fact counts, selected state, attribution digest and coverage digest.
|
||||
`verify-audit` re-hashes and parses every object before an audit export or retention
|
||||
checkpoint. The separate retention signature namespace is
|
||||
`argand-site-registry-audit-retention`; every checkpoint contains
|
||||
`deletion_authorized: false`. Generation v2 remains supported with embedded
|
||||
history for compatibility.
|
||||
|
||||
The normal export emits selected active facts only and excludes rejected facts.
|
||||
Every assertion has `selection_state: "active"`. Audit export includes active,
|
||||
superseded, and rejected facts; rejected assertions include their reason. Both
|
||||
|
|
|
|||
|
|
@ -11,10 +11,15 @@
|
|||
- [Trust](TRUST.md): enforced checks and publisher/consumer responsibilities.
|
||||
- [Publishing](PUBLISHING.md): reviewer keys, candidate acceptance and activation.
|
||||
- [Evaluation](EVALUATION.md): bounded JSONL judgments and result interpretation.
|
||||
- [Benchmarking and recovery](BENCHMARKING.md): repeatable profiles, evidence schema,
|
||||
provider canary, and failure matrix.
|
||||
- [Source candidates](SOURCE-CANDIDATES.md): admitted, held, correlated, and excluded providers.
|
||||
- [Releasing](RELEASING.md): CI, source signing and archive verification.
|
||||
- [Validation](VALIDATION.md): independent builds and native acceptance evidence.
|
||||
- [Version 0.4 security review](SECURITY-REVIEW-0.4.md): threat boundaries,
|
||||
resolved findings and residual operator responsibilities.
|
||||
- [Version 0.5 security review](SECURITY-REVIEW-0.5.md): compact audit storage,
|
||||
provider checksums, ZIP parsing, lineage and benchmark tooling.
|
||||
- [Contributing](../CONTRIBUTING.md), [governance](../GOVERNANCE.md),
|
||||
[security](../SECURITY.md): proposals, decisions and incidents.
|
||||
- [Extraction design](superpowers/specs/2026-09-12-standalone-design.md) and
|
||||
|
|
|
|||
105
docs/SECURITY-REVIEW-0.5.md
Normal file
105
docs/SECURITY-REVIEW-0.5.md
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
# Version 0.5 security review
|
||||
|
||||
Date: 2026-09-13. Scope: all changes from signed version 0.4.0 through the 0.5
|
||||
implementation, including source manifest v3, ROR ZIP acquisition/import, streamed
|
||||
Wikidata JSON, compact generations, external audit bundles, retention signatures,
|
||||
evaluation lineage, benchmark tooling, dependencies and operator documentation.
|
||||
|
||||
No unresolved critical, high, or medium-severity finding remained at release
|
||||
validation. This is a source review and adversarial test result, not a claim that
|
||||
an imported URL is safe or that a future public registry has been reviewed.
|
||||
|
||||
## Boundaries reviewed
|
||||
|
||||
- Provider responses, manifests, JSON/CSV/XML/ZIP/archive members, source records,
|
||||
names, URLs, lineages and audit-store entries are untrusted input.
|
||||
- A source assertion cannot authorize a route. Name, edge and equivalence authority
|
||||
still comes from policy-qualified reviewer signatures; release authority remains
|
||||
a separate publisher signature and activation step.
|
||||
- The cache, writer, audit store, reviewer trust file and keys are operator-owned
|
||||
local resources. Consumer trust starts with an independently obtained receipt
|
||||
pin or publisher trust root.
|
||||
- The project ships no hosted API, browser rendering surface, credential store,
|
||||
production key, preapproved dataset, automatic signer, or automatic activation.
|
||||
|
||||
## Findings fixed
|
||||
|
||||
| ID | Severity before fix | Resolution |
|
||||
| --- | --- | --- |
|
||||
| SR-05-01 | Medium | Audit-store discovery previously used the 512 MiB fact-line ceiling for an untrusted unmatched header. Header reads now stop at 1 MiB before JSON materialization; facts remain bounded and content-authenticated. |
|
||||
| SR-05-02 | Medium | SQLite can auto-rollback a transaction on `SQLITE_FULL`; an unconditional second rollback could return before cleanup and leave a committed incomplete checkpoint. Recovery now detects autocommit, always attempts source discard and growth-ceiling restoration, and has a real growth-cap regression. |
|
||||
| SR-05-03 | Medium | A syntactically valid compact receipt could omit a selected audit reference while still naming other bundles. Opening now validates every source declaration, exact PSL identity, receipt-to-database selected-source equality, and exact equality between selected bundle references and active receipt sources. |
|
||||
| SR-05-04 | Medium | Initial ROR ZIP limits trusted member size metadata. JSON and CSV decompression now have independent actual-byte ceilings in addition to record, count, database-growth, member-count, compression-method, encryption and name checks. |
|
||||
| SR-05-05 | Medium | Foreign keys were disabled during an indexed compaction transaction and the first error on rollback could bypass restoration. Every exit now attempts rollback as needed and restores foreign-key enforcement before returning. Publication still requires `integrity_check` and a zero-row `foreign_key_check`. |
|
||||
| SR-05-06 | Low | The first benchmark stdout cap applied `RLIMIT_FSIZE` to the whole child and could kill SQLite writes. The final runner uses bounded stdout/stderr pipes, kills the isolated process group only on output overflow, records the condition, and never limits dataset/database files. The induced hot journal was recovered and the import resumed to the clean-import totals. |
|
||||
|
||||
The review also capped authenticated custom records at 256 MiB so generated cold
|
||||
bundle lines remain within their verifier ceiling. Removed Wikidata statements
|
||||
carry their exact superseded source IDs in tombstone exports. ROR inactive and
|
||||
withdrawn website assertions remain auditable but ineligible.
|
||||
|
||||
## Acquisition and archive checks
|
||||
|
||||
Source URLs use an HTTPS provider allowlist, reject credentials/fragments/custom
|
||||
ports and revalidate every redirect. Resumption requires a strong unchanged ETag,
|
||||
exact final URL, matching total and Content-Range. Downloads have caller-supplied
|
||||
byte ceilings and immutable completion receipts. Provider checksum evidence is
|
||||
accepted only for reviewed source-bound locations: the matching Zenodo record for
|
||||
ROR or the same Wikimedia dump directory for Wikidata. ROR's MD5 is used only to
|
||||
match Zenodo's published field; the local content identity and all downstream
|
||||
references use SHA-256.
|
||||
|
||||
The ROR reader accepts exactly one version-matched JSON and CSV member, Stored or
|
||||
Deflated, with no path components, encryption or extra members. It streams schema
|
||||
2.1 JSON with duplicate-key rejection and fails on unknown top-level fields. Every
|
||||
source import re-hashes the exact no-follow input descriptor after parsing and
|
||||
publishes completion only after all records commit.
|
||||
|
||||
## Compact history and signing checks
|
||||
|
||||
Cold bundles use digest-derived names and are created without replacement. A
|
||||
reference binds exact bytes, source declaration, selected state, record/fact
|
||||
counts, coverage and attribution. Verification opens with `O_NOFOLLOW`, checks
|
||||
length and SHA-256 on the same descriptor, then parses bounded JSONL in required
|
||||
record-before-fact order. Audit export and retention checkpoint first verify every
|
||||
object. Retention signatures use the distinct
|
||||
`argand-site-registry-audit-retention` namespace, and their authenticated document
|
||||
always requires `deletion_authorized: false`; the software provides no deletion
|
||||
command.
|
||||
|
||||
Runtime compaction occurs only after all bundles are durable and verified. It
|
||||
uses an indexed exact fact keep-set, commits a deterministic runtime copy, restores
|
||||
foreign keys, then runs integrity and foreign-key checks before `COMPLETE.json` is
|
||||
created. Readers authenticate and copy SQLite into a private unlinked file before
|
||||
opening immutable mode, preserving the existing sidecar and post-open mutation
|
||||
defenses.
|
||||
|
||||
## Automated and manual evidence
|
||||
|
||||
- Workspace formatting, locked all-target checks, Clippy with warnings denied,
|
||||
strict rustdoc, Rust/Python tests, consumer parity, reproducible source packaging
|
||||
and extracted-source acceptance are release gates.
|
||||
- `cargo audit --deny warnings` loaded 1,243 RustSec advisories and reported no
|
||||
finding across 277 locked dependency nodes.
|
||||
- The repository and diff contain no private key, token, password, dataset,
|
||||
production review, or generated registry. SQL values remain parameterized;
|
||||
dynamic SQL fragments are fixed internal table/predicate vocabularies.
|
||||
- Adversarial tests cover duplicate JSON keys/records, schema/member drift,
|
||||
symlinks, truncation/substitution, oversized records, malformed coverage,
|
||||
process termination at multiple checkpoints, SQLite growth exhaustion,
|
||||
incomplete update/publication, receipt bundle omission, signature tampering,
|
||||
reviewer/publisher separation, revocation continuity and rollback.
|
||||
|
||||
## Residual operator responsibilities
|
||||
|
||||
Protect cache, writer and audit directories from untrusted local writers and
|
||||
replicate cold objects by exact digest. A deliberately configured 256 MiB record
|
||||
can require comparable memory; choose the smallest inspected limit and enforce
|
||||
process/storage quotas around provider jobs. Measure cold authenticated startup on
|
||||
deployment storage, because every registry open copies the full runtime database.
|
||||
|
||||
Obtain provider checksums and publisher/reviewer trust roots independently. A valid
|
||||
hash or signature authenticates bytes and a decision, not ownership, current site
|
||||
safety, commercial fitness, or reviewer competence. Continue malware/content,
|
||||
drift, expiry, revocation, backup/restore and incident controls before any public
|
||||
dataset or Argand route is activated.
|
||||
23
docs/SOURCE-CANDIDATES.md
Normal file
23
docs/SOURCE-CANDIDATES.md
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# Source admission status
|
||||
|
||||
This registry supports a provider only after current rights, distribution,
|
||||
schema, lineage, attribution, bounded import, and no-auto-approval behavior have
|
||||
all been reviewed. A research entry below is not permission to ingest it.
|
||||
|
||||
| Source | Status | Decision and next gate |
|
||||
| --- | --- | --- |
|
||||
| ROR | Admitted in 0.5 | The official CC0 schema 2.1 ZIP is streamed with exact Zenodo checksum evidence. Organization websites remain assertions; inactive/withdrawn edges are ineligible. GeoNames location lineage is explicit. |
|
||||
| MusicBrainz | Next adapter; held | The [official download documentation](https://musicbrainz.org/doc/MusicBrainz_Database/Download) identifies the core `mbdump.tar.bz2` snapshot as CC0. The live replication/edit/statistics material with noncommercial terms is excluded. Admission still needs a current core snapshot/checksum canary and a bounded relational-table adapter for documented [URL relationships](https://musicbrainz.org/doc/Style/Relationships/URLs). |
|
||||
| GND | Research hold | The [DNB open-data distribution](https://data.dnb.de/opendata/) must be checked at implementation time for the exact file license, current JSON-LD/RDF predicates, checksum and useful homepage coverage. Stop the adapter if explicit homepage coverage does not justify it. |
|
||||
| ORCID public data | Research hold | Its self-declared links need an individuals-only privacy, impersonation and volatility policy in addition to the [public-file terms](https://info.orcid.org/public-data-file-use-policy/). It could never auto-approve a route. |
|
||||
| OpenAlex institutions | Correlated-source hold | Institution metadata can inherit ROR. Any future use must declare ROR upstream and cannot count as independent website corroboration. See the [institution source documentation](https://help.openalex.org/data/institutions/). |
|
||||
| OpenStreetMap | License-architecture hold | No ingestion until an ODbL-compatible attribution, database-right and redistribution design is accepted. See the [OSMF license FAQ](https://osmfoundation.org/wiki/Licence_and_Legal_FAQ). |
|
||||
| Government/corporate registries | Jurisdiction hold | Review one jurisdiction and exact field at a time. Stable identifiers may support crosswalks; the registry cannot infer a website absent an authoritative field. |
|
||||
| DNS, RDAP, certificate transparency, package registries, web crawl data | Observation-only research | Exact commercial reuse terms and retention rules must be approved first. These sources describe current infrastructure and cannot establish entity ownership alone. |
|
||||
| Open Library and unresolved-rights sources | Excluded | Keep excluded until the underlying data rights and redistribution obligations are clear enough for commercial reuse. |
|
||||
|
||||
Cloudflare Radar, default Tranco, Cisco Umbrella, arbitrary mirrors, and any
|
||||
provider without verified commercial-reuse rights remain unsupported. Proposals
|
||||
must update `LICENSE_SOURCES.md`, add an allowlisted source/format pair and
|
||||
source-native test fixtures, and demonstrate that the adapter cannot create or
|
||||
renew an approved navigation route.
|
||||
|
|
@ -7,9 +7,10 @@ reach consumers.
|
|||
|
||||
## What the implementation enforces
|
||||
|
||||
Source adapters accept only documented providers and formats. Manifests bind the
|
||||
Source adapters accept only documented providers and formats. V3 manifests bind the
|
||||
exact object, origin URL, source-native snapshot, license, retrieval time, byte
|
||||
length, digest, and typed coverage. Full, partition and delta graphs reject gaps,
|
||||
length, local digest, source-bound provider checksum where available, per-record
|
||||
parser ceiling, typed coverage and declared lineage. Full, partition and delta graphs reject gaps,
|
||||
cycles, overlap, cross-provider replacement and ambiguous active branches. Failed
|
||||
or incomplete imports cannot replace complete evidence.
|
||||
|
||||
|
|
@ -19,6 +20,13 @@ entities. Source-specific popularity stays separate from identity. Every fact
|
|||
keeps source, source identifier, selector, license, retrieval time, confidence and
|
||||
raw evidence needed for audit.
|
||||
|
||||
Lineage declares the direct provider, known upstream datasets, provider
|
||||
transformations and a conservative independence group. ROR location metadata, for
|
||||
example, records GeoNames upstream. Unknown legacy lineage stays unknown. Source
|
||||
assertion count is not reviewer quorum: the current policy grants authority only
|
||||
to authenticated reviewer identities, independent groups and physical keys, so
|
||||
two providers cannot manufacture approval by copying one upstream claim.
|
||||
|
||||
Names, entity-to-property edges and entity equivalences have independent material
|
||||
fingerprints. Under the reference policy, `resolve` needs two independent votes
|
||||
for the matched name and two for the selected edge. Reviewer groups and physical
|
||||
|
|
@ -57,13 +65,20 @@ and malware-policy results without naming a vendor. No such provider is built in
|
|||
an operator must verify commercial-reuse terms and preserve its exact source and
|
||||
rights declaration before importing those records.
|
||||
|
||||
Generations bind authenticated SQLite bytes, selected coverage, policy, reviewer
|
||||
Compact generations bind authenticated SQLite bytes, selected coverage, policy, reviewer
|
||||
trust bytes, licenses, attribution, and decision-time rules into `COMPLETE.json`.
|
||||
Readers verify the receipt pin and copy the database into a private unlinked file
|
||||
before SQLite opens it. Release signing and activation reverify every stored
|
||||
signature. The strict policy rejects a publisher identity or physical key used for
|
||||
any reviewer vote.
|
||||
|
||||
Complete historical records and facts live in content-addressed cold bundles bound
|
||||
by the same receipt. Their verifier hashes the exact no-follow file descriptor and
|
||||
checks bytes, JSON order, source identity, selection state, counts, coverage and
|
||||
attribution. Runtime lookup cannot weaken when cold storage is offline; audit
|
||||
export and retention fail closed. A separately signed no-delete checkpoint makes
|
||||
the retained object set explicit without adding deletion authority.
|
||||
|
||||
## What a publisher must establish
|
||||
|
||||
The software verifies evidence integrity and decision authorization. A publisher
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
# ADR 0006: Source lineage and independence
|
||||
|
||||
Status: Accepted design; implementation scheduled for 0.5.
|
||||
Status: Implemented in 0.5.
|
||||
|
||||
Corroboration must describe the direct provider, upstream dataset, transformation,
|
||||
and snapshot. Two providers that copied the same upstream assertion do not count
|
||||
as independent evidence merely because their URLs differ. Unknown lineage stays
|
||||
unknown.
|
||||
|
||||
Version 0.4 preserves provider-native provenance and never combines popularity or
|
||||
same-domain evidence into ownership confidence. Version 0.5 will add explicit
|
||||
lineage fields and independence-aware corroboration without rewriting history.
|
||||
Version 0.5 adds explicit source-manifest lineage and exposes it in evidence and
|
||||
evaluation without rewriting legacy history. It preserves provider-native
|
||||
provenance and never combines popularity or same-domain evidence into ownership
|
||||
confidence. Current admission quorum is made only from authenticated reviewers,
|
||||
groups and physical keys, so copied source assertions cannot increase authority.
|
||||
|
||||
## Rejected alternatives
|
||||
|
||||
|
|
|
|||
|
|
@ -454,71 +454,75 @@ deltas and a separately signed, cumulative emergency block feed.
|
|||
|
||||
### Task 3.1: Separate runtime projections from cold audit history
|
||||
|
||||
- [ ] Keep raw cache objects immutable and content-addressed outside Git.
|
||||
- [ ] Package completed source imports into content-addressed audit bundles with
|
||||
- [x] Keep raw cache objects immutable and content-addressed outside Git.
|
||||
- [x] Package completed source imports into content-addressed audit bundles with
|
||||
manifest, record/fact indices, hashes, format version, and attribution.
|
||||
- [ ] Make a runtime generation contain selected facts, normalized projections,
|
||||
- [x] Make a runtime generation contain selected facts, normalized projections,
|
||||
active policy results, required votes/revocations, and signed bundle references.
|
||||
- [ ] Do not copy all historical records and facts into every runtime generation.
|
||||
- [ ] Add audit verification that streams referenced bundles and detects absence,
|
||||
- [x] Do not copy all historical records and facts into every runtime generation.
|
||||
- [x] Add audit verification that streams referenced bundles and detects absence,
|
||||
truncation, substitution, or mismatched attribution.
|
||||
- [ ] Add retention/checkpoint tooling that never deletes the only authenticated copy
|
||||
- [x] Add retention/checkpoint tooling that never deletes the only authenticated copy
|
||||
of evidence and produces a signed deletion/retention report.
|
||||
- [ ] Measure query latency and generation size before and after the split.
|
||||
- [x] Measure query latency and generation size before and after the split.
|
||||
|
||||
### Task 3.2: Add provider-scale benchmark and recovery tooling
|
||||
|
||||
- [ ] Define repeatable small, medium, and provider-representative import profiles.
|
||||
- [ ] Record wall time, CPU time, peak RSS, compressed and expanded bytes, database
|
||||
- [x] Define repeatable small, medium, and provider-representative import profiles.
|
||||
- [x] Record wall time, CPU time, peak RSS, compressed and expanded bytes, database
|
||||
growth, facts/second, checkpoint frequency, restart time, build size, and query
|
||||
latency.
|
||||
- [ ] Interrupt imports at multiple checkpoints and prove idempotent resumption.
|
||||
- [ ] Exercise disk-full, truncated input, cache corruption, duplicate records, and
|
||||
- [x] Interrupt imports at multiple checkpoints and prove idempotent resumption.
|
||||
- [x] Exercise disk-full, truncated input, cache corruption, duplicate records, and
|
||||
interrupted generation publication.
|
||||
- [ ] Make benchmark reports name exact source snapshot hashes and hardware without
|
||||
- [x] Make benchmark reports name exact source snapshot hashes and hardware without
|
||||
committing source data.
|
||||
- [ ] Treat synthetic performance as development evidence, not provider capacity.
|
||||
- [x] Treat synthetic performance as development evidence, not provider capacity.
|
||||
|
||||
### Task 3.3: Harden full Wikidata ingestion and add incremental refresh
|
||||
|
||||
- [ ] Confirm current official full and incremental formats from Wikidata documentation
|
||||
- [x] Confirm current official full and incremental formats from Wikidata documentation
|
||||
and inspected fixtures before changing the adapter.
|
||||
- [ ] Replace the assumption that every useful entity fits in one in-memory 16 MiB
|
||||
- [x] Replace the assumption that every useful entity fits in one in-memory 16 MiB
|
||||
line with bounded disk-spooling or an explicitly receipted oversized-record path.
|
||||
- [ ] Never silently skip an oversized entity that may contain a relevant fact.
|
||||
- [ ] Add incremental add/change ingestion with authenticated base snapshot identity,
|
||||
- [x] Never silently skip an oversized entity that may contain a relevant fact.
|
||||
- [x] Add incremental add/change ingestion with authenticated base snapshot identity,
|
||||
ordered application, checkpoints, and reconciliation against later full dumps.
|
||||
- [ ] Define how deletions and removed P856 statements become tombstones.
|
||||
- [ ] Keep full raw assertion/qualifier/reference provenance for consumed fields.
|
||||
- [ ] Make unrelated `lastrevid` changes visible in audit diffs without invalidating
|
||||
- [x] Define how deletions and removed P856 statements become tombstones.
|
||||
- [x] Keep full raw assertion/qualifier/reference provenance for consumed fields.
|
||||
- [x] Make unrelated `lastrevid` changes visible in audit diffs without invalidating
|
||||
unchanged material edge fingerprints.
|
||||
- [ ] Test real-format pathological entities and multistream compression boundaries.
|
||||
- [x] Test real-format pathological entities and multistream compression boundaries.
|
||||
|
||||
Authoritative format reference:
|
||||
|
||||
- <https://www.wikidata.org/wiki/Wikidata:Database_download>
|
||||
|
||||
Implementation note: the inspected Add/Change distribution is XML and the provider warns that embedded JSON in XML is unstable. Version 0.5 therefore implements ordered typed deltas from official Wikibase entity JSON and deliberately rejects the XML incremental artifact.
|
||||
|
||||
### Task 3.4: Strengthen current-source acquisition verification
|
||||
|
||||
- [ ] Prefer provider-published checksums or signatures when officially available and
|
||||
- [x] Prefer provider-published checksums or signatures when officially available and
|
||||
bind verification method into the source manifest.
|
||||
- [ ] Keep HTTPS allowlists, manual redirect validation, byte bounds, strong-validator
|
||||
- [x] Keep HTTPS allowlists, manual redirect validation, byte bounds, strong-validator
|
||||
resume rules, and immutable local cache behavior.
|
||||
- [ ] Detect and report source format drift before partial import can replace a source.
|
||||
- [ ] Add format-version canaries for Majestic, CrUX, Curlie, PSL, and Wikidata.
|
||||
- [ ] Preserve CrUX billing as explicit opt-in configuration and record job identity,
|
||||
- [x] Detect and report source format drift before partial import can replace a source.
|
||||
- [x] Add format-version canaries for Majestic, CrUX, Curlie, PSL, and Wikidata.
|
||||
- [x] Preserve CrUX billing as explicit opt-in configuration and record job identity,
|
||||
query, result period, and actual cost outside public fixtures.
|
||||
- [ ] Continue frequent PSL refresh and include exact PSL hash in normalization proofs.
|
||||
- [ ] Preserve Curlie attribution and description-redaction tests on every export path.
|
||||
- [x] Continue frequent PSL refresh and include exact PSL hash in normalization proofs.
|
||||
- [x] Preserve Curlie attribution and description-redaction tests on every export path.
|
||||
|
||||
### Task 3.5: Add source lineage and independence metadata
|
||||
|
||||
- [ ] Record direct provider, upstream/origin dataset, transformation, snapshot, and
|
||||
- [x] Record direct provider, upstream/origin dataset, transformation, snapshot, and
|
||||
known dependency relationships for each fact source.
|
||||
- [ ] Prevent policy from counting two assertions as independent corroboration when
|
||||
- [x] Prevent policy from counting two assertions as independent corroboration when
|
||||
one republishes the other.
|
||||
- [ ] Expose lineage in lookup, review bundles, export, diff, and evaluation.
|
||||
- [ ] Keep unknown lineage explicit rather than assuming independence.
|
||||
- [x] Expose lineage in lookup, review bundles, export, diff, and evaluation.
|
||||
- [x] Keep unknown lineage explicit rather than assuming independence.
|
||||
|
||||
Phase 3 completion note: compact v3 generations, audit bundle verification and no-delete retention, the source-pinned benchmark runner, two-checkpoint process-kill recovery, provider-scale ROR/PSL measurements, streamed Wikidata JSON arrays, authenticated record ceilings, typed JSON tombstones, source-bound checksums, current format canaries and explicit source lineage are implemented and tested in version 0.5.
|
||||
|
||||
## Phase 4: Rights-gated additional source adapters
|
||||
|
||||
|
|
@ -539,14 +543,14 @@ Every source follows the same gate:
|
|||
Rationale: ROR is CC0 and directly supplies stable organization IDs, names, aliases,
|
||||
status, locations, links, and domains. It is compact and well aligned with the model.
|
||||
|
||||
- [ ] Verify the current ROR schema version and official release asset from the ROR
|
||||
- [x] Verify the current ROR schema version and official release asset from the ROR
|
||||
data-dump documentation at implementation time.
|
||||
- [ ] Consume only fields confirmed in that inspected schema.
|
||||
- [ ] Map names and aliases without merging ROR entities into Wikidata entities unless
|
||||
- [x] Consume only fields confirmed in that inspected schema.
|
||||
- [x] Map names and aliases without merging ROR entities into Wikidata entities unless
|
||||
an exact external identifier or reviewed equivalence supports the join.
|
||||
- [ ] Preserve links and domains as ROR assertions, not approvals.
|
||||
- [ ] Preserve status, type, country/location, external IDs, and upstream lineage.
|
||||
- [ ] Test domain conflicts, former/inactive organizations, aliases, multiple links,
|
||||
- [x] Preserve links and domains as ROR assertions, not approvals.
|
||||
- [x] Preserve status, type, country/location, external IDs, and upstream lineage.
|
||||
- [x] Test domain conflicts, former/inactive organizations, aliases, multiple links,
|
||||
missing fields, duplicate input, schema drift, and exact-ID equivalence.
|
||||
|
||||
References:
|
||||
|
|
|
|||
278
scripts/benchmark.py
Normal file
278
scripts/benchmark.py
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Run one registry operation and write a bounded, source-pinned benchmark report."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import selectors
|
||||
import signal
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
MAX_MANIFEST_BYTES = 1024 * 1024
|
||||
MAX_OUTPUT_BYTES = 16 * 1024 * 1024
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def file_digest(path):
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
value.update(chunk)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
value = {}
|
||||
for key, item in pairs:
|
||||
if key in value:
|
||||
raise ValueError("duplicate JSON key")
|
||||
value[key] = item
|
||||
return value
|
||||
|
||||
|
||||
def read_manifest(path):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("source manifest must be a regular file")
|
||||
data = path.read_bytes()
|
||||
if len(data) > MAX_MANIFEST_BYTES:
|
||||
raise ValueError("source manifest exceeds 1 MiB")
|
||||
manifest = json.loads(data, object_pairs_hook=unique_object)
|
||||
required = ("schema", "source", "snapshot", "sha256", "bytes")
|
||||
if not isinstance(manifest, dict) or any(key not in manifest for key in required):
|
||||
raise ValueError("source manifest is incomplete")
|
||||
if (not isinstance(manifest["sha256"], str)
|
||||
or len(manifest["sha256"]) != 64
|
||||
or any(char not in "0123456789abcdef" for char in manifest["sha256"])):
|
||||
raise ValueError("source manifest has invalid SHA-256")
|
||||
if not isinstance(manifest["bytes"], int) or manifest["bytes"] <= 0:
|
||||
raise ValueError("source manifest has invalid byte count")
|
||||
return {
|
||||
"manifest_sha256": digest(data),
|
||||
"schema": manifest["schema"],
|
||||
"source": manifest["source"],
|
||||
"snapshot": manifest["snapshot"],
|
||||
"source_object_sha256": manifest["sha256"],
|
||||
"compressed_bytes": manifest["bytes"],
|
||||
}
|
||||
|
||||
|
||||
def database_counts(path):
|
||||
if path is None or not path.exists():
|
||||
return {"bytes": 0, "records": 0, "facts": 0}
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("database must be a regular file")
|
||||
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
||||
try:
|
||||
page_count = connection.execute("PRAGMA page_count").fetchone()[0]
|
||||
page_size = connection.execute("PRAGMA page_size").fetchone()[0]
|
||||
tables = {
|
||||
row[0]
|
||||
for row in connection.execute(
|
||||
"SELECT name FROM sqlite_schema WHERE type='table'"
|
||||
)
|
||||
}
|
||||
records = connection.execute("SELECT count(*) FROM records").fetchone()[0] \
|
||||
if "records" in tables else 0
|
||||
facts = connection.execute("SELECT count(*) FROM facts").fetchone()[0] \
|
||||
if "facts" in tables else 0
|
||||
return {
|
||||
"bytes": page_count * page_size,
|
||||
"records": records,
|
||||
"facts": facts,
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def bounded_file(path):
|
||||
if path.stat().st_size > MAX_OUTPUT_BYTES:
|
||||
return None
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def hardware():
|
||||
cpu = "unknown"
|
||||
cpuinfo = Path("/proc/cpuinfo")
|
||||
if cpuinfo.is_file():
|
||||
for line in cpuinfo.read_text(errors="replace").splitlines():
|
||||
if line.startswith("model name"):
|
||||
cpu = line.partition(":")[2].strip()
|
||||
break
|
||||
memory_kib = None
|
||||
meminfo = Path("/proc/meminfo")
|
||||
if meminfo.is_file():
|
||||
first = meminfo.read_text(errors="replace").splitlines()[0].split()
|
||||
if len(first) >= 2 and first[0] == "MemTotal:":
|
||||
memory_kib = int(first[1])
|
||||
return {
|
||||
"operating_system": platform.system(),
|
||||
"kernel": platform.release(),
|
||||
"architecture": platform.machine(),
|
||||
"cpu": cpu,
|
||||
"logical_cpus": os.cpu_count(),
|
||||
"memory_kib": memory_kib,
|
||||
}
|
||||
|
||||
|
||||
def run_bounded(command, stdout_path, stderr_path):
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
streams = selectors.DefaultSelector()
|
||||
streams.register(process.stdout, selectors.EVENT_READ, (stdout_path, 0))
|
||||
streams.register(process.stderr, selectors.EVENT_READ, (stderr_path, 0))
|
||||
exceeded = False
|
||||
outputs = {
|
||||
stdout_path: stdout_path.open("xb"),
|
||||
stderr_path: stderr_path.open("xb"),
|
||||
}
|
||||
try:
|
||||
while streams.get_map():
|
||||
for key, _events in streams.select():
|
||||
chunk = os.read(key.fileobj.fileno(), 64 * 1024)
|
||||
path, written = key.data
|
||||
if not chunk:
|
||||
streams.unregister(key.fileobj)
|
||||
key.fileobj.close()
|
||||
continue
|
||||
accepted = min(len(chunk), MAX_OUTPUT_BYTES - written)
|
||||
if accepted > 0:
|
||||
outputs[path].write(chunk[:accepted])
|
||||
written += accepted
|
||||
streams.modify(key.fileobj, selectors.EVENT_READ, (path, written))
|
||||
if accepted != len(chunk) and not exceeded:
|
||||
exceeded = True
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
return process.wait(), exceeded
|
||||
finally:
|
||||
streams.close()
|
||||
for output in outputs.values():
|
||||
output.close()
|
||||
|
||||
|
||||
def measured_artifacts(values):
|
||||
result = []
|
||||
labels = set()
|
||||
for value in values:
|
||||
label, separator, raw_path = value.partition("=")
|
||||
if not separator or not label or label in labels:
|
||||
raise ValueError("measured paths require unique LABEL=PATH values")
|
||||
labels.add(label)
|
||||
path = Path(raw_path)
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("measured artifact must be a regular file")
|
||||
result.append({
|
||||
"label": label,
|
||||
"bytes": path.stat().st_size,
|
||||
"sha256": file_digest(path),
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--profile", choices=("small", "medium", "provider"), required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--source-manifest", type=Path, action="append", default=[])
|
||||
parser.add_argument("--database", type=Path)
|
||||
parser.add_argument("--expanded-bytes", type=int)
|
||||
parser.add_argument("--measured-path", action="append", default=[])
|
||||
parser.add_argument("command", nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if not args.command or args.command[0] != "--" or len(args.command) == 1:
|
||||
parser.error("terminate options with -- and provide a command")
|
||||
command = args.command[1:]
|
||||
if args.output.exists():
|
||||
raise FileExistsError("benchmark output already exists")
|
||||
if args.expanded_bytes is not None and args.expanded_bytes < 0:
|
||||
raise ValueError("expanded bytes cannot be negative")
|
||||
sources = [read_manifest(path) for path in args.source_manifest]
|
||||
before = database_counts(args.database)
|
||||
args.output.mkdir(parents=True)
|
||||
stdout_path = args.output / "stdout"
|
||||
stderr_path = args.output / "stderr"
|
||||
started_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
usage_before = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
started = time.monotonic()
|
||||
exit_status, output_limit_exceeded = run_bounded(command, stdout_path, stderr_path)
|
||||
wall_seconds = time.monotonic() - started
|
||||
usage_after = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
after = database_counts(args.database)
|
||||
stdout = bounded_file(stdout_path)
|
||||
operation = None
|
||||
if stdout is not None:
|
||||
try:
|
||||
operation = json.loads(stdout, object_pairs_hook=unique_object)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
fact_growth = after["facts"] - before["facts"]
|
||||
report = {
|
||||
"schema": "argand.site-benchmark/v1",
|
||||
"profile": args.profile,
|
||||
"started_at": started_at,
|
||||
"hardware": hardware(),
|
||||
"sources": sources,
|
||||
"operation": {
|
||||
"program": Path(command[0]).name,
|
||||
"argument_vector_sha256": digest(b"\0".join(os.fsencode(item) for item in command)),
|
||||
"exit_status": exit_status,
|
||||
"output_limit_exceeded": output_limit_exceeded,
|
||||
"stdout_sha256": file_digest(stdout_path),
|
||||
"stderr_sha256": file_digest(stderr_path),
|
||||
"result": operation,
|
||||
},
|
||||
"metrics": {
|
||||
"wall_seconds": round(wall_seconds, 6),
|
||||
"user_cpu_seconds": round(usage_after.ru_utime - usage_before.ru_utime, 6),
|
||||
"system_cpu_seconds": round(usage_after.ru_stime - usage_before.ru_stime, 6),
|
||||
"peak_rss_kib": usage_after.ru_maxrss,
|
||||
"compressed_source_bytes": sum(source["compressed_bytes"] for source in sources),
|
||||
"expanded_source_bytes": args.expanded_bytes,
|
||||
"database_before": before,
|
||||
"database_after": after,
|
||||
"database_growth_bytes": after["bytes"] - before["bytes"],
|
||||
"record_growth": after["records"] - before["records"],
|
||||
"fact_growth": fact_growth,
|
||||
"facts_per_second": round(fact_growth / wall_seconds, 3)
|
||||
if fact_growth > 0 and wall_seconds > 0 else None,
|
||||
"import_checkpoint_records": 256,
|
||||
},
|
||||
"artifacts": measured_artifacts(args.measured_path),
|
||||
}
|
||||
report_path = args.output / "REPORT.json"
|
||||
with report_path.open("xb") as stream:
|
||||
stream.write(json.dumps(report, indent=2, sort_keys=True).encode() + b"\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
directory = os.open(args.output, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
return exit_status
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except (FileExistsError, OSError, ValueError) as error:
|
||||
print(error, file=sys.stderr)
|
||||
sys.exit(2)
|
||||
82
tests/test_benchmark.py
Normal file
82
tests/test_benchmark.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
"""Exercise the bounded benchmark evidence runner."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class BenchmarkTests(unittest.TestCase):
|
||||
def test_report_pins_source_and_refuses_overwrite(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
manifest = root / "source.json"
|
||||
manifest.write_text(json.dumps({
|
||||
"schema": "argand.site-source/v3",
|
||||
"source": "ror",
|
||||
"snapshot": "fixture",
|
||||
"sha256": "1" * 64,
|
||||
"bytes": 42,
|
||||
}))
|
||||
output = root / "benchmark"
|
||||
command = [
|
||||
sys.executable,
|
||||
"scripts/benchmark.py",
|
||||
"--profile", "small",
|
||||
"--output", str(output),
|
||||
"--source-manifest", str(manifest),
|
||||
"--", sys.executable, "-c", "print('{\"passed\": 1}')",
|
||||
]
|
||||
subprocess.run(command, check=True)
|
||||
report = json.loads((output / "REPORT.json").read_text())
|
||||
self.assertEqual(report["schema"], "argand.site-benchmark/v1")
|
||||
self.assertEqual(report["sources"][0]["source"], "ror")
|
||||
self.assertEqual(report["sources"][0]["source_object_sha256"], "1" * 64)
|
||||
self.assertEqual(report["operation"]["result"], {"passed": 1})
|
||||
refused = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
self.assertEqual(refused.returncode, 2)
|
||||
self.assertIn(b"already exists", refused.stderr)
|
||||
|
||||
def test_output_cap_does_not_limit_measured_files(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
output = root / "benchmark"
|
||||
large_file = root / "database-like-file"
|
||||
subprocess.run([
|
||||
sys.executable,
|
||||
"scripts/benchmark.py",
|
||||
"--profile", "small",
|
||||
"--output", str(output),
|
||||
"--measured-path", f"large={large_file}",
|
||||
"--", sys.executable, "-c",
|
||||
(
|
||||
"import json,sys; "
|
||||
"open(sys.argv[1],'wb').write(b'x'*(17*1024*1024)); "
|
||||
"print(json.dumps({'complete':True}))"
|
||||
),
|
||||
str(large_file),
|
||||
], check=True)
|
||||
report = json.loads((output / "REPORT.json").read_text())
|
||||
self.assertFalse(report["operation"]["output_limit_exceeded"])
|
||||
self.assertEqual(report["operation"]["result"], {"complete": True})
|
||||
self.assertEqual(report["artifacts"][0]["bytes"], 17 * 1024 * 1024)
|
||||
|
||||
capped = root / "capped"
|
||||
result = subprocess.run([
|
||||
sys.executable,
|
||||
"scripts/benchmark.py",
|
||||
"--profile", "small",
|
||||
"--output", str(capped),
|
||||
"--", sys.executable, "-c",
|
||||
"import sys; sys.stdout.buffer.write(b'x'*(17*1024*1024))",
|
||||
])
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
capped_report = json.loads((capped / "REPORT.json").read_text())
|
||||
self.assertTrue(capped_report["operation"]["output_limit_exceeded"])
|
||||
self.assertEqual((capped / "stdout").stat().st_size, 16 * 1024 * 1024)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue