feat: harden reviewed registry releases
All checks were successful
Standalone registry checks / check (push) Successful in 3m43s
All checks were successful
Standalone registry checks / check (push) Successful in 3m43s
This commit is contained in:
parent
0bcd4a2fc9
commit
e83f43d00f
37 changed files with 2856 additions and 286 deletions
23
CHANGELOG.md
Normal file
23
CHANGELOG.md
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Changelog
|
||||||
|
|
||||||
|
## 0.2.0 - 2026-09-13
|
||||||
|
|
||||||
|
- Open generations from an authenticated immutable SQLite file descriptor and
|
||||||
|
reject sidecars, symlinks and unexpected generation files.
|
||||||
|
- Authenticate exact reviewer decisions with SSH signatures, retain their proofs
|
||||||
|
and re-verify every decision before release signing.
|
||||||
|
- Explain resolver abstentions with typed statuses and decision counts.
|
||||||
|
- Add exact entity, URL/domain, source-separated popularity, redacted category and
|
||||||
|
registry-statistics audit views.
|
||||||
|
- Stream typed diffs across source selections, projections, decisions and explicit
|
||||||
|
equivalences.
|
||||||
|
- Add bounded native evaluation with multilingual, regional and deceptive-query
|
||||||
|
regressions plus latency evidence.
|
||||||
|
- Normalize rights-bearing future crawler observations without admitting them as
|
||||||
|
identity or ownership evidence.
|
||||||
|
|
||||||
|
## 0.1.0 - 2026-09-12
|
||||||
|
|
||||||
|
- Initial independent extraction with five rights-reviewed source adapters,
|
||||||
|
deterministic normalization, immutable generations and conservative regional
|
||||||
|
resolution.
|
||||||
9
Cargo.lock
generated
9
Cargo.lock
generated
|
|
@ -78,14 +78,14 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "argand-atomic"
|
name = "argand-atomic"
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"tempfile",
|
"tempfile",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "argand-site-registry"
|
name = "argand-site-registry"
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"argand-atomic",
|
"argand-atomic",
|
||||||
|
|
@ -95,6 +95,7 @@ dependencies = [
|
||||||
"csv",
|
"csv",
|
||||||
"flate2",
|
"flate2",
|
||||||
"http",
|
"http",
|
||||||
|
"libc",
|
||||||
"publicsuffix",
|
"publicsuffix",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
|
|
@ -212,9 +213,9 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chacha20"
|
name = "chacha20"
|
||||||
version = "0.10.1"
|
version = "0.10.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
|
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"cpufeatures 0.3.0",
|
"cpufeatures 0.3.0",
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ resolver = "3"
|
||||||
members = ["crates/argand-atomic", "crates/argand-site-registry"]
|
members = ["crates/argand-atomic", "crates/argand-site-registry"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
authors = ["Nic Weyand"]
|
authors = ["Nic Weyand"]
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "AGPL-3.0-or-later"
|
license = "AGPL-3.0-or-later"
|
||||||
|
|
@ -16,6 +16,7 @@ chrono = { version = "0.4.44", features = ["serde"] }
|
||||||
clap = { version = "4.5.60", features = ["derive"] }
|
clap = { version = "4.5.60", features = ["derive"] }
|
||||||
flate2 = "1.1.9"
|
flate2 = "1.1.9"
|
||||||
http = "1.4.0"
|
http = "1.4.0"
|
||||||
|
libc = "0.2.189"
|
||||||
reqwest = { version = "0.13.2", default-features = false, features = ["json", "query", "rustls", "stream"] }
|
reqwest = { version = "0.13.2", default-features = false, features = ["json", "query", "rustls", "stream"] }
|
||||||
serde = { version = "1.0.228", features = ["derive"] }
|
serde = { version = "1.0.228", features = ["derive"] }
|
||||||
serde_json = "1.0.149"
|
serde_json = "1.0.149"
|
||||||
|
|
|
||||||
|
|
@ -15,8 +15,8 @@ a consumer's accepted destinations or signing keys.
|
||||||
Policy, license, normalization, source-allowlist and signature changes receive
|
Policy, license, normalization, source-allowlist and signature changes receive
|
||||||
explicit maintainer review and complete acceptance checks. Additional independent
|
explicit maintainer review and complete acceptance checks. Additional independent
|
||||||
review is appropriate for trust-boundary changes when another qualified reviewer
|
review is appropriate for trust-boundary changes when another qualified reviewer
|
||||||
is available. This is a governance expectation; the current software does not
|
is available. The software authenticates each decision to an allowed SSH reviewer
|
||||||
enforce a multi-reviewer quorum or authenticate a free-text reviewer name.
|
identity, but does not enforce a multi-reviewer quorum.
|
||||||
|
|
||||||
Corrections and appeals must identify the exact assertion or review fingerprint
|
Corrections and appeals must identify the exact assertion or review fingerprint
|
||||||
and supply contrary evidence. Retain the original claim and decision, append the
|
and supply contrary evidence. Retain the original claim and decision, append the
|
||||||
|
|
|
||||||
14
README.md
14
README.md
|
|
@ -83,6 +83,10 @@ own admission policy. A null destination is a meaningful abstention.
|
||||||
|
|
||||||
The [consumer contract](docs/CONSUMERS.md) covers Rust dependencies, CLI JSON,
|
The [consumer contract](docs/CONSUMERS.md) covers Rust dependencies, CLI JSON,
|
||||||
SQLite/JSONL distribution, compatibility and Argand's eventual upstream cutover.
|
SQLite/JSONL distribution, compatibility and Argand's eventual upstream cutover.
|
||||||
|
Exact entity, URL/domain, source-separated popularity and redacted Curlie category
|
||||||
|
queries support audit tools. `resolve` also reports a stable abstention reason and
|
||||||
|
decision counts. The bounded `evaluate` command replays JSONL judgments and reports
|
||||||
|
accuracy plus native p50/p95 latency for one pinned generation.
|
||||||
|
|
||||||
## Sources and trust
|
## Sources and trust
|
||||||
|
|
||||||
|
|
@ -92,7 +96,9 @@ and the Public Suffix List (MPL 2.0) remain logically separate. Read the exact
|
||||||
Curlie attribution applies to names and categories as well as descriptions.
|
Curlie attribution applies to names and categories as well as descriptions.
|
||||||
|
|
||||||
The [trust policy](docs/TRUST.md) explains enforced checks, publisher responsibilities,
|
The [trust policy](docs/TRUST.md) explains enforced checks, publisher responsibilities,
|
||||||
evidence standards, expiry and revocation. [CONTRIBUTING.md](CONTRIBUTING.md),
|
evidence standards, expiry and revocation. The [publisher runbook](docs/PUBLISHING.md)
|
||||||
|
covers authenticated reviewer decisions, candidate inspection and activation.
|
||||||
|
[CONTRIBUTING.md](CONTRIBUTING.md),
|
||||||
[GOVERNANCE.md](GOVERNANCE.md) and [SECURITY.md](SECURITY.md) cover contributions,
|
[GOVERNANCE.md](GOVERNANCE.md) and [SECURITY.md](SECURITY.md) cover contributions,
|
||||||
decisions, disputes and incidents. Pull requests cannot directly approve destinations.
|
decisions, disputes and incidents. Pull requests cannot directly approve destinations.
|
||||||
|
|
||||||
|
|
@ -109,9 +115,9 @@ documentation, Python release tests, and native Rust/Python consumer parity.
|
||||||
verification and rebuilding outside the checkout. The Forgejo workflow requires
|
verification and rebuilding outside the checkout. The Forgejo workflow requires
|
||||||
a dedicated isolated runner; it has no signing or dataset-promotion authority.
|
a dedicated isolated runner; it has no signing or dataset-promotion authority.
|
||||||
|
|
||||||
The [validation record](docs/VALIDATION.md) reports the initial independent builds
|
The [validation record](docs/VALIDATION.md) reports independent builds and native
|
||||||
and native acceptance. Hosted CI requires runner provisioning; the workflow is
|
acceptance. Hosted checks use the repository's isolated Forgejo runner label and
|
||||||
included and Actions remains disabled until an isolated runner is ready.
|
have no dataset, review, signing or activation authority.
|
||||||
|
|
||||||
The code remains **AGPL-3.0-or-later**; the complete license is in [LICENSE](LICENSE).
|
The code remains **AGPL-3.0-or-later**; the complete license is in [LICENSE](LICENSE).
|
||||||
Original attribution is retained. [UPSTREAM.json](UPSTREAM.json) records the signed
|
Original attribution is retained. [UPSTREAM.json](UPSTREAM.json) records the signed
|
||||||
|
|
|
||||||
13
ci/Dockerfile
Normal file
13
ci/Dockerfile
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
# By Nic Weyand! Pinned toolchain image for the repository-scoped isolated runner.
|
||||||
|
FROM rust:1.98.0-trixie@sha256:620dbcd124499c59e2406d3741574b5c5838cf9eb9656f0c3a03948f79b02959
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
cmake \
|
||||||
|
openssh-client \
|
||||||
|
python3 \
|
||||||
|
shellcheck \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& rustup component add clippy rustfmt
|
||||||
|
|
||||||
|
WORKDIR /workspace
|
||||||
19
ci/README.md
Normal file
19
ci/README.md
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Isolated Forgejo runner image
|
||||||
|
|
||||||
|
Build the reviewed pinned image and register a repository-scoped runner with only
|
||||||
|
the container label below:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build --pull -t argand-site-registry-ci:0.2.0 -f ci/Dockerfile .
|
||||||
|
forgejo-runner register --no-interactive \
|
||||||
|
--instance https://git.argand.org \
|
||||||
|
--token REPOSITORY_REGISTRATION_TOKEN \
|
||||||
|
--name argand-site-registry-isolated \
|
||||||
|
--labels site-registry-isolated:docker://argand-site-registry-ci:0.2.0
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the daemon with capacity one, no host label, no bind-volume allowlist, no
|
||||||
|
Docker socket inside jobs, `privileged: false`, and container limits of two CPUs,
|
||||||
|
4 GiB memory and 4 GiB memory plus swap. The workflow fetches the exact public
|
||||||
|
commit without repository credentials. This runner contains no dataset, reviewer,
|
||||||
|
release-signing or activation authority.
|
||||||
|
|
@ -15,6 +15,7 @@ chrono.workspace = true
|
||||||
clap.workspace = true
|
clap.workspace = true
|
||||||
csv = "1.4.0"
|
csv = "1.4.0"
|
||||||
flate2.workspace = true
|
flate2.workspace = true
|
||||||
|
libc.workspace = true
|
||||||
publicsuffix = "=2.3.0"
|
publicsuffix = "=2.3.0"
|
||||||
reqwest.workspace = true
|
reqwest.workspace = true
|
||||||
rusqlite = { version = "=0.40.2", features = ["bundled"] }
|
rusqlite = { version = "=0.40.2", features = ["bundled"] }
|
||||||
|
|
@ -22,13 +23,13 @@ serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
sha2.workspace = true
|
sha2.workspace = true
|
||||||
tar = "0.4.46"
|
tar = "0.4.46"
|
||||||
|
tempfile.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
toml.workspace = true
|
toml.workspace = true
|
||||||
unicode-normalization.workspace = true
|
unicode-normalization.workspace = true
|
||||||
url.workspace = true
|
url.workspace = true
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile.workspace = true
|
|
||||||
http.workspace = true
|
http.workspace = true
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
|
|
|
||||||
|
|
@ -206,11 +206,18 @@ Replace the example evidence and dates; approvals expire within 90 days. Use
|
||||||
`role: "primary"` for the independently verified default and `country: "DE"`
|
`role: "primary"` for the independently verified default and `country: "DE"`
|
||||||
for a separately verified German regional property. A country-scoped review
|
for a separately verified German regional property. A country-scoped review
|
||||||
can leave locale empty. If both are specified, both must match the request.
|
can leave locale empty. If both are specified, both must match the request.
|
||||||
|
Sign the exact decision bytes under the dedicated reviewer namespace. The reviewer
|
||||||
|
identity must match the JSON and an independently maintained OpenSSH allowed-signers
|
||||||
|
file. The release signing key may be separate from reviewer keys.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
ssh-keygen -Y sign -n argand-site-registry-review \
|
||||||
|
-f /secure/reviewer-key review.json
|
||||||
argand-site-registry review --database "$ARGAND_SITE_DATA/import.sqlite" \
|
argand-site-registry review --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||||
--generation "$ARGAND_SITE_DATA/generation-1" --pin "$ARGAND_SITE_PIN" \
|
--generation "$ARGAND_SITE_DATA/generation-1" --pin "$ARGAND_SITE_PIN" \
|
||||||
--decision review.json
|
--decision review.json --signature review.json.sig \
|
||||||
|
--allowed-reviewers /secure/reviewer-allowed-signers \
|
||||||
|
--identity operator@example.org
|
||||||
argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \
|
argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||||
--output "$ARGAND_SITE_DATA/generation-2" > "$ARGAND_SITE_DATA/build-2.json"
|
--output "$ARGAND_SITE_DATA/generation-2" > "$ARGAND_SITE_DATA/build-2.json"
|
||||||
export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-2.json")"
|
export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-2.json")"
|
||||||
|
|
@ -221,7 +228,8 @@ argand-site-registry resolve --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||||
After the corresponding real reviews, GB selects the reviewed UK property;
|
After the corresponding real reviews, GB selects the reviewed UK property;
|
||||||
DE selects the reviewed German property; otherwise an explicitly reviewed
|
DE selects the reviewed German property; otherwise an explicitly reviewed
|
||||||
primary may be used. Unknown, expired, tied or entity-ambiguous requests return
|
primary may be used. Unknown, expired, tied or entity-ambiguous requests return
|
||||||
`"destination": null`. Result limits never hide ambiguity. Name/alias changes,
|
`"destination": null` with `status` and rejection counts. Result limits never hide
|
||||||
|
ambiguity. Name/alias changes,
|
||||||
changed statements/revisions, URLs or normalization evidence invalidate reviews.
|
changed statements/revisions, URLs or normalization evidence invalidate reviews.
|
||||||
Deprecated, end-dated and non-value statements remain audit evidence and cannot
|
Deprecated, end-dated and non-value statements remain audit evidence and cannot
|
||||||
be admitted. An unchanged PSL file with a new retrieval time preserves reviews.
|
be admitted. An unchanged PSL file with a new retrieval time preserves reviews.
|
||||||
|
|
@ -237,9 +245,10 @@ argand-site-registry equivalence --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||||
```
|
```
|
||||||
|
|
||||||
Use the returned fingerprint in a review JSON with `role: "unspecified"`, empty
|
Use the returned fingerprint in a review JSON with `role: "unspecified"`, empty
|
||||||
locale/country, a reason, immutable identity evidence and an expiry. Then repeat
|
locale/country, a reason, immutable identity evidence and an expiry. Sign
|
||||||
the command with `--database "$ARGAND_SITE_DATA/import.sqlite" --decision
|
`identity-review.json` with the reviewer namespace and repeat the command with
|
||||||
identity-review.json` and rebuild. `resolve` follows only active, explicitly
|
`--database`, `--decision`, `--signature`, `--allowed-reviewers` and `--identity`,
|
||||||
|
then rebuild. `resolve` follows only active, explicitly
|
||||||
reviewed equivalences and includes their provenance. Each destination still
|
reviewed equivalences and includes their provenance. Each destination still
|
||||||
needs its own review. The original IDs, raw ambiguity counts and conflicting
|
needs its own review. The original IDs, raw ambiguity counts and conflicting
|
||||||
assertions remain visible. Changed names or website assertions invalidate the
|
assertions remain visible. Changed names or website assertions invalidate the
|
||||||
|
|
@ -258,13 +267,16 @@ source-bearing JSONL and omits descriptions by default:
|
||||||
argand-site-registry export --generation "$ARGAND_SITE_DATA/generation-2" \
|
argand-site-registry export --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||||
--pin "$ARGAND_SITE_PIN" --output "$ARGAND_SITE_DATA/assertions.jsonl"
|
--pin "$ARGAND_SITE_PIN" --output "$ARGAND_SITE_DATA/assertions.jsonl"
|
||||||
argand-site-registry sign --generation "$ARGAND_SITE_DATA/generation-2" \
|
argand-site-registry sign --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||||
--pin "$ARGAND_SITE_PIN" --key /secure/registry-signing-key
|
--pin "$ARGAND_SITE_PIN" --key /secure/registry-signing-key \
|
||||||
|
--allowed-reviewers /secure/reviewer-allowed-signers
|
||||||
argand-site-registry activate --generation "$ARGAND_SITE_DATA/generation-2" \
|
argand-site-registry activate --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||||
--current "$ARGAND_SITE_DATA/current.json" \
|
--current "$ARGAND_SITE_DATA/current.json" \
|
||||||
--allowed-signers /secure/registry-allowed-signers --identity registry-publisher
|
--allowed-signers /secure/registry-allowed-signers --identity registry-publisher
|
||||||
```
|
```
|
||||||
|
|
||||||
Use an existing operator-controlled SSH signing key. The external allowed-signers
|
Before signing, the CLI replays every stored reviewer signature against the supplied
|
||||||
|
reviewer trust file and rejects missing, forged or altered proofs. Use an existing
|
||||||
|
operator-controlled SSH release key. The external release allowed-signers
|
||||||
file follows OpenSSH syntax: `registry-publisher ssh-ed25519 PUBLIC_KEY`. Neither
|
file follows OpenSSH syntax: `registry-publisher ssh-ed25519 PUBLIC_KEY`. Neither
|
||||||
keys nor the trust file should come from the downloaded dataset. Consumers can
|
keys nor the trust file should come from the downloaded dataset. Consumers can
|
||||||
open `Registry::open(path, trusted_receipt_sha256)` once and reuse its indexed
|
open `Registry::open(path, trusted_receipt_sha256)` once and reuse its indexed
|
||||||
|
|
@ -272,8 +284,10 @@ queries, or verify a publisher with `release::verify_signed` first. A hash prove
|
||||||
integrity only relative to a trusted pin. Signature verification authenticates
|
integrity only relative to a trusted pin. Signature verification authenticates
|
||||||
the publisher, not the truth of a source assertion.
|
the publisher, not the truth of a source assertion.
|
||||||
|
|
||||||
`diff --old PATH --old-pin HASH --new PATH --new-pin HASH` streams added/removed
|
`diff --old PATH --old-pin HASH --new PATH --new-pin HASH` streams typed added,
|
||||||
edge fingerprints. `verify --generation PATH --pin HASH` checks every artifact
|
removed and changed source selections, entities, names, properties, edges,
|
||||||
|
popularity observations, reviews and equivalences. `verify --generation PATH
|
||||||
|
--pin HASH` checks every artifact
|
||||||
bound by the receipt. To revoke, append a review with `decision: "revoke"`, then
|
bound by the receipt. To revoke, append a review with `decision: "revoke"`, then
|
||||||
rebuild, sign and activate. Re-activating an older signed generation supports
|
rebuild, sign and activate. Re-activating an older signed generation supports
|
||||||
rollback **only if it retains every distributed revocation**. Otherwise rebuild
|
rollback **only if it retains every distributed revocation**. Otherwise rebuild
|
||||||
|
|
@ -299,9 +313,9 @@ downloads. No scheduled job signs, approves, renews approvals or activates links
|
||||||
|
|
||||||
## Storage and operating limits
|
## Storage and operating limits
|
||||||
|
|
||||||
Migration `migrations/001.sql` owns schema version 1. `sources`, `records` and
|
Migrations `migrations/001.sql` and `002.sql` own schema version 2. `sources`, `records` and
|
||||||
`facts` preserve snapshot/native IDs, licenses, retrieval times and confidence;
|
`facts` preserve snapshot/native IDs, licenses, retrieval times and confidence;
|
||||||
`reviews` is append-only. Complete source selection is latest retrieval time per
|
`reviews` and their cryptographic `review_auth` proofs are append-only. Complete source selection is latest retrieval time per
|
||||||
provider/scope, with digest as the deterministic tie break. Use the **same scope**
|
provider/scope, with digest as the deterministic tie break. Use the **same scope**
|
||||||
for a replacement snapshot, and separate scopes for deliberate independent
|
for a replacement snapshot, and separate scopes for deliberate independent
|
||||||
selections. History and conflicts remain stored. A failed source cannot replace
|
selections. History and conflicts remain stored. A failed source cannot replace
|
||||||
|
|
@ -317,7 +331,10 @@ join. The canonical label rule prefers labels, then English,
|
||||||
then language/text order. Original labels/aliases are kept. Popularity has its
|
then language/text order. Original labels/aliases are kept. Popularity has its
|
||||||
own source, target, observation period and audience scope and never creates an
|
own source, target, observation period and audience scope and never creates an
|
||||||
ownership edge. All derivations bind input fact IDs, PSL identity and the
|
ownership edge. All derivations bind input fact IDs, PSL identity and the
|
||||||
`argand.site-rules/v1` contract through their generation receipt.
|
`argand.site-rules/v2` contract through their generation receipt. A v1 writer store
|
||||||
|
migrates in place while retaining review history. Any legacy unauthenticated
|
||||||
|
decision makes release signing fail closed; start a reviewed v2 store from the
|
||||||
|
pinned source inputs rather than deleting historical decisions.
|
||||||
|
|
||||||
Imports use transactions of 256 relevant records with durable replay checkpoints.
|
Imports use transactions of 256 relevant records with durable replay checkpoints.
|
||||||
Restart replays the compressed stream and skips committed records. Large source
|
Restart replays the compressed stream and skips committed records. Large source
|
||||||
|
|
@ -330,9 +347,15 @@ The full store/history and each generation consume disk; there is no automatic
|
||||||
pruning. The original compressed source is hashed before/after import, so expect
|
pruning. The original compressed source is hashed before/after import, so expect
|
||||||
extra sequential disk reads. These bounds are not a full-dump throughput claim.
|
extra sequential disk reads. These bounds are not a full-dump throughput claim.
|
||||||
|
|
||||||
The `observation` module defines future crawler evidence for redirects, canonical
|
The `observation` module validates and deterministically normalizes future crawler
|
||||||
links, hreflang, JSON-LD sameAs, sitemaps and country selectors, with capture IDs,
|
evidence for redirects, canonical links, hreflang, JSON-LD sameAs, sitemaps and
|
||||||
hashes, rights and confidence. It does not crawl or automatically infer ownership.
|
country selectors, with capture IDs, hashes, rights and confidence. It does not
|
||||||
|
crawl, admit a new source or automatically infer ownership.
|
||||||
|
|
||||||
|
Use `entity`, `lookup-web`, `popularity`, `category` and `stats` for reverse/audit
|
||||||
|
views. Curlie descriptions remain redacted on the category surface. The
|
||||||
|
`evaluate` command accepts bounded JSONL judgments; see the repository
|
||||||
|
[evaluation guide](../../docs/EVALUATION.md) and [publisher runbook](../../docs/PUBLISHING.md).
|
||||||
|
|
||||||
Source vandalism, compromised publishers and a domain changing ownership cannot
|
Source vandalism, compromised publishers and a domain changing ownership cannot
|
||||||
be eliminated by hashes or popularity. Review expiration, exact evidence binding,
|
be eliminated by hashes or popularity. Review expiration, exact evidence binding,
|
||||||
|
|
|
||||||
14
crates/argand-site-registry/migrations/002.sql
Normal file
14
crates/argand-site-registry/migrations/002.sql
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
-- By Nic Weyand! Authenticate operator decisions and move new builds to rules v2.
|
||||||
|
CREATE TABLE review_auth (
|
||||||
|
sequence INTEGER PRIMARY KEY REFERENCES reviews(sequence),
|
||||||
|
signer TEXT NOT NULL,
|
||||||
|
signature_sha256 TEXT NOT NULL,
|
||||||
|
namespace TEXT NOT NULL CHECK(namespace='argand-site-registry-review'),
|
||||||
|
decision_sha256 TEXT NOT NULL,
|
||||||
|
decision_json BLOB NOT NULL,
|
||||||
|
signature BLOB NOT NULL
|
||||||
|
) STRICT;
|
||||||
|
CREATE TRIGGER review_auth_no_update BEFORE UPDATE ON review_auth BEGIN SELECT RAISE(ABORT,'review authentication is immutable'); END;
|
||||||
|
CREATE TRIGGER review_auth_no_delete BEFORE DELETE ON review_auth BEGIN SELECT RAISE(ABORT,'review authentication is immutable'); END;
|
||||||
|
UPDATE registry_metadata SET rules='argand.site-rules/v2' WHERE singleton=1;
|
||||||
|
PRAGMA user_version=2;
|
||||||
|
|
@ -148,6 +148,11 @@ fn copy_canonical(source: &Connection, destination: &Connection) -> anyhow::Resu
|
||||||
8,
|
8,
|
||||||
),
|
),
|
||||||
("reviews", "SELECT * FROM reviews ORDER BY sequence", 11),
|
("reviews", "SELECT * FROM reviews ORDER BY sequence", 11),
|
||||||
|
(
|
||||||
|
"review_auth",
|
||||||
|
"SELECT * FROM review_auth ORDER BY sequence",
|
||||||
|
7,
|
||||||
|
),
|
||||||
(
|
(
|
||||||
"equivalences",
|
"equivalences",
|
||||||
"SELECT * FROM equivalences ORDER BY fingerprint",
|
"SELECT * FROM equivalences ORDER BY fingerprint",
|
||||||
|
|
@ -342,9 +347,11 @@ fn project_edge(
|
||||||
};
|
};
|
||||||
let evidence = json!({"source":source,"native_id":native,"selector":selector,"assertion":value,"names_fingerprint":names,"normalization":store::RULE_VERSION});
|
let evidence = json!({"source":source,"native_id":native,"selector":selector,"assertion":value,"names_fingerprint":names,"normalization":store::RULE_VERSION});
|
||||||
let mut identity_property = property.clone();
|
let mut identity_property = property.clone();
|
||||||
// Bind review to actual PSL bytes and derived fields, not a fresh timestamp
|
// Bind reviews to the normalization result. The complete PSL identity remains
|
||||||
// for an otherwise identical list. Full retrieval provenance stays on property.
|
// on the property, while comment-only or unrelated rule changes do not force
|
||||||
|
// re-review when this hostname's derived fields are identical.
|
||||||
identity_property.domain.psl_source.clear();
|
identity_property.domain.psl_source.clear();
|
||||||
|
identity_property.domain.psl_sha256.clear();
|
||||||
let fingerprint = crate::digest(&serde_json::to_vec(&(
|
let fingerprint = crate::digest(&serde_json::to_vec(&(
|
||||||
entity,
|
entity,
|
||||||
&identity_property,
|
&identity_property,
|
||||||
|
|
|
||||||
427
crates/argand-site-registry/src/catalog.rs
Normal file
427
crates/argand-site-registry/src/catalog.rs
Normal file
|
|
@ -0,0 +1,427 @@
|
||||||
|
// By Nic Weyand!
|
||||||
|
//! Exact reverse registry views. These never infer entity ownership from domains.
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
evidence,
|
||||||
|
normalize::{Normalizer, WebProperty},
|
||||||
|
query::{Candidate, Registry},
|
||||||
|
};
|
||||||
|
use anyhow::{Context, ensure};
|
||||||
|
use rusqlite::{OptionalExtension, params};
|
||||||
|
use serde::Serialize;
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
/// Auditable generation size, evidence and review counts.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct RegistryStats {
|
||||||
|
/// Trusted receipt pin used to open this generation.
|
||||||
|
pub registry: String,
|
||||||
|
/// Versioned derivation and review contract.
|
||||||
|
pub rules: String,
|
||||||
|
/// SQLite logical byte size from page count and page size.
|
||||||
|
pub database_bytes: u64,
|
||||||
|
/// All retained complete source snapshots.
|
||||||
|
pub source_snapshots: u64,
|
||||||
|
/// Active source selections used for projections.
|
||||||
|
pub selected_sources: u64,
|
||||||
|
/// Retained source records.
|
||||||
|
pub records: u64,
|
||||||
|
/// Retained source facts.
|
||||||
|
pub facts: u64,
|
||||||
|
/// Operator decisions.
|
||||||
|
pub reviews: u64,
|
||||||
|
/// Decisions carrying verified reviewer authentication.
|
||||||
|
pub authenticated_reviews: u64,
|
||||||
|
/// Explicit cross-source identity proposals.
|
||||||
|
pub equivalences: u64,
|
||||||
|
/// Current approvals expiring during the next seven days.
|
||||||
|
pub approvals_expiring_within_seven_days: u64,
|
||||||
|
/// Receipt-level entity count.
|
||||||
|
pub entities: u64,
|
||||||
|
/// Receipt-level strict URL count.
|
||||||
|
pub properties: u64,
|
||||||
|
/// Receipt-level entity/property edge count.
|
||||||
|
pub edges: u64,
|
||||||
|
/// Receipt-level rejected fact count.
|
||||||
|
pub rejected: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One entity and all bounded website assertions attached to its stable ID.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct EntityView {
|
||||||
|
/// Provenance-bearing names and metadata.
|
||||||
|
pub entity: evidence::Entity,
|
||||||
|
/// Complete edge count before the output limit.
|
||||||
|
pub total_edges: u64,
|
||||||
|
/// True when candidates were omitted by the limit.
|
||||||
|
pub truncated: bool,
|
||||||
|
/// Source assertions; review status remains explicit on each candidate.
|
||||||
|
pub candidates: Vec<Candidate>,
|
||||||
|
/// Attribution required when displaying imported fields.
|
||||||
|
pub attribution: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One exact URL/hostname/registrable-domain match.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct WebMatch {
|
||||||
|
/// `url`, `hostname`, or `registrable_domain`.
|
||||||
|
pub matched_by: String,
|
||||||
|
/// Original entity/property assertion and review state.
|
||||||
|
pub candidate: Candidate,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reverse property lookup without an ownership inference.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct WebLookup {
|
||||||
|
/// Original input.
|
||||||
|
pub input: String,
|
||||||
|
/// Strict normalized URL or domain evidence.
|
||||||
|
pub normalized: Value,
|
||||||
|
/// Complete property count before the output limit.
|
||||||
|
pub total_properties: u64,
|
||||||
|
/// Complete assertion count before the output limit.
|
||||||
|
pub total_edges: u64,
|
||||||
|
/// True when assertions were omitted by the limit.
|
||||||
|
pub truncated: bool,
|
||||||
|
/// Exact assertion matches.
|
||||||
|
pub matches: Vec<WebMatch>,
|
||||||
|
/// Attribution required when displaying imported fields.
|
||||||
|
pub attribution: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One source-specific popularity observation with complete provenance.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct PopularityObservation {
|
||||||
|
/// Provider namespace; signals never combine implicitly.
|
||||||
|
pub source: String,
|
||||||
|
/// Original source target.
|
||||||
|
pub target: String,
|
||||||
|
/// Source-native rank/period/audience values.
|
||||||
|
pub value: Value,
|
||||||
|
/// PSL-derived hostname/domain evidence.
|
||||||
|
pub domain: Value,
|
||||||
|
/// Complete imported fact declaration.
|
||||||
|
pub provenance: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exact popularity lookup, kept separate from entity ownership.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct PopularityLookup {
|
||||||
|
/// Original input.
|
||||||
|
pub input: String,
|
||||||
|
/// Strict normalized target.
|
||||||
|
pub normalized: Value,
|
||||||
|
/// Complete observation count.
|
||||||
|
pub total: u64,
|
||||||
|
/// True when observations were omitted by the limit.
|
||||||
|
pub truncated: bool,
|
||||||
|
/// Source-specific observations.
|
||||||
|
pub observations: Vec<PopularityObservation>,
|
||||||
|
/// Required provider attribution.
|
||||||
|
pub attribution: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Redacted category metadata and member assertions.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct CategoryLookup {
|
||||||
|
/// Exact Curlie category ID.
|
||||||
|
pub category_id: String,
|
||||||
|
/// Category facts with descriptions removed from this display surface.
|
||||||
|
pub metadata: Vec<Value>,
|
||||||
|
/// Complete member assertion count.
|
||||||
|
pub total_members: u64,
|
||||||
|
/// True when members were omitted by the limit.
|
||||||
|
pub truncated: bool,
|
||||||
|
/// Directory assertions for category members.
|
||||||
|
pub members: Vec<Candidate>,
|
||||||
|
/// Curlie and other provider attribution.
|
||||||
|
pub attribution: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Registry {
|
||||||
|
/// Reports bounded storage and decision-health measurements.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns malformed database state or query failures.
|
||||||
|
pub fn stats(&self, now: chrono::DateTime<chrono::Utc>) -> anyhow::Result<RegistryStats> {
|
||||||
|
let count = |table: &str| -> anyhow::Result<u64> {
|
||||||
|
Ok(self
|
||||||
|
.db
|
||||||
|
.query_row(&format!("SELECT count(*) FROM {table}"), [], |row| {
|
||||||
|
crate::store::unsigned(row, 0)
|
||||||
|
})?)
|
||||||
|
};
|
||||||
|
let page_count: u64 = self.db.query_row("PRAGMA page_count", [], |row| {
|
||||||
|
crate::store::unsigned(row, 0)
|
||||||
|
})?;
|
||||||
|
let page_size: u64 = self
|
||||||
|
.db
|
||||||
|
.query_row("PRAGMA page_size", [], |row| crate::store::unsigned(row, 0))?;
|
||||||
|
let deadline = now + chrono::Duration::days(7);
|
||||||
|
let approvals_expiring_within_seven_days = self.db.query_row(
|
||||||
|
"SELECT count(*) FROM reviews r WHERE r.decision='approve' AND r.sequence=(SELECT max(sequence) FROM reviews WHERE fingerprint=r.fingerprint) AND r.expires_at>?1 AND r.expires_at<=?2",
|
||||||
|
params![now.to_rfc3339(), deadline.to_rfc3339()],
|
||||||
|
|row| crate::store::unsigned(row, 0),
|
||||||
|
)?;
|
||||||
|
Ok(RegistryStats {
|
||||||
|
registry: self.identity.clone(),
|
||||||
|
rules: self.receipt.rules.clone(),
|
||||||
|
database_bytes: page_count.saturating_mul(page_size),
|
||||||
|
source_snapshots: count("sources")?,
|
||||||
|
selected_sources: count("selected_sources")?,
|
||||||
|
records: count("records")?,
|
||||||
|
facts: count("facts")?,
|
||||||
|
reviews: count("reviews")?,
|
||||||
|
authenticated_reviews: count("review_auth")?,
|
||||||
|
equivalences: count("equivalences")?,
|
||||||
|
approvals_expiring_within_seven_days,
|
||||||
|
entities: self.receipt.entities,
|
||||||
|
properties: self.receipt.properties,
|
||||||
|
edges: self.receipt.edges,
|
||||||
|
rejected: self.receipt.rejected,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Looks up one source-derived stable entity ID.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects malformed IDs, output limits, absent entities, or corrupt data.
|
||||||
|
pub fn entity_by_id(&self, id: &str, limit: u32) -> anyhow::Result<Option<EntityView>> {
|
||||||
|
bound_limit(limit)?;
|
||||||
|
ensure!(
|
||||||
|
id.starts_with("argand:entity:") && id.len() <= 256,
|
||||||
|
"invalid entity ID"
|
||||||
|
);
|
||||||
|
let name: Option<String> = self
|
||||||
|
.db
|
||||||
|
.query_row(
|
||||||
|
"SELECT canonical_name FROM entities WHERE id=?1",
|
||||||
|
[id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
let Some(name) = name else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let total_edges =
|
||||||
|
self.db
|
||||||
|
.query_row("SELECT count(*) FROM edges WHERE entity=?1", [id], |row| {
|
||||||
|
crate::store::unsigned(row, 0)
|
||||||
|
})?;
|
||||||
|
let candidates = self.candidates_for(
|
||||||
|
"SELECT fingerprint FROM edges WHERE entity=?1 ORDER BY property,fingerprint LIMIT ?2",
|
||||||
|
params![id, limit],
|
||||||
|
)?;
|
||||||
|
Ok(Some(EntityView {
|
||||||
|
entity: evidence::entity(&self.db, id, &name)?,
|
||||||
|
total_edges,
|
||||||
|
truncated: total_edges > u64::from(limit),
|
||||||
|
candidates,
|
||||||
|
attribution: crate::release::attribution(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Finds source assertions for an exact URL, hostname, or registrable domain.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects malformed targets/limits and corrupt registry evidence.
|
||||||
|
pub fn lookup_web(&self, input: &str, limit: u32) -> anyhow::Result<WebLookup> {
|
||||||
|
bound_limit(limit)?;
|
||||||
|
let normalizer = self.normalizer()?;
|
||||||
|
let (target, properties, total_properties, total_edges) = if input.contains("://") {
|
||||||
|
let property = normalizer.url(input)?;
|
||||||
|
let properties = self.property_edges("p.url=?1", params![&property.url], limit)?;
|
||||||
|
let counts = self.property_counts("url=?1", params![&property.url])?;
|
||||||
|
(
|
||||||
|
serde_json::to_value(property)?,
|
||||||
|
properties,
|
||||||
|
counts.0,
|
||||||
|
counts.1,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
let domain = normalizer.domain(input)?;
|
||||||
|
let properties = self.property_edges(
|
||||||
|
"(p.hostname=?1 OR p.domain=?2)",
|
||||||
|
params![&domain.hostname, &domain.registrable_domain],
|
||||||
|
limit,
|
||||||
|
)?;
|
||||||
|
let counts = self.property_counts(
|
||||||
|
"hostname=?1 OR domain=?2",
|
||||||
|
params![&domain.hostname, &domain.registrable_domain],
|
||||||
|
)?;
|
||||||
|
(
|
||||||
|
serde_json::to_value(domain)?,
|
||||||
|
properties,
|
||||||
|
counts.0,
|
||||||
|
counts.1,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let matches = properties
|
||||||
|
.into_iter()
|
||||||
|
.map(|candidate| {
|
||||||
|
let property: WebProperty = serde_json::from_value(candidate.web_property.clone())?;
|
||||||
|
let matched_by = if input.contains("://") {
|
||||||
|
"url"
|
||||||
|
} else if target["hostname"] == property.domain.hostname {
|
||||||
|
"hostname"
|
||||||
|
} else {
|
||||||
|
"registrable_domain"
|
||||||
|
};
|
||||||
|
Ok(WebMatch {
|
||||||
|
matched_by: matched_by.into(),
|
||||||
|
candidate,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||||
|
Ok(WebLookup {
|
||||||
|
input: input.into(),
|
||||||
|
normalized: target,
|
||||||
|
total_properties,
|
||||||
|
total_edges,
|
||||||
|
truncated: total_edges > u64::from(limit),
|
||||||
|
matches,
|
||||||
|
attribution: crate::release::attribution(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns exact source-specific popularity observations for a web target.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects malformed targets/limits and corrupt registry evidence.
|
||||||
|
pub fn popularity(&self, input: &str, limit: u32) -> anyhow::Result<PopularityLookup> {
|
||||||
|
bound_limit(limit)?;
|
||||||
|
let normalizer = self.normalizer()?;
|
||||||
|
let domain = if input.contains("://") {
|
||||||
|
normalizer.url(input)?.domain
|
||||||
|
} else {
|
||||||
|
normalizer.domain(input)?
|
||||||
|
};
|
||||||
|
let total = self.db.query_row(
|
||||||
|
"SELECT count(*) FROM popularity WHERE hostname=?1 OR domain=?2",
|
||||||
|
params![&domain.hostname, &domain.registrable_domain],
|
||||||
|
|row| crate::store::unsigned(row, 0),
|
||||||
|
)?;
|
||||||
|
let mut statement = self.db.prepare("SELECT fact,source,target,value,derived_json FROM popularity WHERE hostname=?1 OR domain=?2 ORDER BY source,fact LIMIT ?3")?;
|
||||||
|
let rows = statement.query_map(
|
||||||
|
params![&domain.hostname, &domain.registrable_domain, limit],
|
||||||
|
|row| {
|
||||||
|
Ok((
|
||||||
|
row.get::<_, String>(0)?,
|
||||||
|
row.get::<_, String>(1)?,
|
||||||
|
row.get::<_, String>(2)?,
|
||||||
|
row.get::<_, String>(3)?,
|
||||||
|
row.get::<_, String>(4)?,
|
||||||
|
))
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
let mut observations = Vec::new();
|
||||||
|
for row in rows {
|
||||||
|
let (fact, source, target, value, derived) = row?;
|
||||||
|
observations.push(PopularityObservation {
|
||||||
|
source,
|
||||||
|
target,
|
||||||
|
value: serde_json::from_str(&value)?,
|
||||||
|
domain: serde_json::from_str(&derived)?,
|
||||||
|
provenance: evidence::fact(&self.db, &fact)?,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(PopularityLookup {
|
||||||
|
input: input.into(),
|
||||||
|
normalized: serde_json::to_value(domain)?,
|
||||||
|
total,
|
||||||
|
truncated: total > u64::from(limit),
|
||||||
|
observations,
|
||||||
|
attribution: crate::release::attribution(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns one exact Curlie category without exposing copied descriptions.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects malformed category IDs/limits and corrupt registry evidence.
|
||||||
|
pub fn category(&self, category_id: &str, limit: u32) -> anyhow::Result<CategoryLookup> {
|
||||||
|
bound_limit(limit)?;
|
||||||
|
ensure!(
|
||||||
|
!category_id.is_empty()
|
||||||
|
&& category_id.len() <= 32
|
||||||
|
&& category_id.bytes().all(|byte| byte.is_ascii_digit()),
|
||||||
|
"invalid category ID"
|
||||||
|
);
|
||||||
|
let mut statement = self.db.prepare("SELECT f.id FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category' AND json_extract(f.value,'$.category_id')=?1 ORDER BY f.id")?;
|
||||||
|
let mut metadata = Vec::new();
|
||||||
|
for id in statement.query_map([category_id], |row| row.get::<_, String>(0))? {
|
||||||
|
let mut fact = evidence::fact(&self.db, &id?)?;
|
||||||
|
if let Some(value) = fact["value"].as_object_mut() {
|
||||||
|
value.remove("description");
|
||||||
|
value.insert("description_redacted".into(), json!(true));
|
||||||
|
}
|
||||||
|
metadata.push(fact);
|
||||||
|
}
|
||||||
|
let total_members = self.db.query_row("SELECT count(*) FROM edges e WHERE e.entity IN(SELECT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category_membership' AND json_extract(f.value,'$.category_id')=?1)",[category_id],|row|crate::store::unsigned(row,0))?;
|
||||||
|
let members = self.candidates_for("SELECT e.fingerprint FROM edges e WHERE e.entity IN(SELECT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='category_membership' AND json_extract(f.value,'$.category_id')=?1) ORDER BY e.entity,e.fingerprint LIMIT ?2",params![category_id,limit])?;
|
||||||
|
Ok(CategoryLookup {
|
||||||
|
category_id: category_id.into(),
|
||||||
|
metadata,
|
||||||
|
total_members,
|
||||||
|
truncated: total_members > u64::from(limit),
|
||||||
|
members,
|
||||||
|
attribution: crate::release::attribution(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalizer(&self) -> anyhow::Result<Normalizer> {
|
||||||
|
let (source, raw): (String, String) = self.db.query_row("SELECT f.source_id,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl' ORDER BY f.source_id LIMIT 1",[],|row|Ok((row.get(0)?,row.get(1)?))).context("generation has no PSL")?;
|
||||||
|
let text: String = serde_json::from_str(&raw)?;
|
||||||
|
Normalizer::new(text.as_bytes(), source)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn candidates_for<P: rusqlite::Params>(
|
||||||
|
&self,
|
||||||
|
sql: &str,
|
||||||
|
params: P,
|
||||||
|
) -> anyhow::Result<Vec<Candidate>> {
|
||||||
|
let mut statement = self.db.prepare(sql)?;
|
||||||
|
let fingerprints = statement
|
||||||
|
.query_map(params, |row| row.get::<_, String>(0))?
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
fingerprints
|
||||||
|
.iter()
|
||||||
|
.map(|fingerprint| self.candidate(fingerprint))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn property_edges<P: rusqlite::Params>(
|
||||||
|
&self,
|
||||||
|
predicate: &str,
|
||||||
|
params: P,
|
||||||
|
limit: u32,
|
||||||
|
) -> anyhow::Result<Vec<Candidate>> {
|
||||||
|
self.candidates_for(
|
||||||
|
&format!("SELECT e.fingerprint FROM edges e JOIN properties p ON p.id=e.property WHERE {predicate} ORDER BY p.id,e.entity,e.fingerprint LIMIT {limit}"),
|
||||||
|
params,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn property_counts<P: rusqlite::Params + Clone>(
|
||||||
|
&self,
|
||||||
|
predicate: &str,
|
||||||
|
params: P,
|
||||||
|
) -> anyhow::Result<(u64, u64)> {
|
||||||
|
let properties = self.db.query_row(
|
||||||
|
&format!("SELECT count(*) FROM properties WHERE {predicate}"),
|
||||||
|
params.clone(),
|
||||||
|
|row| crate::store::unsigned(row, 0),
|
||||||
|
)?;
|
||||||
|
let edges = self.db.query_row(
|
||||||
|
&format!("SELECT count(*) FROM edges e JOIN properties p ON p.id=e.property WHERE {predicate}"),
|
||||||
|
params,
|
||||||
|
|row| crate::store::unsigned(row, 0),
|
||||||
|
)?;
|
||||||
|
Ok((properties, edges))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bound_limit(limit: u32) -> anyhow::Result<()> {
|
||||||
|
ensure!((1..=100).contains(&limit), "lookup limit must be 1..100");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
// By Nic Weyand!
|
// By Nic Weyand!
|
||||||
//! Explicit source acquisition, import, review, and immutable generation commands.
|
//! Explicit source acquisition, import, review, and immutable generation commands.
|
||||||
|
|
||||||
use anyhow::ensure;
|
use anyhow::{Context, ensure};
|
||||||
use argand_site_registry as registry;
|
use argand_site_registry as registry;
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use registry::{
|
use registry::{
|
||||||
|
|
@ -36,6 +36,12 @@ enum Command {
|
||||||
decision: Option<PathBuf>,
|
decision: Option<PathBuf>,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
database: Option<PathBuf>,
|
database: Option<PathBuf>,
|
||||||
|
#[arg(long, requires = "decision")]
|
||||||
|
signature: Option<PathBuf>,
|
||||||
|
#[arg(long, requires = "decision")]
|
||||||
|
allowed_reviewers: Option<PathBuf>,
|
||||||
|
#[arg(long, requires = "decision")]
|
||||||
|
identity: Option<String>,
|
||||||
},
|
},
|
||||||
/// Download one allowlisted source into an immutable local cache.
|
/// Download one allowlisted source into an immutable local cache.
|
||||||
Download {
|
Download {
|
||||||
|
|
@ -111,6 +117,70 @@ enum Command {
|
||||||
#[arg(long, default_value_t = 20)]
|
#[arg(long, default_value_t = 20)]
|
||||||
limit: u32,
|
limit: u32,
|
||||||
},
|
},
|
||||||
|
/// Inspect one stable entity ID and its bounded website assertions.
|
||||||
|
Entity {
|
||||||
|
#[arg(long)]
|
||||||
|
generation: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
pin: String,
|
||||||
|
#[arg(long)]
|
||||||
|
id: String,
|
||||||
|
#[arg(long, default_value_t = 20)]
|
||||||
|
limit: u32,
|
||||||
|
},
|
||||||
|
/// Reverse lookup an exact URL, hostname, or registrable domain.
|
||||||
|
LookupWeb {
|
||||||
|
#[arg(long)]
|
||||||
|
generation: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
pin: String,
|
||||||
|
#[arg(long)]
|
||||||
|
target: String,
|
||||||
|
#[arg(long, default_value_t = 20)]
|
||||||
|
limit: u32,
|
||||||
|
},
|
||||||
|
/// Inspect source-separated popularity observations for a web target.
|
||||||
|
Popularity {
|
||||||
|
#[arg(long)]
|
||||||
|
generation: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
pin: String,
|
||||||
|
#[arg(long)]
|
||||||
|
target: String,
|
||||||
|
#[arg(long, default_value_t = 20)]
|
||||||
|
limit: u32,
|
||||||
|
},
|
||||||
|
/// Inspect a Curlie category with copied descriptions redacted.
|
||||||
|
Category {
|
||||||
|
#[arg(long)]
|
||||||
|
generation: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
pin: String,
|
||||||
|
#[arg(long)]
|
||||||
|
id: String,
|
||||||
|
#[arg(long, default_value_t = 20)]
|
||||||
|
limit: u32,
|
||||||
|
},
|
||||||
|
/// Report generation size, evidence counts and upcoming approval expiry.
|
||||||
|
Stats {
|
||||||
|
#[arg(long)]
|
||||||
|
generation: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
pin: String,
|
||||||
|
},
|
||||||
|
/// Replay bounded JSONL judgments and report accuracy and native latency.
|
||||||
|
Evaluate {
|
||||||
|
#[arg(long)]
|
||||||
|
generation: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
pin: String,
|
||||||
|
#[arg(long)]
|
||||||
|
cases: PathBuf,
|
||||||
|
#[arg(long, default_value_t = 10_000)]
|
||||||
|
maximum_cases: u32,
|
||||||
|
#[arg(long)]
|
||||||
|
at: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
|
},
|
||||||
/// Resolve only an unambiguous, explicitly reviewed, unexpired property.
|
/// Resolve only an unambiguous, explicitly reviewed, unexpired property.
|
||||||
Resolve {
|
Resolve {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
|
|
@ -134,6 +204,12 @@ enum Command {
|
||||||
pin: String,
|
pin: String,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
decision: PathBuf,
|
decision: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
signature: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
allowed_reviewers: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
identity: String,
|
||||||
},
|
},
|
||||||
/// Export facts as streaming JSONL with source licenses and attribution.
|
/// Export facts as streaming JSONL with source licenses and attribution.
|
||||||
Export {
|
Export {
|
||||||
|
|
@ -172,6 +248,8 @@ enum Command {
|
||||||
pin: String,
|
pin: String,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
key: PathBuf,
|
key: PathBuf,
|
||||||
|
#[arg(long)]
|
||||||
|
allowed_reviewers: PathBuf,
|
||||||
},
|
},
|
||||||
/// Verify publisher signature and atomically activate (also supports rollback).
|
/// Verify publisher signature and atomically activate (also supports rollback).
|
||||||
Activate {
|
Activate {
|
||||||
|
|
@ -197,6 +275,14 @@ pub(super) async fn run() -> anyhow::Result<()> {
|
||||||
| Command::CruxDownload { .. }
|
| Command::CruxDownload { .. }
|
||||||
| Command::Manifest { .. }
|
| Command::Manifest { .. }
|
||||||
| Command::Equivalence { .. }) => acquire(command).await?,
|
| Command::Equivalence { .. }) => acquire(command).await?,
|
||||||
|
command @ (Command::Lookup { .. }
|
||||||
|
| Command::Entity { .. }
|
||||||
|
| Command::LookupWeb { .. }
|
||||||
|
| Command::Popularity { .. }
|
||||||
|
| Command::Category { .. }
|
||||||
|
| Command::Stats { .. }
|
||||||
|
| Command::Evaluate { .. }
|
||||||
|
| Command::Resolve { .. }) => inspect(command)?,
|
||||||
Command::Import {
|
Command::Import {
|
||||||
database,
|
database,
|
||||||
input,
|
input,
|
||||||
|
|
@ -209,28 +295,22 @@ pub(super) async fn run() -> anyhow::Result<()> {
|
||||||
let receipt = registry::build::build(®istry::store::open(&database)?, &output)?;
|
let receipt = registry::build::build(®istry::store::open(&database)?, &output)?;
|
||||||
serde_json::json!({"receipt":receipt,"pin":registry::file_digest(&output.join("COMPLETE.json"))?,"generation":output})
|
serde_json::json!({"receipt":receipt,"pin":registry::file_digest(&output.join("COMPLETE.json"))?,"generation":output})
|
||||||
}
|
}
|
||||||
Command::Lookup {
|
|
||||||
generation,
|
|
||||||
pin,
|
|
||||||
query,
|
|
||||||
limit,
|
|
||||||
} => serde_json::to_value(Registry::open(&generation, &pin)?.lookup(&query, limit)?)?,
|
|
||||||
Command::Resolve {
|
|
||||||
generation,
|
|
||||||
pin,
|
|
||||||
query,
|
|
||||||
locale,
|
|
||||||
country,
|
|
||||||
} => {
|
|
||||||
serde_json::json!({"destination":Registry::open(&generation,&pin)?.resolve(&query,locale.as_deref(),country.as_deref(),chrono::Utc::now())?,"attribution":registry::release::attribution()})
|
|
||||||
}
|
|
||||||
Command::Review {
|
Command::Review {
|
||||||
database,
|
database,
|
||||||
generation,
|
generation,
|
||||||
pin,
|
pin,
|
||||||
decision,
|
decision,
|
||||||
|
signature,
|
||||||
|
allowed_reviewers,
|
||||||
|
identity,
|
||||||
} => {
|
} => {
|
||||||
serde_json::json!({"review_sequence":registry::review::record(®istry::store::open(&database)?,&Registry::open(&generation,&pin)?,®istry::read_json(&decision)?)?,"rebuild_required":true})
|
let (review, authentication) = registry::review::authenticate(
|
||||||
|
&decision,
|
||||||
|
&signature,
|
||||||
|
&allowed_reviewers,
|
||||||
|
&identity,
|
||||||
|
)?;
|
||||||
|
serde_json::json!({"review_sequence":registry::review::record_authenticated(®istry::store::open(&database)?,&Registry::open(&generation,&pin)?,&review,&authentication)?,"authenticated_reviewer":identity,"rebuild_required":true})
|
||||||
}
|
}
|
||||||
Command::Export {
|
Command::Export {
|
||||||
generation,
|
generation,
|
||||||
|
|
@ -265,8 +345,9 @@ pub(super) async fn run() -> anyhow::Result<()> {
|
||||||
generation,
|
generation,
|
||||||
pin,
|
pin,
|
||||||
key,
|
key,
|
||||||
|
allowed_reviewers,
|
||||||
} => {
|
} => {
|
||||||
registry::release::sign(&generation, &key, &pin)?;
|
registry::release::sign(&generation, &key, &pin, &allowed_reviewers)?;
|
||||||
serde_json::json!({"signed":generation})
|
serde_json::json!({"signed":generation})
|
||||||
}
|
}
|
||||||
Command::Activate {
|
Command::Activate {
|
||||||
|
|
@ -283,14 +364,81 @@ pub(super) async fn run() -> anyhow::Result<()> {
|
||||||
serde_json::json!({"candidate":registry::update::run(&config).await?})
|
serde_json::json!({"candidate":registry::update::run(&config).await?})
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
write_json(&value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_json(value: &serde_json::Value) -> anyhow::Result<()> {
|
||||||
writeln!(
|
writeln!(
|
||||||
std::io::stdout().lock(),
|
std::io::stdout().lock(),
|
||||||
"{}",
|
"{}",
|
||||||
serde_json::to_string_pretty(&value)?
|
serde_json::to_string_pretty(value)?
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn inspect(command: Command) -> anyhow::Result<serde_json::Value> {
|
||||||
|
Ok(match command {
|
||||||
|
Command::Lookup {
|
||||||
|
generation,
|
||||||
|
pin,
|
||||||
|
query,
|
||||||
|
limit,
|
||||||
|
} => serde_json::to_value(Registry::open(&generation, &pin)?.lookup(&query, limit)?)?,
|
||||||
|
Command::Entity {
|
||||||
|
generation,
|
||||||
|
pin,
|
||||||
|
id,
|
||||||
|
limit,
|
||||||
|
} => serde_json::to_value(Registry::open(&generation, &pin)?.entity_by_id(&id, limit)?)?,
|
||||||
|
Command::LookupWeb {
|
||||||
|
generation,
|
||||||
|
pin,
|
||||||
|
target,
|
||||||
|
limit,
|
||||||
|
} => serde_json::to_value(Registry::open(&generation, &pin)?.lookup_web(&target, limit)?)?,
|
||||||
|
Command::Popularity {
|
||||||
|
generation,
|
||||||
|
pin,
|
||||||
|
target,
|
||||||
|
limit,
|
||||||
|
} => serde_json::to_value(Registry::open(&generation, &pin)?.popularity(&target, limit)?)?,
|
||||||
|
Command::Category {
|
||||||
|
generation,
|
||||||
|
pin,
|
||||||
|
id,
|
||||||
|
limit,
|
||||||
|
} => serde_json::to_value(Registry::open(&generation, &pin)?.category(&id, limit)?)?,
|
||||||
|
Command::Stats { generation, pin } => {
|
||||||
|
serde_json::to_value(Registry::open(&generation, &pin)?.stats(chrono::Utc::now())?)?
|
||||||
|
}
|
||||||
|
Command::Evaluate {
|
||||||
|
generation,
|
||||||
|
pin,
|
||||||
|
cases,
|
||||||
|
maximum_cases,
|
||||||
|
at,
|
||||||
|
} => serde_json::to_value(registry::evaluation::run_file(
|
||||||
|
&Registry::open(&generation, &pin)?,
|
||||||
|
&cases,
|
||||||
|
maximum_cases,
|
||||||
|
at.unwrap_or_else(chrono::Utc::now),
|
||||||
|
)?)?,
|
||||||
|
Command::Resolve {
|
||||||
|
generation,
|
||||||
|
pin,
|
||||||
|
query,
|
||||||
|
locale,
|
||||||
|
country,
|
||||||
|
} => serde_json::to_value(Registry::open(&generation, &pin)?.resolve_explained(
|
||||||
|
&query,
|
||||||
|
locale.as_deref(),
|
||||||
|
country.as_deref(),
|
||||||
|
chrono::Utc::now(),
|
||||||
|
)?)?,
|
||||||
|
_ => anyhow::bail!("expected a registry inspection command"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn acquire(command: Command) -> anyhow::Result<serde_json::Value> {
|
async fn acquire(command: Command) -> anyhow::Result<serde_json::Value> {
|
||||||
Ok(match command {
|
Ok(match command {
|
||||||
Command::Equivalence {
|
Command::Equivalence {
|
||||||
|
|
@ -300,13 +448,22 @@ async fn acquire(command: Command) -> anyhow::Result<serde_json::Value> {
|
||||||
right,
|
right,
|
||||||
decision,
|
decision,
|
||||||
database,
|
database,
|
||||||
|
signature,
|
||||||
|
allowed_reviewers,
|
||||||
|
identity,
|
||||||
} => {
|
} => {
|
||||||
let registry = Registry::open(&generation, &pin)?;
|
let registry = Registry::open(&generation, &pin)?;
|
||||||
let pair = registry::identity::propose(®istry, &left, &right)?;
|
let pair = registry::identity::propose(®istry, &left, &right)?;
|
||||||
if let Some(decision) = decision {
|
if let Some(decision) = decision {
|
||||||
let database = database
|
let database = database
|
||||||
.ok_or_else(|| anyhow::anyhow!("identity review needs a writer database"))?;
|
.ok_or_else(|| anyhow::anyhow!("identity review needs a writer database"))?;
|
||||||
serde_json::json!({"review_sequence":registry::identity::record(®istry::store::open(&database)?,®istry,&pair,®istry::read_json(&decision)?)?,"rebuild_required":true})
|
let signature = signature.context("identity review needs --signature")?;
|
||||||
|
let allowed =
|
||||||
|
allowed_reviewers.context("identity review needs --allowed-reviewers")?;
|
||||||
|
let identity = identity.context("identity review needs --identity")?;
|
||||||
|
let (review, authentication) =
|
||||||
|
registry::review::authenticate(&decision, &signature, &allowed, &identity)?;
|
||||||
|
serde_json::json!({"review_sequence":registry::identity::record_authenticated(®istry::store::open(&database)?,®istry,&pair,&review,&authentication)?,"authenticated_reviewer":identity,"rebuild_required":true})
|
||||||
} else {
|
} else {
|
||||||
serde_json::to_value(pair)?
|
serde_json::to_value(pair)?
|
||||||
}
|
}
|
||||||
|
|
|
||||||
174
crates/argand-site-registry/src/diff.rs
Normal file
174
crates/argand-site-registry/src/diff.rs
Normal file
|
|
@ -0,0 +1,174 @@
|
||||||
|
// By Nic Weyand!
|
||||||
|
//! Streaming, typed comparison of evidence, decisions and derived registry state.
|
||||||
|
|
||||||
|
use crate::query::Registry;
|
||||||
|
use rusqlite::{Connection, OptionalExtension};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
|
struct Table {
|
||||||
|
subject: &'static str,
|
||||||
|
scan: &'static str,
|
||||||
|
find: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
const TABLES: &[Table] = &[
|
||||||
|
Table {
|
||||||
|
subject: "source_selection",
|
||||||
|
scan: "SELECT s.id,json_object('source_id',s.id,'manifest',json(s.manifest)) FROM sources s JOIN selected_sources a USING(id) ORDER BY s.id",
|
||||||
|
find: "SELECT json_object('source_id',s.id,'manifest',json(s.manifest)) FROM sources s JOIN selected_sources a USING(id) WHERE s.id=?1",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "entity",
|
||||||
|
scan: "SELECT id,json_object('id',id,'canonical_name',canonical_name,'names_fingerprint',names_fingerprint) FROM entities ORDER BY id",
|
||||||
|
find: "SELECT json_object('id',id,'canonical_name',canonical_name,'names_fingerprint',names_fingerprint) FROM entities WHERE id=?1",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "name",
|
||||||
|
scan: "SELECT fact,json_object('entity',entity,'fact',fact,'key',key,'text',text,'language',language,'kind',kind) FROM names ORDER BY fact",
|
||||||
|
find: "SELECT json_object('entity',entity,'fact',fact,'key',key,'text',text,'language',language,'kind',kind) FROM names WHERE fact=?1",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "fact",
|
||||||
|
scan: "SELECT f.id,json_object('id',f.id,'source_id',f.source_id,'subject',f.subject,'predicate',f.predicate,'value',json(f.value),'selector',f.selector,'confidence',f.confidence) FROM facts f JOIN selected_sources s ON s.id=f.source_id ORDER BY f.id",
|
||||||
|
find: "SELECT json_object('id',f.id,'source_id',f.source_id,'subject',f.subject,'predicate',f.predicate,'value',json(f.value),'selector',f.selector,'confidence',f.confidence) FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.id=?1",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "property",
|
||||||
|
scan: "SELECT id,json_object('id',id,'url',url,'hostname',hostname,'domain',domain,'suffix',suffix,'derived',json(derived_json)) FROM properties ORDER BY id",
|
||||||
|
find: "SELECT json_object('id',id,'url',url,'hostname',hostname,'domain',domain,'suffix',suffix,'derived',json(derived_json)) FROM properties WHERE id=?1",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "edge",
|
||||||
|
scan: "SELECT fingerprint,json_object('fingerprint',fingerprint,'entity',entity,'property',property,'relation',relation,'facts',json(facts),'evidence',json(evidence),'eligible',json(eligible)) FROM edges ORDER BY fingerprint",
|
||||||
|
find: "SELECT json_object('fingerprint',fingerprint,'entity',entity,'property',property,'relation',relation,'facts',json(facts),'evidence',json(evidence),'eligible',json(eligible)) FROM edges WHERE fingerprint=?1",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "popularity",
|
||||||
|
scan: "SELECT fact,json_object('fact',fact,'source',source,'target',target,'hostname',hostname,'domain',domain,'value',json(value),'derived',json(derived_json)) FROM popularity ORDER BY fact",
|
||||||
|
find: "SELECT json_object('fact',fact,'source',source,'target',target,'hostname',hostname,'domain',domain,'value',json(value),'derived',json(derived_json)) FROM popularity WHERE fact=?1",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "rejected_fact",
|
||||||
|
scan: "SELECT fact,json_object('fact',fact,'reason',reason) FROM rejected ORDER BY fact",
|
||||||
|
find: "SELECT json_object('fact',fact,'reason',reason) FROM rejected WHERE fact=?1",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "review",
|
||||||
|
scan: "SELECT CAST(r.sequence AS TEXT),json_object('sequence',r.sequence,'fingerprint',r.fingerprint,'decision',r.decision,'reviewer',r.reviewer,'reason',r.reason,'evidence',r.evidence,'reviewed_at',r.reviewed_at,'expires_at',r.expires_at,'role',r.role,'locale',r.locale,'country',r.country,'authentication',CASE WHEN a.sequence IS NULL THEN NULL ELSE json_object('identity',a.signer,'signature_sha256',a.signature_sha256,'decision_sha256',a.decision_sha256,'namespace',a.namespace) END) FROM reviews r LEFT JOIN review_auth a USING(sequence) ORDER BY r.sequence",
|
||||||
|
find: "SELECT json_object('sequence',r.sequence,'fingerprint',r.fingerprint,'decision',r.decision,'reviewer',r.reviewer,'reason',r.reason,'evidence',r.evidence,'reviewed_at',r.reviewed_at,'expires_at',r.expires_at,'role',r.role,'locale',r.locale,'country',r.country,'authentication',CASE WHEN a.sequence IS NULL THEN NULL ELSE json_object('identity',a.signer,'signature_sha256',a.signature_sha256,'decision_sha256',a.decision_sha256,'namespace',a.namespace) END) FROM reviews r LEFT JOIN review_auth a USING(sequence) WHERE r.sequence=CAST(?1 AS INTEGER)",
|
||||||
|
},
|
||||||
|
Table {
|
||||||
|
subject: "equivalence",
|
||||||
|
scan: "SELECT fingerprint,json_object('fingerprint',fingerprint,'left_entity',left_entity,'right_entity',right_entity,'left_signature',left_signature,'right_signature',right_signature) FROM equivalences ORDER BY fingerprint",
|
||||||
|
find: "SELECT json_object('fingerprint',fingerprint,'left_entity',left_entity,'right_entity',right_entity,'left_signature',left_signature,'right_signature',right_signature) FROM equivalences WHERE fingerprint=?1",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Streams every material change between two authenticated generations.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns database, JSON, or output failures.
|
||||||
|
pub fn write(old: &Registry, new: &Registry, output: &mut dyn Write) -> anyhow::Result<()> {
|
||||||
|
writeln!(
|
||||||
|
output,
|
||||||
|
"{}",
|
||||||
|
json!({"schema":"argand.site-diff/v2","type":"header","old":old.identity,"new":new.identity})
|
||||||
|
)?;
|
||||||
|
let mut changes = 0_u64;
|
||||||
|
for table in TABLES {
|
||||||
|
changes += removed_or_changed(table, &old.db, &new.db, output)?;
|
||||||
|
changes += added(table, &new.db, &old.db, output)?;
|
||||||
|
}
|
||||||
|
writeln!(
|
||||||
|
output,
|
||||||
|
"{}",
|
||||||
|
json!({"schema":"argand.site-diff/v2","type":"summary","changes":changes})
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn removed_or_changed(
|
||||||
|
table: &Table,
|
||||||
|
from: &Connection,
|
||||||
|
to: &Connection,
|
||||||
|
output: &mut dyn Write,
|
||||||
|
) -> anyhow::Result<u64> {
|
||||||
|
let mut count = 0;
|
||||||
|
let mut scan = from.prepare(table.scan)?;
|
||||||
|
let mut find = to.prepare(table.find)?;
|
||||||
|
let rows = scan.query_map([], |row| {
|
||||||
|
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||||
|
})?;
|
||||||
|
for row in rows {
|
||||||
|
let (key, before) = row?;
|
||||||
|
let after: Option<String> = find.query_row([&key], |row| row.get(0)).optional()?;
|
||||||
|
match after {
|
||||||
|
None => {
|
||||||
|
event(output, table.subject, "removed", &key, Some(&before), None)?;
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
Some(after) if after != before => {
|
||||||
|
event(
|
||||||
|
output,
|
||||||
|
table.subject,
|
||||||
|
"changed",
|
||||||
|
&key,
|
||||||
|
Some(&before),
|
||||||
|
Some(&after),
|
||||||
|
)?;
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
Some(_) => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(count)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn added(
|
||||||
|
table: &Table,
|
||||||
|
from: &Connection,
|
||||||
|
to: &Connection,
|
||||||
|
output: &mut dyn Write,
|
||||||
|
) -> anyhow::Result<u64> {
|
||||||
|
let mut count = 0;
|
||||||
|
let mut scan = from.prepare(table.scan)?;
|
||||||
|
let mut find = to.prepare(table.find)?;
|
||||||
|
let rows = scan.query_map([], |row| {
|
||||||
|
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||||
|
})?;
|
||||||
|
for row in rows {
|
||||||
|
let (key, after) = row?;
|
||||||
|
let exists: bool = find
|
||||||
|
.query_row([&key], |_| Ok(true))
|
||||||
|
.optional()?
|
||||||
|
.unwrap_or(false);
|
||||||
|
if !exists {
|
||||||
|
event(output, table.subject, "added", &key, None, Some(&after))?;
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(count)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn event(
|
||||||
|
output: &mut dyn Write,
|
||||||
|
subject: &str,
|
||||||
|
change: &str,
|
||||||
|
key: &str,
|
||||||
|
before: Option<&str>,
|
||||||
|
after: Option<&str>,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let parse = |value: Option<&str>| -> anyhow::Result<Option<Value>> {
|
||||||
|
value
|
||||||
|
.map(serde_json::from_str)
|
||||||
|
.transpose()
|
||||||
|
.map_err(Into::into)
|
||||||
|
};
|
||||||
|
writeln!(
|
||||||
|
output,
|
||||||
|
"{}",
|
||||||
|
json!({"schema":"argand.site-diff/v2","type":"change","subject":subject,"change":change,"key":key,"before":parse(before)?,"after":parse(after)?})
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
224
crates/argand-site-registry/src/evaluation.rs
Normal file
224
crates/argand-site-registry/src/evaluation.rs
Normal file
|
|
@ -0,0 +1,224 @@
|
||||||
|
// By Nic Weyand!
|
||||||
|
//! Bounded, replayable destination-resolution evaluation.
|
||||||
|
|
||||||
|
use crate::{ResolutionStatus, query::Registry};
|
||||||
|
use anyhow::{Context, ensure};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::{
|
||||||
|
io::{BufRead, BufReader},
|
||||||
|
path::Path,
|
||||||
|
time::Instant,
|
||||||
|
};
|
||||||
|
|
||||||
|
const MAXIMUM_INPUT_BYTES: usize = 16 * 1024 * 1024;
|
||||||
|
const MAXIMUM_LINE_BYTES: usize = 64 * 1024;
|
||||||
|
|
||||||
|
/// One authored judgment against the native resolver.
|
||||||
|
#[derive(Debug, Deserialize, Serialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct Case {
|
||||||
|
/// Stable case identifier.
|
||||||
|
pub id: String,
|
||||||
|
/// Exact entity name or alias query.
|
||||||
|
pub query: String,
|
||||||
|
/// Optional requested language/locale.
|
||||||
|
#[serde(default)]
|
||||||
|
pub locale: Option<String>,
|
||||||
|
/// Optional requested two-letter country.
|
||||||
|
#[serde(default)]
|
||||||
|
pub country: Option<String>,
|
||||||
|
/// Required resolver outcome.
|
||||||
|
pub expected_status: ResolutionStatus,
|
||||||
|
/// Optional stable entity assertion.
|
||||||
|
#[serde(default)]
|
||||||
|
pub expected_entity_id: Option<String>,
|
||||||
|
/// Optional exact normalized destination assertion.
|
||||||
|
#[serde(default)]
|
||||||
|
pub expected_url: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A judgment that differed from the native result.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct Failure {
|
||||||
|
/// Case identifier.
|
||||||
|
pub id: String,
|
||||||
|
/// Expected resolver status.
|
||||||
|
pub expected_status: ResolutionStatus,
|
||||||
|
/// Actual resolver status.
|
||||||
|
pub actual_status: ResolutionStatus,
|
||||||
|
/// Expected entity when one was asserted.
|
||||||
|
pub expected_entity_id: Option<String>,
|
||||||
|
/// Actual selected entity, if any.
|
||||||
|
pub actual_entity_id: Option<String>,
|
||||||
|
/// Expected exact URL when one was asserted.
|
||||||
|
pub expected_url: Option<String>,
|
||||||
|
/// Actual selected URL, if any.
|
||||||
|
pub actual_url: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Aggregate correctness and native per-query latency evidence.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct Report {
|
||||||
|
/// Versioned evaluation output contract.
|
||||||
|
pub schema: String,
|
||||||
|
/// Exact pinned generation under evaluation.
|
||||||
|
pub registry: String,
|
||||||
|
/// Explicit resolver clock used for every case.
|
||||||
|
pub evaluated_at: chrono::DateTime<chrono::Utc>,
|
||||||
|
/// Parsed cases.
|
||||||
|
pub total: u64,
|
||||||
|
/// Cases matching every asserted field.
|
||||||
|
pub passed: u64,
|
||||||
|
/// Cases with at least one mismatch.
|
||||||
|
pub failed: u64,
|
||||||
|
/// Median native resolution time in microseconds.
|
||||||
|
pub latency_p50_us: u64,
|
||||||
|
/// 95th-percentile native resolution time in microseconds.
|
||||||
|
pub latency_p95_us: u64,
|
||||||
|
/// Complete mismatch list; bounded by `maximum_cases`.
|
||||||
|
pub failures: Vec<Failure>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Evaluates newline-delimited cases against one already verified registry.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects oversized/malformed inputs, duplicate IDs and unsafe case bounds.
|
||||||
|
pub fn run(
|
||||||
|
registry: &Registry,
|
||||||
|
input: impl BufRead,
|
||||||
|
maximum_cases: u32,
|
||||||
|
evaluated_at: chrono::DateTime<chrono::Utc>,
|
||||||
|
) -> anyhow::Result<Report> {
|
||||||
|
ensure!(
|
||||||
|
(1..=100_000).contains(&maximum_cases),
|
||||||
|
"maximum cases must be 1..100000"
|
||||||
|
);
|
||||||
|
let mut ids = std::collections::BTreeSet::new();
|
||||||
|
let mut durations = Vec::new();
|
||||||
|
let mut failures = Vec::new();
|
||||||
|
let mut total = 0_u64;
|
||||||
|
for line in input.lines() {
|
||||||
|
let line = line?;
|
||||||
|
if line.trim().is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
line.len() <= MAXIMUM_LINE_BYTES,
|
||||||
|
"evaluation line exceeds 64 KiB"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
total < u64::from(maximum_cases),
|
||||||
|
"evaluation exceeds maximum cases"
|
||||||
|
);
|
||||||
|
let case: Case = serde_json::from_value(crate::json::parse(line.as_bytes())?)?;
|
||||||
|
validate(&case)?;
|
||||||
|
ensure!(ids.insert(case.id.clone()), "duplicate evaluation case ID");
|
||||||
|
total += 1;
|
||||||
|
let started = Instant::now();
|
||||||
|
let actual = registry.resolve_explained(
|
||||||
|
&case.query,
|
||||||
|
case.locale.as_deref(),
|
||||||
|
case.country.as_deref(),
|
||||||
|
evaluated_at,
|
||||||
|
)?;
|
||||||
|
durations.push(u64::try_from(started.elapsed().as_micros())?);
|
||||||
|
let entity = actual
|
||||||
|
.destination
|
||||||
|
.as_ref()
|
||||||
|
.map(|value| value.entity_id.clone());
|
||||||
|
let url = actual.destination.as_ref().map(|value| value.url.clone());
|
||||||
|
if actual.status != case.expected_status
|
||||||
|
|| case
|
||||||
|
.expected_entity_id
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|value| Some(value) != entity.as_ref())
|
||||||
|
|| case
|
||||||
|
.expected_url
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|value| Some(value) != url.as_ref())
|
||||||
|
{
|
||||||
|
failures.push(Failure {
|
||||||
|
id: case.id,
|
||||||
|
expected_status: case.expected_status,
|
||||||
|
actual_status: actual.status,
|
||||||
|
expected_entity_id: case.expected_entity_id,
|
||||||
|
actual_entity_id: entity,
|
||||||
|
expected_url: case.expected_url,
|
||||||
|
actual_url: url,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ensure!(total > 0, "evaluation contains no cases");
|
||||||
|
durations.sort_unstable();
|
||||||
|
let failed = u64::try_from(failures.len())?;
|
||||||
|
Ok(Report {
|
||||||
|
schema: "argand.site-evaluation/v1".into(),
|
||||||
|
registry: registry.identity.clone(),
|
||||||
|
evaluated_at,
|
||||||
|
total,
|
||||||
|
passed: total - failed,
|
||||||
|
failed,
|
||||||
|
latency_p50_us: percentile(&durations, 50),
|
||||||
|
latency_p95_us: percentile(&durations, 95),
|
||||||
|
failures,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opens a bounded regular JSONL file and runs [`run`].
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns secure-open, input, or evaluation errors.
|
||||||
|
pub fn run_file(
|
||||||
|
registry: &Registry,
|
||||||
|
path: &Path,
|
||||||
|
maximum_cases: u32,
|
||||||
|
evaluated_at: chrono::DateTime<chrono::Utc>,
|
||||||
|
) -> anyhow::Result<Report> {
|
||||||
|
let input = crate::ssh::sealed_input(path, MAXIMUM_INPUT_BYTES)
|
||||||
|
.with_context(|| format!("open evaluation {}", path.display()))?;
|
||||||
|
run(
|
||||||
|
registry,
|
||||||
|
BufReader::new(input.file),
|
||||||
|
maximum_cases,
|
||||||
|
evaluated_at,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate(case: &Case) -> anyhow::Result<()> {
|
||||||
|
ensure!(
|
||||||
|
!case.id.trim().is_empty() && case.id.len() <= 256,
|
||||||
|
"evaluation case ID is required and bounded"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
!case.query.trim().is_empty() && case.query.len() <= 4096,
|
||||||
|
"evaluation query is required and bounded"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
case.locale.as_ref().is_none_or(|value| value.len() <= 64),
|
||||||
|
"evaluation locale is too long"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
case.country.as_ref().is_none_or(|value| {
|
||||||
|
value.len() == 2 && value.bytes().all(|byte| byte.is_ascii_uppercase())
|
||||||
|
}),
|
||||||
|
"evaluation country must be uppercase two-letter code"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
case.expected_entity_id
|
||||||
|
.as_ref()
|
||||||
|
.is_none_or(|value| value.starts_with("argand:entity:") && value.len() <= 256),
|
||||||
|
"invalid expected entity ID"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
case.expected_url
|
||||||
|
.as_ref()
|
||||||
|
.is_none_or(|value| value.len() <= 8192),
|
||||||
|
"expected URL is too long"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn percentile(values: &[u64], percentile: usize) -> u64 {
|
||||||
|
let index = values.len().saturating_mul(percentile).div_ceil(100);
|
||||||
|
values[index.saturating_sub(1).min(values.len() - 1)]
|
||||||
|
}
|
||||||
168
crates/argand-site-registry/src/generation.rs
Normal file
168
crates/argand-site-registry/src/generation.rs
Normal file
|
|
@ -0,0 +1,168 @@
|
||||||
|
// By Nic Weyand!
|
||||||
|
//! Exact immutable generation opening and descriptor-bound SQLite verification.
|
||||||
|
|
||||||
|
use crate::{build::Receipt, query::Registry};
|
||||||
|
use anyhow::ensure;
|
||||||
|
use rusqlite::Connection;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::{
|
||||||
|
collections::BTreeSet,
|
||||||
|
fs::{File, OpenOptions},
|
||||||
|
io::{Read, Seek},
|
||||||
|
path::Path,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
use std::os::{fd::AsRawFd, unix::fs::OpenOptionsExt};
|
||||||
|
|
||||||
|
impl Registry {
|
||||||
|
/// Opens only a complete, externally pinned generation.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects altered receipts/databases and unsupported contracts.
|
||||||
|
pub fn open(path: &Path, expected_pin: &str) -> anyhow::Result<Self> {
|
||||||
|
ensure!(
|
||||||
|
crate::model::valid_digest(expected_pin),
|
||||||
|
"provide a full trusted receipt SHA-256"
|
||||||
|
);
|
||||||
|
verify_layout(path)?;
|
||||||
|
let receipt_bytes = read_sealed(&path.join("COMPLETE.json"), 1024 * 1024)?;
|
||||||
|
ensure!(
|
||||||
|
crate::digest(&receipt_bytes) == expected_pin,
|
||||||
|
"receipt pin mismatch"
|
||||||
|
);
|
||||||
|
let receipt: Receipt = serde_json::from_value(crate::json::parse(&receipt_bytes)?)?;
|
||||||
|
let licenses = read_sealed(&path.join("LICENSE_SOURCES.md"), 1024 * 1024)?;
|
||||||
|
let attribution = read_sealed(&path.join("ATTRIBUTION.json"), 1024 * 1024)?;
|
||||||
|
ensure!(
|
||||||
|
crate::digest(&licenses) == receipt.licenses_sha256
|
||||||
|
&& crate::digest(&attribution) == receipt.attribution_sha256,
|
||||||
|
"registry license or attribution digest mismatch"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
receipt.schema == "argand.site-registry/v1"
|
||||||
|
&& crate::store::supported_rule_version(&receipt.rules),
|
||||||
|
"unsupported registry contract"
|
||||||
|
);
|
||||||
|
let database = path.join("registry.sqlite");
|
||||||
|
let (db, database) = open_authenticated_database(&database, &receipt.database_sha256)?;
|
||||||
|
crate::store::configure(&db)?;
|
||||||
|
Ok(Self {
|
||||||
|
db,
|
||||||
|
_database: database,
|
||||||
|
identity: expected_pin.into(),
|
||||||
|
receipt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_layout(path: &Path) -> anyhow::Result<()> {
|
||||||
|
ensure!(
|
||||||
|
std::fs::symlink_metadata(path)?.is_dir(),
|
||||||
|
"generation must be a real directory"
|
||||||
|
);
|
||||||
|
let allowed = BTreeSet::from([
|
||||||
|
"ATTRIBUTION.json",
|
||||||
|
"COMPLETE.json",
|
||||||
|
"COMPLETE.json.sig",
|
||||||
|
"LICENSE_SOURCES.md",
|
||||||
|
"registry.sqlite",
|
||||||
|
]);
|
||||||
|
let mut found = BTreeSet::new();
|
||||||
|
for entry in std::fs::read_dir(path)? {
|
||||||
|
let entry = entry?;
|
||||||
|
let name = entry
|
||||||
|
.file_name()
|
||||||
|
.into_string()
|
||||||
|
.map_err(|_| anyhow::anyhow!("non-UTF8 generation entry"))?;
|
||||||
|
ensure!(
|
||||||
|
allowed.contains(name.as_str()),
|
||||||
|
"unexpected generation entry: {name}"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
entry.file_type()?.is_file(),
|
||||||
|
"generation entries must be regular files"
|
||||||
|
);
|
||||||
|
found.insert(name);
|
||||||
|
}
|
||||||
|
for required in [
|
||||||
|
"ATTRIBUTION.json",
|
||||||
|
"COMPLETE.json",
|
||||||
|
"LICENSE_SOURCES.md",
|
||||||
|
"registry.sqlite",
|
||||||
|
] {
|
||||||
|
ensure!(found.contains(required), "generation is missing {required}");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_sealed(path: &Path, maximum: usize) -> anyhow::Result<Vec<u8>> {
|
||||||
|
let mut file = open_no_follow(path)?;
|
||||||
|
ensure!(
|
||||||
|
file.metadata()?.is_file(),
|
||||||
|
"generation artifact must be a regular file"
|
||||||
|
);
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
file.by_ref()
|
||||||
|
.take(u64::try_from(maximum)? + 1)
|
||||||
|
.read_to_end(&mut bytes)?;
|
||||||
|
ensure!(bytes.len() <= maximum, "generation metadata exceeds 1 MiB");
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
pub(crate) fn open_no_follow(path: &Path) -> anyhow::Result<File> {
|
||||||
|
Ok(
|
||||||
|
OpenOptions::new() // atomic-writes: allow read-only no-follow descriptor open
|
||||||
|
.read(true)
|
||||||
|
.custom_flags(libc::O_NOFOLLOW)
|
||||||
|
.open(path)?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "linux"))]
|
||||||
|
pub(crate) fn open_no_follow(_path: &Path) -> anyhow::Result<File> {
|
||||||
|
anyhow::bail!("secure immutable generation reads currently require Linux")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn open_authenticated_database(path: &Path, expected: &str) -> anyhow::Result<(Connection, File)> {
|
||||||
|
let mut file = open_no_follow(path)?;
|
||||||
|
ensure!(
|
||||||
|
file.metadata()?.is_file(),
|
||||||
|
"database must be a regular file"
|
||||||
|
);
|
||||||
|
let mut hash = Sha256::new();
|
||||||
|
let mut buffer = [0; 8192];
|
||||||
|
loop {
|
||||||
|
let count = file.read(&mut buffer)?;
|
||||||
|
if count == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
hash.update(&buffer[..count]);
|
||||||
|
}
|
||||||
|
ensure!(
|
||||||
|
format!("{:x}", hash.finalize()) == expected,
|
||||||
|
"registry database digest mismatch"
|
||||||
|
);
|
||||||
|
file.rewind()?;
|
||||||
|
let descriptor = format!("/proc/self/fd/{}", file.as_raw_fd());
|
||||||
|
let mut uri = url::Url::from_file_path(descriptor)
|
||||||
|
.map_err(|()| anyhow::anyhow!("cannot construct immutable SQLite URI"))?;
|
||||||
|
uri.query_pairs_mut()
|
||||||
|
.append_pair("mode", "ro")
|
||||||
|
.append_pair("immutable", "1");
|
||||||
|
let flags = rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY
|
||||||
|
| rusqlite::OpenFlags::SQLITE_OPEN_URI
|
||||||
|
| rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX;
|
||||||
|
let db = Connection::open_with_flags(uri.as_str(), flags)?;
|
||||||
|
Ok((db, file))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "linux"))]
|
||||||
|
fn open_authenticated_database(
|
||||||
|
_path: &Path,
|
||||||
|
_expected: &str,
|
||||||
|
) -> anyhow::Result<(Connection, File)> {
|
||||||
|
anyhow::bail!("secure immutable generation reads currently require Linux")
|
||||||
|
}
|
||||||
|
|
@ -69,6 +69,32 @@ pub fn record(
|
||||||
registry: &Registry,
|
registry: &Registry,
|
||||||
pair: &Equivalence,
|
pair: &Equivalence,
|
||||||
review: &Review,
|
review: &Review,
|
||||||
|
) -> anyhow::Result<u64> {
|
||||||
|
record_inner(db, registry, pair, review, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Appends a cryptographically authenticated identity decision.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects stale evidence, invalid scope, missing source identities, or proof
|
||||||
|
/// that does not match the decision's reviewer.
|
||||||
|
pub fn record_authenticated(
|
||||||
|
db: &Connection,
|
||||||
|
registry: &Registry,
|
||||||
|
pair: &Equivalence,
|
||||||
|
review: &Review,
|
||||||
|
authentication: &crate::review::Authentication,
|
||||||
|
) -> anyhow::Result<u64> {
|
||||||
|
crate::review::validate_authentication(review, authentication)?;
|
||||||
|
record_inner(db, registry, pair, review, Some(authentication))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_inner(
|
||||||
|
db: &Connection,
|
||||||
|
registry: &Registry,
|
||||||
|
pair: &Equivalence,
|
||||||
|
review: &Review,
|
||||||
|
authentication: Option<&crate::review::Authentication>,
|
||||||
) -> anyhow::Result<u64> {
|
) -> anyhow::Result<u64> {
|
||||||
crate::review::validate(review)?;
|
crate::review::validate(review)?;
|
||||||
let expected = propose(registry, &pair.entities[0], &pair.entities[1])?;
|
let expected = propose(registry, &pair.entities[0], &pair.entities[1])?;
|
||||||
|
|
@ -98,6 +124,9 @@ pub fn record(
|
||||||
],
|
],
|
||||||
)?;
|
)?;
|
||||||
let sequence = crate::review::append(db, review)?;
|
let sequence = crate::review::append(db, review)?;
|
||||||
|
if let Some(authentication) = authentication {
|
||||||
|
crate::review::append_authentication(db, sequence, authentication)?;
|
||||||
|
}
|
||||||
transaction.commit()?;
|
transaction.commit()?;
|
||||||
Ok(sequence)
|
Ok(sequence)
|
||||||
}
|
}
|
||||||
|
|
@ -181,11 +210,25 @@ pub(crate) fn expand(
|
||||||
Ok(Some((entities, evidence.into_values().collect())))
|
Ok(Some((entities, evidence.into_values().collect())))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn candidates(
|
pub(crate) enum CandidateSearch {
|
||||||
|
Ready {
|
||||||
|
matched_entities: u64,
|
||||||
|
candidates: Vec<crate::query::Candidate>,
|
||||||
|
},
|
||||||
|
NoMatch,
|
||||||
|
AmbiguousIdentity {
|
||||||
|
matched_entities: u64,
|
||||||
|
},
|
||||||
|
SafetyLimitExceeded {
|
||||||
|
matched_entities: u64,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn candidate_search(
|
||||||
registry: &Registry,
|
registry: &Registry,
|
||||||
query: &str,
|
query: &str,
|
||||||
now: DateTime<Utc>,
|
now: DateTime<Utc>,
|
||||||
) -> anyhow::Result<Vec<crate::query::Candidate>> {
|
) -> anyhow::Result<CandidateSearch> {
|
||||||
let key = crate::normalize::name_key(query)?;
|
let key = crate::normalize::name_key(query)?;
|
||||||
let mut statement = registry
|
let mut statement = registry
|
||||||
.db
|
.db
|
||||||
|
|
@ -193,14 +236,23 @@ pub(crate) fn candidates(
|
||||||
let matched = statement
|
let matched = statement
|
||||||
.query_map([key], |r| r.get::<_, String>(0))?
|
.query_map([key], |r| r.get::<_, String>(0))?
|
||||||
.collect::<Result<Vec<_>, _>>()?;
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
if matched.is_empty() || matched.len() > 64 {
|
if matched.is_empty() {
|
||||||
return Ok(Vec::new());
|
return Ok(CandidateSearch::NoMatch);
|
||||||
|
}
|
||||||
|
if matched.len() > 64 {
|
||||||
|
return Ok(CandidateSearch::SafetyLimitExceeded {
|
||||||
|
matched_entities: u64::try_from(matched.len())?,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
let Some((entities, evidence)) = expand(registry, &matched[0], now)? else {
|
let Some((entities, evidence)) = expand(registry, &matched[0], now)? else {
|
||||||
return Ok(Vec::new());
|
return Ok(CandidateSearch::SafetyLimitExceeded {
|
||||||
|
matched_entities: u64::try_from(matched.len())?,
|
||||||
|
});
|
||||||
};
|
};
|
||||||
if matched.iter().any(|id| !entities.contains(id)) {
|
if matched.iter().any(|id| !entities.contains(id)) {
|
||||||
return Ok(Vec::new());
|
return Ok(CandidateSearch::AmbiguousIdentity {
|
||||||
|
matched_entities: u64::try_from(matched.len())?,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
let mut output = Vec::new();
|
let mut output = Vec::new();
|
||||||
for entity in entities {
|
for entity in entities {
|
||||||
|
|
@ -212,9 +264,14 @@ pub(crate) fn candidates(
|
||||||
candidate.identity_provenance.clone_from(&evidence);
|
candidate.identity_provenance.clone_from(&evidence);
|
||||||
output.push(candidate);
|
output.push(candidate);
|
||||||
if output.len() > 100 {
|
if output.len() > 100 {
|
||||||
return Ok(Vec::new());
|
return Ok(CandidateSearch::SafetyLimitExceeded {
|
||||||
|
matched_entities: u64::try_from(matched.len())?,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(output)
|
Ok(CandidateSearch::Ready {
|
||||||
|
matched_entities: u64::try_from(matched.len())?,
|
||||||
|
candidates: output,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,9 +3,13 @@
|
||||||
|
|
||||||
pub mod adapters;
|
pub mod adapters;
|
||||||
pub mod build;
|
pub mod build;
|
||||||
|
pub mod catalog;
|
||||||
pub mod crux;
|
pub mod crux;
|
||||||
|
pub mod diff;
|
||||||
pub mod download;
|
pub mod download;
|
||||||
|
pub mod evaluation;
|
||||||
pub mod evidence;
|
pub mod evidence;
|
||||||
|
mod generation;
|
||||||
pub mod identity;
|
pub mod identity;
|
||||||
mod json;
|
mod json;
|
||||||
pub mod model;
|
pub mod model;
|
||||||
|
|
@ -13,8 +17,12 @@ pub mod normalize;
|
||||||
pub mod observation;
|
pub mod observation;
|
||||||
pub mod query;
|
pub mod query;
|
||||||
pub mod release;
|
pub mod release;
|
||||||
|
mod resolution;
|
||||||
pub mod review;
|
pub mod review;
|
||||||
|
mod ssh;
|
||||||
pub mod store;
|
pub mod store;
|
||||||
|
|
||||||
|
pub use resolution::{Resolution, ResolutionCounts, ResolutionStatus};
|
||||||
pub mod update;
|
pub mod update;
|
||||||
|
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
|
||||||
|
|
@ -58,3 +58,130 @@ pub struct Observation {
|
||||||
/// Confidence on the same 0..10000 policy scale as source facts.
|
/// Confidence on the same 0..10000 policy scale as source facts.
|
||||||
pub confidence: u16,
|
pub confidence: u16,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Deterministic crawler evidence prepared for a later rights-reviewed adapter.
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
pub struct NormalizedObservation {
|
||||||
|
/// Versioned extension contract.
|
||||||
|
pub schema: String,
|
||||||
|
/// Stable fingerprint over all normalized evidence fields.
|
||||||
|
pub fingerprint: String,
|
||||||
|
/// Observed relationship without any ownership inference.
|
||||||
|
pub relation: ObservationKind,
|
||||||
|
/// Strict normalized source page.
|
||||||
|
pub from: crate::normalize::WebProperty,
|
||||||
|
/// Strict normalized target page.
|
||||||
|
pub to: crate::normalize::WebProperty,
|
||||||
|
/// Source provider/capture collection.
|
||||||
|
pub source: String,
|
||||||
|
/// Immutable source-native capture identifier.
|
||||||
|
pub source_identifier: String,
|
||||||
|
/// Rights declaration retained with the observation.
|
||||||
|
pub license: String,
|
||||||
|
/// Rights evidence URL retained with the observation.
|
||||||
|
pub license_url: String,
|
||||||
|
/// Retrieval instant.
|
||||||
|
pub retrieved_at: DateTime<Utc>,
|
||||||
|
/// Captured source-content hash.
|
||||||
|
pub content_sha256: String,
|
||||||
|
/// Exact assertion locator.
|
||||||
|
pub selector: String,
|
||||||
|
/// Source confidence, separate from destination review.
|
||||||
|
pub confidence: u16,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Observation {
|
||||||
|
/// Validates and normalizes an observation without admitting it as ownership.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects malformed evidence coordinates, URL values and relation scopes.
|
||||||
|
pub fn normalize(
|
||||||
|
&self,
|
||||||
|
normalizer: &crate::normalize::Normalizer,
|
||||||
|
) -> anyhow::Result<NormalizedObservation> {
|
||||||
|
use anyhow::ensure;
|
||||||
|
ensure!(
|
||||||
|
!self.source.trim().is_empty()
|
||||||
|
&& self.source.len() <= 128
|
||||||
|
&& !self.source_identifier.trim().is_empty()
|
||||||
|
&& self.source_identifier.len() <= 1024,
|
||||||
|
"observation source coordinates are required and bounded"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
!self.license.trim().is_empty()
|
||||||
|
&& self.license.len() <= 128
|
||||||
|
&& self.license_url.len() <= 2048,
|
||||||
|
"observation rights declaration is required and bounded"
|
||||||
|
);
|
||||||
|
let license_url = url::Url::parse(&self.license_url)?;
|
||||||
|
ensure!(
|
||||||
|
license_url.scheme() == "https" && license_url.host_str().is_some(),
|
||||||
|
"observation license evidence must be an HTTPS URL"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
crate::model::valid_digest(&self.content_sha256),
|
||||||
|
"invalid observation content digest"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
!self.selector.trim().is_empty() && self.selector.len() <= 2048,
|
||||||
|
"observation selector is required and bounded"
|
||||||
|
);
|
||||||
|
ensure!(self.confidence <= 10_000, "invalid observation confidence");
|
||||||
|
validate_relation(&self.relation)?;
|
||||||
|
let from = normalizer.url(&self.from_url)?;
|
||||||
|
let to = normalizer.url(&self.to_url)?;
|
||||||
|
let evidence = (
|
||||||
|
"argand.site-observation/v1",
|
||||||
|
&self.relation,
|
||||||
|
&from,
|
||||||
|
&to,
|
||||||
|
&self.source,
|
||||||
|
&self.source_identifier,
|
||||||
|
&self.license,
|
||||||
|
&self.license_url,
|
||||||
|
self.retrieved_at,
|
||||||
|
&self.content_sha256,
|
||||||
|
&self.selector,
|
||||||
|
self.confidence,
|
||||||
|
);
|
||||||
|
Ok(NormalizedObservation {
|
||||||
|
schema: "argand.site-observation/v1".into(),
|
||||||
|
fingerprint: crate::digest(&serde_json::to_vec(&evidence)?),
|
||||||
|
relation: self.relation.clone(),
|
||||||
|
from,
|
||||||
|
to,
|
||||||
|
source: self.source.clone(),
|
||||||
|
source_identifier: self.source_identifier.clone(),
|
||||||
|
license: self.license.clone(),
|
||||||
|
license_url: self.license_url.clone(),
|
||||||
|
retrieved_at: self.retrieved_at,
|
||||||
|
content_sha256: self.content_sha256.clone(),
|
||||||
|
selector: self.selector.clone(),
|
||||||
|
confidence: self.confidence,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_relation(relation: &ObservationKind) -> anyhow::Result<()> {
|
||||||
|
use anyhow::ensure;
|
||||||
|
match relation {
|
||||||
|
ObservationKind::Redirect { status } => ensure!(
|
||||||
|
matches!(status, 301 | 302 | 303 | 307 | 308),
|
||||||
|
"invalid HTTP redirect status"
|
||||||
|
),
|
||||||
|
ObservationKind::Hreflang { locale } => ensure!(
|
||||||
|
!locale.is_empty()
|
||||||
|
&& locale.len() <= 64
|
||||||
|
&& locale
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-'),
|
||||||
|
"invalid hreflang locale"
|
||||||
|
),
|
||||||
|
ObservationKind::CountrySelector { country } => ensure!(
|
||||||
|
country.len() == 2 && country.bytes().all(|byte| byte.is_ascii_uppercase()),
|
||||||
|
"invalid country selector scope"
|
||||||
|
),
|
||||||
|
ObservationKind::Canonical | ObservationKind::JsonLdSameAs | ObservationKind::Sitemap => {}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,16 +2,19 @@
|
||||||
//! Bounded native lookup; ambiguity is counted before limits or policy filtering.
|
//! Bounded native lookup; ambiguity is counted before limits or policy filtering.
|
||||||
|
|
||||||
use crate::{build::Receipt, normalize::name_key};
|
use crate::{build::Receipt, normalize::name_key};
|
||||||
use anyhow::{Context, ensure};
|
use anyhow::ensure;
|
||||||
use chrono::{DateTime, Utc};
|
|
||||||
use rusqlite::{Connection, OptionalExtension, params};
|
use rusqlite::{Connection, OptionalExtension, params};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use std::path::Path;
|
use std::fs::File;
|
||||||
|
|
||||||
|
pub use crate::resolution::{Resolution, ResolutionCounts, ResolutionStatus};
|
||||||
|
|
||||||
/// Open verified generation. Hashes are checked once, outside the query path.
|
/// Open verified generation. Hashes are checked once, outside the query path.
|
||||||
pub struct Registry {
|
pub struct Registry {
|
||||||
pub(crate) db: Connection,
|
pub(crate) db: Connection,
|
||||||
|
// Keep the authenticated inode alive for the SQLite /proc/self/fd reader.
|
||||||
|
pub(crate) _database: File,
|
||||||
/// External receipt pin supplied by the caller.
|
/// External receipt pin supplied by the caller.
|
||||||
pub identity: String,
|
pub identity: String,
|
||||||
/// Verified manifest.
|
/// Verified manifest.
|
||||||
|
|
@ -138,49 +141,6 @@ impl Registry {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Opens only a complete, externally pinned generation.
|
|
||||||
///
|
|
||||||
/// # Errors
|
|
||||||
/// Rejects altered receipts/databases and unsupported contracts.
|
|
||||||
pub fn open(path: &Path, expected_pin: &str) -> anyhow::Result<Self> {
|
|
||||||
ensure!(
|
|
||||||
crate::model::valid_digest(expected_pin),
|
|
||||||
"provide a full trusted receipt SHA-256"
|
|
||||||
);
|
|
||||||
ensure!(
|
|
||||||
crate::file_digest(&path.join("COMPLETE.json"))? == expected_pin,
|
|
||||||
"receipt pin mismatch"
|
|
||||||
);
|
|
||||||
let receipt: Receipt = crate::read_json(&path.join("COMPLETE.json"))?;
|
|
||||||
ensure!(
|
|
||||||
crate::file_digest(&path.join("LICENSE_SOURCES.md"))? == receipt.licenses_sha256
|
|
||||||
&& crate::file_digest(&path.join("ATTRIBUTION.json"))?
|
|
||||||
== receipt.attribution_sha256,
|
|
||||||
"registry license or attribution digest mismatch"
|
|
||||||
);
|
|
||||||
ensure!(
|
|
||||||
receipt.schema == "argand.site-registry/v1"
|
|
||||||
&& receipt.rules == crate::store::RULE_VERSION,
|
|
||||||
"unsupported registry contract"
|
|
||||||
);
|
|
||||||
let database = path.join("registry.sqlite");
|
|
||||||
ensure!(
|
|
||||||
std::fs::symlink_metadata(&database)?.is_file(),
|
|
||||||
"database must be a regular file"
|
|
||||||
);
|
|
||||||
ensure!(
|
|
||||||
crate::file_digest(&database)? == receipt.database_sha256,
|
|
||||||
"registry database digest mismatch"
|
|
||||||
);
|
|
||||||
let db = Connection::open_with_flags(database, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
|
||||||
crate::store::configure(&db)?;
|
|
||||||
Ok(Self {
|
|
||||||
db,
|
|
||||||
identity: expected_pin.into(),
|
|
||||||
receipt,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Indexed exact-name/alias lookup, with counts independent of result limits.
|
/// Indexed exact-name/alias lookup, with counts independent of result limits.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
|
|
@ -227,7 +187,11 @@ impl Registry {
|
||||||
for id in ids {
|
for id in ids {
|
||||||
provenance.push(crate::evidence::fact(&self.db, &id)?);
|
provenance.push(crate::evidence::fact(&self.db, &id)?);
|
||||||
}
|
}
|
||||||
let review=self.db.query_row("SELECT sequence,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country FROM reviews WHERE fingerprint=?1 ORDER BY sequence DESC LIMIT 1",[fingerprint],|r|Ok(json!({"sequence":crate::store::unsigned(r,0)?,"decision":r.get::<_,String>(1)?,"reviewer":r.get::<_,String>(2)?,"reason":r.get::<_,String>(3)?,"evidence":r.get::<_,String>(4)?,"retrieved_at":r.get::<_,String>(5)?,"expires_at":r.get::<_,String>(6)?,"role":r.get::<_,String>(7)?,"locale":r.get::<_,String>(8)?,"country":r.get::<_,String>(9)?,"source":"argand_operator_review","source_identifier":format!("{}:{}",fingerprint,crate::store::unsigned(r,0)?),"license":"CC0-1.0","license_url":"https://creativecommons.org/publicdomain/zero/1.0/","confidence":9000}))).optional()?;
|
let review=self.db.query_row("SELECT r.sequence,r.decision,r.reviewer,r.reason,r.evidence,r.reviewed_at,r.expires_at,r.role,r.locale,r.country,a.signer,a.signature_sha256,a.namespace FROM reviews r LEFT JOIN review_auth a ON a.sequence=r.sequence WHERE r.fingerprint=?1 ORDER BY r.sequence DESC LIMIT 1",[fingerprint],|r|{
|
||||||
|
let sequence=crate::store::unsigned(r,0)?;
|
||||||
|
let signer:Option<String>=r.get(10)?;
|
||||||
|
Ok(json!({"sequence":sequence,"decision":r.get::<_,String>(1)?,"reviewer":r.get::<_,String>(2)?,"reason":r.get::<_,String>(3)?,"evidence":r.get::<_,String>(4)?,"retrieved_at":r.get::<_,String>(5)?,"expires_at":r.get::<_,String>(6)?,"role":r.get::<_,String>(7)?,"locale":r.get::<_,String>(8)?,"country":r.get::<_,String>(9)?,"authentication":signer.map(|identity|json!({"identity":identity,"signature_sha256":r.get::<_,String>(11).unwrap_or_default(),"namespace":r.get::<_,String>(12).unwrap_or_default()})),"source":"argand_operator_review","source_identifier":format!("{}:{sequence}",fingerprint),"license":"CC0-1.0","license_url":"https://creativecommons.org/publicdomain/zero/1.0/","confidence":9000}))
|
||||||
|
}).optional()?;
|
||||||
Ok(Candidate {
|
Ok(Candidate {
|
||||||
identity_provenance: Vec::new(),
|
identity_provenance: Vec::new(),
|
||||||
entity: crate::evidence::entity(&self.db, &entity, &name)?,
|
entity: crate::evidence::entity(&self.db, &entity, &name)?,
|
||||||
|
|
@ -255,78 +219,4 @@ impl Registry {
|
||||||
eligible,
|
eligible,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Chooses an approved regional property or an explicitly reviewed primary.
|
|
||||||
/// Never returns a destination for ambiguous entities or tied properties.
|
|
||||||
///
|
|
||||||
/// # Errors
|
|
||||||
/// Returns malformed requests/data or database failures.
|
|
||||||
pub fn resolve(
|
|
||||||
&self,
|
|
||||||
query: &str,
|
|
||||||
locale: Option<&str>,
|
|
||||||
country: Option<&str>,
|
|
||||||
now: DateTime<Utc>,
|
|
||||||
) -> anyhow::Result<Option<Candidate>> {
|
|
||||||
let candidates = crate::identity::candidates(self, query, now)?;
|
|
||||||
let mut best = None;
|
|
||||||
let mut score = 0;
|
|
||||||
let mut ambiguous = false;
|
|
||||||
for candidate in candidates {
|
|
||||||
if !candidate.eligible {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let Some(review) = candidate.review.as_ref() else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
if review["decision"] != "approve" {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let expires = DateTime::parse_from_rfc3339(
|
|
||||||
review["expires_at"]
|
|
||||||
.as_str()
|
|
||||||
.context("invalid review expiry")?,
|
|
||||||
)?;
|
|
||||||
let starts = DateTime::parse_from_rfc3339(
|
|
||||||
review["retrieved_at"]
|
|
||||||
.as_str()
|
|
||||||
.context("invalid review timestamp")?,
|
|
||||||
)?;
|
|
||||||
if now < starts || now >= expires {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let region = review["country"].as_str().unwrap_or_default();
|
|
||||||
let language = review["locale"].as_str().unwrap_or_default();
|
|
||||||
let matches_country =
|
|
||||||
!region.is_empty() && country.is_some_and(|c| c.eq_ignore_ascii_case(region));
|
|
||||||
let matches_locale =
|
|
||||||
!language.is_empty() && locale.is_some_and(|l| l.eq_ignore_ascii_case(language));
|
|
||||||
let current = if review["role"] == "regional" {
|
|
||||||
// All asserted dimensions must match; a language alone cannot
|
|
||||||
// override an explicit country mismatch.
|
|
||||||
if (!region.is_empty() && !matches_country)
|
|
||||||
|| (!language.is_empty() && !matches_locale)
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
2 + u8::from(matches_country) + u8::from(matches_locale)
|
|
||||||
} else if review["role"] == "primary" {
|
|
||||||
1
|
|
||||||
} else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
if current > score {
|
|
||||||
best = Some(candidate);
|
|
||||||
score = current;
|
|
||||||
ambiguous = false;
|
|
||||||
} else if current == score
|
|
||||||
&& best
|
|
||||||
.as_ref()
|
|
||||||
.is_some_and(|b: &Candidate| b.url != candidate.url)
|
|
||||||
{
|
|
||||||
ambiguous = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(if ambiguous { None } else { best })
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ use std::{
|
||||||
fs::{self, File, OpenOptions},
|
fs::{self, File, OpenOptions},
|
||||||
io::{BufWriter, Write},
|
io::{BufWriter, Write},
|
||||||
path::Path,
|
path::Path,
|
||||||
process::{Command, Stdio},
|
process::Command,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Source terms shipped and authenticated with every generation.
|
/// Source terms shipped and authenticated with every generation.
|
||||||
|
|
@ -80,8 +80,14 @@ fn export_inner(
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Returns missing key, existing signature, and signing process failures.
|
/// Returns missing key, existing signature, and signing process failures.
|
||||||
pub fn sign(generation: &Path, key: &Path, pin: &str) -> anyhow::Result<()> {
|
pub fn sign(
|
||||||
Registry::open(generation, pin)?;
|
generation: &Path,
|
||||||
|
key: &Path,
|
||||||
|
pin: &str,
|
||||||
|
allowed_reviewers: &Path,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let registry = Registry::open(generation, pin)?;
|
||||||
|
crate::review::verify_all(®istry.db, allowed_reviewers)?;
|
||||||
ensure!(
|
ensure!(
|
||||||
!generation.join("COMPLETE.json.sig").exists(),
|
!generation.join("COMPLETE.json.sig").exists(),
|
||||||
"signature already exists"
|
"signature already exists"
|
||||||
|
|
@ -106,19 +112,20 @@ pub fn verify_signed(
|
||||||
signers: &Path,
|
signers: &Path,
|
||||||
identity: &str,
|
identity: &str,
|
||||||
) -> anyhow::Result<Registry> {
|
) -> anyhow::Result<Registry> {
|
||||||
let pin = crate::file_digest(&generation.join("COMPLETE.json"))?;
|
let receipt = crate::ssh::sealed_input(&generation.join("COMPLETE.json"), 1024 * 1024)?;
|
||||||
let status = Command::new("ssh-keygen")
|
let signature = crate::ssh::sealed_input(&generation.join("COMPLETE.json.sig"), 64 * 1024)?;
|
||||||
.args(["-Y", "verify", "-n", "argand-site-registry", "-f"])
|
let signers = crate::ssh::sealed_input(signers, 1024 * 1024)?;
|
||||||
.arg(signers)
|
let pin = crate::digest(&receipt.bytes);
|
||||||
.arg("-I")
|
crate::ssh::verify(
|
||||||
.arg(identity)
|
"argand-site-registry",
|
||||||
.arg("-s")
|
&receipt.bytes,
|
||||||
.arg(generation.join("COMPLETE.json.sig"))
|
&signature.bytes,
|
||||||
.stdin(Stdio::from(File::open(generation.join("COMPLETE.json"))?))
|
&signers.bytes,
|
||||||
.stdout(Stdio::null())
|
identity,
|
||||||
.status()?;
|
)?;
|
||||||
ensure!(status.success(), "untrusted registry signature");
|
let registry = Registry::open(generation, &pin)?;
|
||||||
Registry::open(generation, &pin)
|
crate::review::ensure_all_authenticated(®istry.db)?;
|
||||||
|
Ok(registry)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Activates a verified generation using one durable pointer. Refuses rollback
|
/// Activates a verified generation using one durable pointer. Refuses rollback
|
||||||
|
|
@ -212,26 +219,5 @@ fn preserve_revocations(old: &Registry, new: &Registry) -> anyhow::Result<()> {
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Returns query or output errors.
|
/// Returns query or output errors.
|
||||||
pub fn diff(old: &Registry, new: &Registry, output: &mut dyn Write) -> anyhow::Result<()> {
|
pub fn diff(old: &Registry, new: &Registry, output: &mut dyn Write) -> anyhow::Result<()> {
|
||||||
for (kind, from, to) in [("removed", old, new), ("added", new, old)] {
|
crate::diff::write(old, new, output)
|
||||||
let mut stmt = from
|
|
||||||
.db
|
|
||||||
.prepare("SELECT fingerprint,entity,property FROM edges ORDER BY fingerprint")?;
|
|
||||||
let mut rows = stmt.query([])?;
|
|
||||||
while let Some(row) = rows.next()? {
|
|
||||||
let fingerprint: String = row.get(0)?;
|
|
||||||
let exists: bool = to.db.query_row(
|
|
||||||
"SELECT EXISTS(SELECT 1 FROM edges WHERE fingerprint=?1)",
|
|
||||||
[&fingerprint],
|
|
||||||
|r| r.get(0),
|
|
||||||
)?;
|
|
||||||
if !exists {
|
|
||||||
writeln!(
|
|
||||||
output,
|
|
||||||
"{}",
|
|
||||||
json!({"change":kind,"fingerprint":fingerprint,"entity":row.get::<_,String>(1)?,"property":row.get::<_,String>(2)?})
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
238
crates/argand-site-registry/src/resolution.rs
Normal file
238
crates/argand-site-registry/src/resolution.rs
Normal file
|
|
@ -0,0 +1,238 @@
|
||||||
|
// By Nic Weyand!
|
||||||
|
//! Conservative destination selection with machine-readable abstention reasons.
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
normalize::name_key,
|
||||||
|
query::{Candidate, Registry},
|
||||||
|
};
|
||||||
|
use anyhow::Context;
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Explainable outcome from conservative destination resolution.
|
||||||
|
#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum ResolutionStatus {
|
||||||
|
/// One reviewed destination was selected.
|
||||||
|
Resolved,
|
||||||
|
/// No entity name or alias matched exactly after normalization.
|
||||||
|
NoNameMatch,
|
||||||
|
/// Matching source entities lack an active reviewed equivalence chain.
|
||||||
|
AmbiguousIdentity,
|
||||||
|
/// Identity or edge expansion exceeded defensive bounds.
|
||||||
|
SafetyLimitExceeded,
|
||||||
|
/// Matching entities have no currently eligible website assertion.
|
||||||
|
NoEligibleDestination,
|
||||||
|
/// Eligible assertions exist but none has a current approval.
|
||||||
|
NoActiveReview,
|
||||||
|
/// Active regional approvals do not match the requested scope.
|
||||||
|
RegionMismatch,
|
||||||
|
/// Equally specific active approvals point to different URLs.
|
||||||
|
AmbiguousDestination,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Counts explaining why resolution selected or rejected candidates.
|
||||||
|
#[derive(Clone, Debug, Default, Serialize)]
|
||||||
|
pub struct ResolutionCounts {
|
||||||
|
/// Source entities matching the normalized name or alias.
|
||||||
|
pub matched_entities: u64,
|
||||||
|
/// Assertion candidates examined after identity review.
|
||||||
|
pub considered: u64,
|
||||||
|
/// Candidates eligible for destination review.
|
||||||
|
pub eligible: u64,
|
||||||
|
/// Eligible candidates without any review entry.
|
||||||
|
pub missing_review: u64,
|
||||||
|
/// Candidates whose latest decision is a revocation.
|
||||||
|
pub revoked: u64,
|
||||||
|
/// Approvals whose validity interval has ended.
|
||||||
|
pub expired: u64,
|
||||||
|
/// Approvals whose validity interval has not begun.
|
||||||
|
pub not_yet_valid: u64,
|
||||||
|
/// Current approvals that do not match requested locale/country.
|
||||||
|
pub region_mismatch: u64,
|
||||||
|
/// Current approvals considered for final selection.
|
||||||
|
pub active_approvals: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolution envelope; a null destination always has a machine-readable reason.
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct Resolution {
|
||||||
|
/// Normalized query key.
|
||||||
|
pub query: String,
|
||||||
|
/// Selection outcome.
|
||||||
|
pub status: ResolutionStatus,
|
||||||
|
/// Reviewed destination, present only for [`ResolutionStatus::Resolved`].
|
||||||
|
pub destination: Option<Candidate>,
|
||||||
|
/// Complete bounded decision counts.
|
||||||
|
pub counts: ResolutionCounts,
|
||||||
|
/// Source attribution required when displaying an imported name.
|
||||||
|
pub attribution: serde_json::Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Registry {
|
||||||
|
/// Chooses an approved regional property or an explicitly reviewed primary.
|
||||||
|
/// Never returns a destination for ambiguous entities or tied properties.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns malformed requests/data or database failures.
|
||||||
|
pub fn resolve(
|
||||||
|
&self,
|
||||||
|
query: &str,
|
||||||
|
locale: Option<&str>,
|
||||||
|
country: Option<&str>,
|
||||||
|
now: DateTime<Utc>,
|
||||||
|
) -> anyhow::Result<Option<Candidate>> {
|
||||||
|
Ok(self
|
||||||
|
.resolve_explained(query, locale, country, now)?
|
||||||
|
.destination)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolves a destination and explains every conservative abstention.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns malformed requests/data or database failures.
|
||||||
|
pub fn resolve_explained(
|
||||||
|
&self,
|
||||||
|
query: &str,
|
||||||
|
locale: Option<&str>,
|
||||||
|
country: Option<&str>,
|
||||||
|
now: DateTime<Utc>,
|
||||||
|
) -> anyhow::Result<Resolution> {
|
||||||
|
let key = name_key(query)?;
|
||||||
|
let mut counts = ResolutionCounts::default();
|
||||||
|
let candidates = match crate::identity::candidate_search(self, query, now)? {
|
||||||
|
crate::identity::CandidateSearch::Ready {
|
||||||
|
matched_entities,
|
||||||
|
candidates,
|
||||||
|
} => {
|
||||||
|
counts.matched_entities = matched_entities;
|
||||||
|
candidates
|
||||||
|
}
|
||||||
|
crate::identity::CandidateSearch::NoMatch => {
|
||||||
|
return Ok(result(key, ResolutionStatus::NoNameMatch, None, counts));
|
||||||
|
}
|
||||||
|
crate::identity::CandidateSearch::AmbiguousIdentity { matched_entities } => {
|
||||||
|
counts.matched_entities = matched_entities;
|
||||||
|
return Ok(result(
|
||||||
|
key,
|
||||||
|
ResolutionStatus::AmbiguousIdentity,
|
||||||
|
None,
|
||||||
|
counts,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
crate::identity::CandidateSearch::SafetyLimitExceeded { matched_entities } => {
|
||||||
|
counts.matched_entities = matched_entities;
|
||||||
|
return Ok(result(
|
||||||
|
key,
|
||||||
|
ResolutionStatus::SafetyLimitExceeded,
|
||||||
|
None,
|
||||||
|
counts,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let (status, destination) = choose(candidates, locale, country, now, &mut counts)?;
|
||||||
|
Ok(result(key, status, destination, counts))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn choose(
|
||||||
|
candidates: Vec<Candidate>,
|
||||||
|
locale: Option<&str>,
|
||||||
|
country: Option<&str>,
|
||||||
|
now: DateTime<Utc>,
|
||||||
|
counts: &mut ResolutionCounts,
|
||||||
|
) -> anyhow::Result<(ResolutionStatus, Option<Candidate>)> {
|
||||||
|
let mut best = None;
|
||||||
|
let mut score = 0;
|
||||||
|
let mut ambiguous = false;
|
||||||
|
for candidate in candidates {
|
||||||
|
counts.considered += 1;
|
||||||
|
if !candidate.eligible {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
counts.eligible += 1;
|
||||||
|
let Some(review) = candidate.review.as_ref() else {
|
||||||
|
counts.missing_review += 1;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if review["decision"] != "approve" {
|
||||||
|
counts.revoked += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let expires = DateTime::parse_from_rfc3339(
|
||||||
|
review["expires_at"]
|
||||||
|
.as_str()
|
||||||
|
.context("invalid review expiry")?,
|
||||||
|
)?;
|
||||||
|
let starts = DateTime::parse_from_rfc3339(
|
||||||
|
review["retrieved_at"]
|
||||||
|
.as_str()
|
||||||
|
.context("invalid review timestamp")?,
|
||||||
|
)?;
|
||||||
|
if now < starts || now >= expires {
|
||||||
|
if now < starts {
|
||||||
|
counts.not_yet_valid += 1;
|
||||||
|
} else {
|
||||||
|
counts.expired += 1;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let region = review["country"].as_str().unwrap_or_default();
|
||||||
|
let language = review["locale"].as_str().unwrap_or_default();
|
||||||
|
let matches_country =
|
||||||
|
!region.is_empty() && country.is_some_and(|c| c.eq_ignore_ascii_case(region));
|
||||||
|
let matches_locale =
|
||||||
|
!language.is_empty() && locale.is_some_and(|l| l.eq_ignore_ascii_case(language));
|
||||||
|
let current = if review["role"] == "regional" {
|
||||||
|
if (!region.is_empty() && !matches_country) || (!language.is_empty() && !matches_locale)
|
||||||
|
{
|
||||||
|
counts.region_mismatch += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
2 + u8::from(matches_country) + u8::from(matches_locale)
|
||||||
|
} else if review["role"] == "primary" {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
counts.region_mismatch += 1;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
counts.active_approvals += 1;
|
||||||
|
if current > score {
|
||||||
|
best = Some(candidate);
|
||||||
|
score = current;
|
||||||
|
ambiguous = false;
|
||||||
|
} else if current == score
|
||||||
|
&& best
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|b: &Candidate| b.url != candidate.url)
|
||||||
|
{
|
||||||
|
ambiguous = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(if ambiguous {
|
||||||
|
(ResolutionStatus::AmbiguousDestination, None)
|
||||||
|
} else if let Some(candidate) = best {
|
||||||
|
(ResolutionStatus::Resolved, Some(candidate))
|
||||||
|
} else if counts.eligible == 0 {
|
||||||
|
(ResolutionStatus::NoEligibleDestination, None)
|
||||||
|
} else if counts.region_mismatch > 0 {
|
||||||
|
(ResolutionStatus::RegionMismatch, None)
|
||||||
|
} else {
|
||||||
|
(ResolutionStatus::NoActiveReview, None)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn result(
|
||||||
|
query: String,
|
||||||
|
status: ResolutionStatus,
|
||||||
|
destination: Option<Candidate>,
|
||||||
|
counts: ResolutionCounts,
|
||||||
|
) -> Resolution {
|
||||||
|
Resolution {
|
||||||
|
query,
|
||||||
|
status,
|
||||||
|
destination,
|
||||||
|
counts,
|
||||||
|
attribution: crate::release::attribution(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -6,9 +6,13 @@ use anyhow::ensure;
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use rusqlite::{Connection, params};
|
use rusqlite::{Connection, params};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
/// SSH signature namespace for exact operator decision JSON bytes.
|
||||||
|
pub const SIGNATURE_NAMESPACE: &str = "argand-site-registry-review";
|
||||||
|
|
||||||
/// Explicit review input, independent of imported source confidence.
|
/// Explicit review input, independent of imported source confidence.
|
||||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct Review {
|
pub struct Review {
|
||||||
/// Exact fingerprint printed by pinned lookup.
|
/// Exact fingerprint printed by pinned lookup.
|
||||||
|
|
@ -33,12 +37,95 @@ pub struct Review {
|
||||||
pub country: String,
|
pub country: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Authentication attached to one immutable review-log entry.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct Authentication {
|
||||||
|
/// Identity selected from the external allowed-reviewers file.
|
||||||
|
pub signer: String,
|
||||||
|
/// SHA-256 of the detached SSH signature bytes.
|
||||||
|
pub signature_sha256: String,
|
||||||
|
/// Domain-separated SSH signature namespace.
|
||||||
|
pub namespace: String,
|
||||||
|
/// SHA-256 of the exact signed decision JSON.
|
||||||
|
pub decision_sha256: String,
|
||||||
|
decision_json: Vec<u8>,
|
||||||
|
signature: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verifies exact decision bytes against an independently supplied reviewer key.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects oversized/non-regular inputs, malformed decisions, identity mismatch,
|
||||||
|
/// or an untrusted detached SSH signature.
|
||||||
|
pub fn authenticate(
|
||||||
|
decision: &Path,
|
||||||
|
signature: &Path,
|
||||||
|
allowed_reviewers: &Path,
|
||||||
|
identity: &str,
|
||||||
|
) -> anyhow::Result<(Review, Authentication)> {
|
||||||
|
ensure!(
|
||||||
|
!identity.trim().is_empty() && identity.len() <= 256,
|
||||||
|
"reviewer identity is required and bounded"
|
||||||
|
);
|
||||||
|
let decision = crate::ssh::sealed_input(decision, 1024 * 1024)?;
|
||||||
|
let signature = crate::ssh::sealed_input(signature, 64 * 1024)?;
|
||||||
|
let allowed_reviewers = crate::ssh::sealed_input(allowed_reviewers, 1024 * 1024)?;
|
||||||
|
let review: Review = serde_json::from_value(crate::json::parse(&decision.bytes)?)?;
|
||||||
|
ensure!(
|
||||||
|
review.reviewer == identity,
|
||||||
|
"reviewer must equal the authenticated identity"
|
||||||
|
);
|
||||||
|
crate::ssh::verify(
|
||||||
|
SIGNATURE_NAMESPACE,
|
||||||
|
&decision.bytes,
|
||||||
|
&signature.bytes,
|
||||||
|
&allowed_reviewers.bytes,
|
||||||
|
identity,
|
||||||
|
)?;
|
||||||
|
let signature_sha256 = crate::digest(&signature.bytes);
|
||||||
|
Ok((
|
||||||
|
review,
|
||||||
|
Authentication {
|
||||||
|
signer: identity.into(),
|
||||||
|
signature_sha256,
|
||||||
|
namespace: SIGNATURE_NAMESPACE.into(),
|
||||||
|
decision_sha256: crate::digest(&decision.bytes),
|
||||||
|
decision_json: decision.bytes,
|
||||||
|
signature: signature.bytes,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records a verified review and its authentication atomically.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns the same validation failures as [`record`] and rejects invalid proof.
|
||||||
|
pub fn record_authenticated(
|
||||||
|
db: &Connection,
|
||||||
|
registry: &Registry,
|
||||||
|
review: &Review,
|
||||||
|
authentication: &Authentication,
|
||||||
|
) -> anyhow::Result<u64> {
|
||||||
|
validate_authentication(review, authentication)?;
|
||||||
|
let transaction = db.unchecked_transaction()?;
|
||||||
|
validate_candidate(db, registry, review)?;
|
||||||
|
let sequence = append(db, review)?;
|
||||||
|
append_authentication(db, sequence, authentication)?;
|
||||||
|
transaction.commit()?;
|
||||||
|
Ok(sequence)
|
||||||
|
}
|
||||||
|
|
||||||
/// Appends a decision after verifying the exact generation and fingerprint.
|
/// Appends a decision after verifying the exact generation and fingerprint.
|
||||||
/// Rebuild and promote to distribute it; existing immutable artifacts never mutate.
|
/// Rebuild and promote to distribute it; existing immutable artifacts never mutate.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Rejects missing evidence, expired/overlong approval, and ineligible assertions.
|
/// Rejects missing evidence, expired/overlong approval, and ineligible assertions.
|
||||||
pub fn record(db: &Connection, registry: &Registry, review: &Review) -> anyhow::Result<u64> {
|
pub fn record(db: &Connection, registry: &Registry, review: &Review) -> anyhow::Result<u64> {
|
||||||
|
validate_candidate(db, registry, review)?;
|
||||||
|
append(db, review)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_candidate(db: &Connection, registry: &Registry, review: &Review) -> anyhow::Result<()> {
|
||||||
validate(review)?;
|
validate(review)?;
|
||||||
let candidate = registry.candidate(&review.fingerprint)?;
|
let candidate = registry.candidate(&review.fingerprint)?;
|
||||||
if review.decision == "approve" {
|
if review.decision == "approve" {
|
||||||
|
|
@ -57,7 +144,7 @@ pub fn record(db: &Connection, registry: &Registry, review: &Review) -> anyhow::
|
||||||
)?;
|
)?;
|
||||||
ensure!(exists, "review source is absent from this store");
|
ensure!(exists, "review source is absent from this store");
|
||||||
}
|
}
|
||||||
append(db, review)
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn validate(review: &Review) -> anyhow::Result<()> {
|
pub(crate) fn validate(review: &Review) -> anyhow::Result<()> {
|
||||||
|
|
@ -114,3 +201,117 @@ pub(crate) fn append(db: &Connection, review: &Review) -> anyhow::Result<u64> {
|
||||||
db.execute("INSERT INTO reviews(fingerprint,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",params![review.fingerprint,review.decision,review.reviewer,review.reason,review.evidence,review.reviewed_at.to_rfc3339(),review.expires_at.to_rfc3339(),review.role,review.locale,review.country])?;
|
db.execute("INSERT INTO reviews(fingerprint,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",params![review.fingerprint,review.decision,review.reviewer,review.reason,review.evidence,review.reviewed_at.to_rfc3339(),review.expires_at.to_rfc3339(),review.role,review.locale,review.country])?;
|
||||||
Ok(u64::try_from(db.last_insert_rowid())?)
|
Ok(u64::try_from(db.last_insert_rowid())?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn validate_authentication(
|
||||||
|
review: &Review,
|
||||||
|
authentication: &Authentication,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
validate(review)?;
|
||||||
|
ensure!(
|
||||||
|
authentication.decision_json.len() <= 1024 * 1024
|
||||||
|
&& authentication.signature.len() <= 64 * 1024,
|
||||||
|
"review authentication proof exceeds its bound"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
review.reviewer == authentication.signer
|
||||||
|
&& authentication.namespace == SIGNATURE_NAMESPACE
|
||||||
|
&& crate::model::valid_digest(&authentication.signature_sha256)
|
||||||
|
&& authentication.signature_sha256 == crate::digest(&authentication.signature)
|
||||||
|
&& crate::model::valid_digest(&authentication.decision_sha256)
|
||||||
|
&& authentication.decision_sha256 == crate::digest(&authentication.decision_json),
|
||||||
|
"review authentication does not match the decision"
|
||||||
|
);
|
||||||
|
let signed: Review =
|
||||||
|
serde_json::from_value(crate::json::parse(&authentication.decision_json)?)?;
|
||||||
|
ensure!(
|
||||||
|
&signed == review,
|
||||||
|
"signed decision differs from stored review"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn append_authentication(
|
||||||
|
db: &Connection,
|
||||||
|
sequence: u64,
|
||||||
|
authentication: &Authentication,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
db.execute(
|
||||||
|
"INSERT INTO review_auth VALUES(?1,?2,?3,?4,?5,?6,?7)",
|
||||||
|
params![
|
||||||
|
i64::try_from(sequence)?,
|
||||||
|
authentication.signer,
|
||||||
|
authentication.signature_sha256,
|
||||||
|
authentication.namespace,
|
||||||
|
authentication.decision_sha256,
|
||||||
|
authentication.decision_json,
|
||||||
|
authentication.signature
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Re-verifies every stored reviewer signature against external trust roots.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Rejects missing/altered proofs, review-row mismatches or untrusted signers.
|
||||||
|
pub fn verify_all(db: &Connection, allowed_reviewers: &Path) -> anyhow::Result<()> {
|
||||||
|
let allowed = crate::ssh::sealed_input(allowed_reviewers, 1024 * 1024)?;
|
||||||
|
verify_rows(db, Some(&allowed.bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn ensure_all_authenticated(db: &Connection) -> anyhow::Result<()> {
|
||||||
|
verify_rows(db, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_rows(db: &Connection, allowed_reviewers: Option<&[u8]>) -> anyhow::Result<()> {
|
||||||
|
let missing: u64 = db.query_row(
|
||||||
|
"SELECT count(*) FROM reviews r LEFT JOIN review_auth a USING(sequence) WHERE a.sequence IS NULL",
|
||||||
|
[],
|
||||||
|
|row| crate::store::unsigned(row, 0),
|
||||||
|
)?;
|
||||||
|
ensure!(
|
||||||
|
missing == 0,
|
||||||
|
"generation contains unauthenticated operator reviews"
|
||||||
|
);
|
||||||
|
let mut statement = db.prepare(
|
||||||
|
"SELECT r.fingerprint,r.decision,r.reviewer,r.reason,r.evidence,r.reviewed_at,r.expires_at,r.role,r.locale,r.country,a.signer,a.signature_sha256,a.namespace,a.decision_sha256,a.decision_json,a.signature FROM reviews r JOIN review_auth a USING(sequence) ORDER BY r.sequence",
|
||||||
|
)?;
|
||||||
|
let mut rows = statement.query([])?;
|
||||||
|
while let Some(row) = rows.next()? {
|
||||||
|
let authentication = Authentication {
|
||||||
|
signer: row.get(10)?,
|
||||||
|
signature_sha256: row.get(11)?,
|
||||||
|
namespace: row.get(12)?,
|
||||||
|
decision_sha256: row.get(13)?,
|
||||||
|
decision_json: row.get(14)?,
|
||||||
|
signature: row.get(15)?,
|
||||||
|
};
|
||||||
|
let review = Review {
|
||||||
|
fingerprint: row.get(0)?,
|
||||||
|
decision: row.get(1)?,
|
||||||
|
reviewer: row.get(2)?,
|
||||||
|
reason: row.get(3)?,
|
||||||
|
evidence: row.get(4)?,
|
||||||
|
reviewed_at: parse_time(&row.get::<_, String>(5)?)?,
|
||||||
|
expires_at: parse_time(&row.get::<_, String>(6)?)?,
|
||||||
|
role: row.get(7)?,
|
||||||
|
locale: row.get(8)?,
|
||||||
|
country: row.get(9)?,
|
||||||
|
};
|
||||||
|
validate_authentication(&review, &authentication)?;
|
||||||
|
if let Some(allowed) = allowed_reviewers {
|
||||||
|
crate::ssh::verify(
|
||||||
|
SIGNATURE_NAMESPACE,
|
||||||
|
&authentication.decision_json,
|
||||||
|
&authentication.signature,
|
||||||
|
allowed,
|
||||||
|
&authentication.signer,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_time(value: &str) -> anyhow::Result<DateTime<Utc>> {
|
||||||
|
Ok(DateTime::parse_from_rfc3339(value)?.with_timezone(&Utc))
|
||||||
|
}
|
||||||
|
|
|
||||||
64
crates/argand-site-registry/src/ssh.rs
Normal file
64
crates/argand-site-registry/src/ssh.rs
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
// By Nic Weyand!
|
||||||
|
//! Descriptor-bound inputs for external OpenSSH signature verification.
|
||||||
|
|
||||||
|
use anyhow::ensure;
|
||||||
|
use std::{
|
||||||
|
fs::File,
|
||||||
|
io::{Read, Seek, Write},
|
||||||
|
path::Path,
|
||||||
|
process::{Command, Stdio},
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(crate) struct SealedInput {
|
||||||
|
pub file: File,
|
||||||
|
pub bytes: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn sealed_input(path: &Path, maximum: usize) -> anyhow::Result<SealedInput> {
|
||||||
|
let mut file = crate::generation::open_no_follow(path)?;
|
||||||
|
ensure!(
|
||||||
|
file.metadata()?.is_file(),
|
||||||
|
"signature input must be a regular file"
|
||||||
|
);
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
Read::by_ref(&mut file)
|
||||||
|
.take(u64::try_from(maximum)? + 1)
|
||||||
|
.read_to_end(&mut bytes)?;
|
||||||
|
ensure!(bytes.len() <= maximum, "signature input exceeds its bound");
|
||||||
|
file.rewind()?;
|
||||||
|
Ok(SealedInput { file, bytes })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn verify(
|
||||||
|
namespace: &str,
|
||||||
|
message: &[u8],
|
||||||
|
signature: &[u8],
|
||||||
|
allowed_signers: &[u8],
|
||||||
|
identity: &str,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
// OpenSSH requires paths for its signature and trust file. Private,
|
||||||
|
// create-new temporary copies bind the command to the exact bytes already
|
||||||
|
// checked by this process instead of reopening attacker-controlled paths.
|
||||||
|
let mut message_file = tempfile::NamedTempFile::new()?;
|
||||||
|
message_file.write_all(message)?;
|
||||||
|
message_file.flush()?;
|
||||||
|
let mut signature_file = tempfile::NamedTempFile::new()?;
|
||||||
|
signature_file.write_all(signature)?;
|
||||||
|
signature_file.flush()?;
|
||||||
|
let mut allowed_file = tempfile::NamedTempFile::new()?;
|
||||||
|
allowed_file.write_all(allowed_signers)?;
|
||||||
|
allowed_file.flush()?;
|
||||||
|
|
||||||
|
let status = Command::new("ssh-keygen")
|
||||||
|
.args(["-Y", "verify", "-n", namespace, "-f"])
|
||||||
|
.arg(allowed_file.path())
|
||||||
|
.arg("-I")
|
||||||
|
.arg(identity)
|
||||||
|
.arg("-s")
|
||||||
|
.arg(signature_file.path())
|
||||||
|
.stdin(Stdio::from(message_file.reopen()?))
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.status()?;
|
||||||
|
ensure!(status.success(), "untrusted SSH signature");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -15,7 +15,13 @@ use std::{
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Adapter/normalization contract recorded in all generation identities.
|
/// Adapter/normalization contract recorded in all generation identities.
|
||||||
pub const RULE_VERSION: &str = "argand.site-rules/v1";
|
pub const RULE_VERSION: &str = "argand.site-rules/v2";
|
||||||
|
|
||||||
|
/// Whether a signed immutable generation uses a reader-compatible rule contract.
|
||||||
|
#[must_use]
|
||||||
|
pub fn supported_rule_version(version: &str) -> bool {
|
||||||
|
version == RULE_VERSION
|
||||||
|
}
|
||||||
|
|
||||||
/// Opens or migrates the local assertion store with bounded page cache.
|
/// Opens or migrates the local assertion store with bounded page cache.
|
||||||
///
|
///
|
||||||
|
|
@ -32,9 +38,15 @@ pub fn open(path: &Path) -> anyhow::Result<Connection> {
|
||||||
0 => {
|
0 => {
|
||||||
db.execute_batch("BEGIN IMMEDIATE")?;
|
db.execute_batch("BEGIN IMMEDIATE")?;
|
||||||
db.execute_batch(include_str!("../migrations/001.sql"))?;
|
db.execute_batch(include_str!("../migrations/001.sql"))?;
|
||||||
|
db.execute_batch(include_str!("../migrations/002.sql"))?;
|
||||||
db.execute_batch("COMMIT")?;
|
db.execute_batch("COMMIT")?;
|
||||||
}
|
}
|
||||||
1 => {}
|
1 => {
|
||||||
|
db.execute_batch("BEGIN IMMEDIATE")?;
|
||||||
|
db.execute_batch(include_str!("../migrations/002.sql"))?;
|
||||||
|
db.execute_batch("COMMIT")?;
|
||||||
|
}
|
||||||
|
2 => {}
|
||||||
_ => anyhow::bail!("unsupported registry schema {version}"),
|
_ => anyhow::bail!("unsupported registry schema {version}"),
|
||||||
}
|
}
|
||||||
let rules: String = db.query_row(
|
let rules: String = db.query_row(
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,7 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu
|
||||||
if configured.is_some() {
|
if configured.is_some() {
|
||||||
fs::create_dir(root)?;
|
fs::create_dir(root)?;
|
||||||
}
|
}
|
||||||
|
prepare_signer(root)?;
|
||||||
let database = root.join("store.sqlite");
|
let database = root.join("store.sqlite");
|
||||||
import_sources(root, &database)?;
|
import_sources(root, &database)?;
|
||||||
let candidate = root.join("candidate");
|
let candidate = root.join("candidate");
|
||||||
|
|
@ -58,33 +59,24 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu
|
||||||
lookup["candidates"][0]["web_property"]["domain"]["registrable_domain"],
|
lookup["candidates"][0]["web_property"]["domain"]["registrable_domain"],
|
||||||
"facebook.com"
|
"facebook.com"
|
||||||
);
|
);
|
||||||
assert!(
|
inspect_commands(&candidate, pin, &lookup)?;
|
||||||
run(&[
|
let unresolved = run(&[
|
||||||
"resolve",
|
"resolve",
|
||||||
"--generation",
|
"--generation",
|
||||||
text(&candidate)?,
|
text(&candidate)?,
|
||||||
"--pin",
|
"--pin",
|
||||||
pin,
|
pin,
|
||||||
"--query",
|
"--query",
|
||||||
"facebook"
|
"facebook",
|
||||||
])?["destination"]
|
])?;
|
||||||
.is_null()
|
assert!(unresolved["destination"].is_null());
|
||||||
);
|
assert_eq!(unresolved["status"], "no_active_review");
|
||||||
let now = chrono::Utc::now() - chrono::Duration::seconds(1);
|
let now = chrono::Utc::now() - chrono::Duration::seconds(1);
|
||||||
let decision = json!({"fingerprint":lookup["candidates"][0]["fingerprint"],"decision":"approve","reviewer":"synthetic fixture reviewer","reason":"E2E test only, not actual site verification","evidence":"synthetic:fixture","reviewed_at":now,"expires_at":now+chrono::Duration::days(1),"role":"primary","locale":"","country":""});
|
let decision = json!({"fingerprint":lookup["candidates"][0]["fingerprint"],"decision":"approve","reviewer":"fixture","reason":"E2E test only, not actual site verification","evidence":"synthetic:fixture","reviewed_at":now,"expires_at":now+chrono::Duration::days(1),"role":"primary","locale":"","country":""});
|
||||||
let decision_path = root.join("review.json");
|
let decision_path = root.join("review.json");
|
||||||
fs::write(&decision_path, serde_json::to_vec(&decision)?)?;
|
fs::write(&decision_path, serde_json::to_vec(&decision)?)?;
|
||||||
run(&[
|
reject_tampered_review(root, &database, &candidate, pin, &decision)?;
|
||||||
"review",
|
record_review(root, &database, &candidate, pin, &decision_path)?;
|
||||||
"--database",
|
|
||||||
text(&database)?,
|
|
||||||
"--generation",
|
|
||||||
text(&candidate)?,
|
|
||||||
"--pin",
|
|
||||||
pin,
|
|
||||||
"--decision",
|
|
||||||
text(&decision_path)?,
|
|
||||||
])?;
|
|
||||||
let approved = root.join("approved");
|
let approved = root.join("approved");
|
||||||
review_identity(root, &database, &candidate, pin, &decision)?;
|
review_identity(root, &database, &candidate, pin, &decision)?;
|
||||||
let built = run(&[
|
let built = run(&[
|
||||||
|
|
@ -107,6 +99,18 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu
|
||||||
])?["destination"]["url"],
|
])?["destination"]["url"],
|
||||||
"https://facebook.com/"
|
"https://facebook.com/"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
run(&[
|
||||||
|
"lookup",
|
||||||
|
"--generation",
|
||||||
|
text(&approved)?,
|
||||||
|
"--pin",
|
||||||
|
approved_pin,
|
||||||
|
"--query",
|
||||||
|
"FB",
|
||||||
|
])?["candidates"][0]["review"]["authentication"]["identity"],
|
||||||
|
"fixture"
|
||||||
|
);
|
||||||
let (revoked, revoked_pin) = release_lifecycle(
|
let (revoked, revoked_pin) = release_lifecycle(
|
||||||
root,
|
root,
|
||||||
&database,
|
&database,
|
||||||
|
|
@ -120,6 +124,129 @@ fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Resu
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn inspect_commands(generation: &Path, pin: &str, lookup: &Value) -> anyhow::Result<()> {
|
||||||
|
let entity_id = lookup["candidates"][0]["entity_id"]
|
||||||
|
.as_str()
|
||||||
|
.context("entity ID")?;
|
||||||
|
assert_eq!(
|
||||||
|
run(&[
|
||||||
|
"entity",
|
||||||
|
"--generation",
|
||||||
|
text(generation)?,
|
||||||
|
"--pin",
|
||||||
|
pin,
|
||||||
|
"--id",
|
||||||
|
entity_id,
|
||||||
|
])?["entity"]["canonical_name"],
|
||||||
|
"Facebook"
|
||||||
|
);
|
||||||
|
let reverse = run(&[
|
||||||
|
"lookup-web",
|
||||||
|
"--generation",
|
||||||
|
text(generation)?,
|
||||||
|
"--pin",
|
||||||
|
pin,
|
||||||
|
"--target",
|
||||||
|
"facebook.com",
|
||||||
|
])?;
|
||||||
|
assert_eq!(reverse["total_edges"], 2);
|
||||||
|
assert!(reverse["matches"].as_array().is_some_and(|matches| {
|
||||||
|
matches
|
||||||
|
.iter()
|
||||||
|
.any(|entry| entry["candidate"]["canonical_name"] == "Facebook")
|
||||||
|
}));
|
||||||
|
assert_eq!(
|
||||||
|
run(&[
|
||||||
|
"popularity",
|
||||||
|
"--generation",
|
||||||
|
text(generation)?,
|
||||||
|
"--pin",
|
||||||
|
pin,
|
||||||
|
"--target",
|
||||||
|
"facebook.com",
|
||||||
|
])?["total"],
|
||||||
|
2
|
||||||
|
);
|
||||||
|
let category = run(&[
|
||||||
|
"category",
|
||||||
|
"--generation",
|
||||||
|
text(generation)?,
|
||||||
|
"--pin",
|
||||||
|
pin,
|
||||||
|
"--id",
|
||||||
|
"42",
|
||||||
|
])?;
|
||||||
|
assert_eq!(category["total_members"], 1);
|
||||||
|
assert_eq!(
|
||||||
|
category["metadata"][0]["value"]["description_redacted"],
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
category["metadata"][0]["value"]
|
||||||
|
.get("description")
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
let stats = run(&["stats", "--generation", text(generation)?, "--pin", pin])?;
|
||||||
|
assert_eq!(stats["selected_sources"], 5);
|
||||||
|
assert_eq!(stats["authenticated_reviews"], 0);
|
||||||
|
let cases = generation
|
||||||
|
.parent()
|
||||||
|
.context("generation parent")?
|
||||||
|
.join("cli-evaluation.jsonl");
|
||||||
|
fs::write(
|
||||||
|
&cases,
|
||||||
|
b"{\"id\":\"missing\",\"query\":\"not a fixture entity\",\"expected_status\":\"no_name_match\"}\n",
|
||||||
|
)?;
|
||||||
|
let report = run(&[
|
||||||
|
"evaluate",
|
||||||
|
"--generation",
|
||||||
|
text(generation)?,
|
||||||
|
"--pin",
|
||||||
|
pin,
|
||||||
|
"--cases",
|
||||||
|
text(&cases)?,
|
||||||
|
])?;
|
||||||
|
assert_eq!(report["passed"], 1);
|
||||||
|
assert_eq!(report["failed"], 0);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reject_tampered_review(
|
||||||
|
root: &Path,
|
||||||
|
database: &Path,
|
||||||
|
generation: &Path,
|
||||||
|
pin: &str,
|
||||||
|
decision: &Value,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let path = root.join("tampered-review.json");
|
||||||
|
fs::write(&path, serde_json::to_vec(decision)?)?;
|
||||||
|
let signature = sign_review(root, &path)?;
|
||||||
|
let mut altered = decision.clone();
|
||||||
|
altered["reason"] = json!("changed after signature");
|
||||||
|
fs::write(&path, serde_json::to_vec(&altered)?)?;
|
||||||
|
assert!(
|
||||||
|
run(&[
|
||||||
|
"review",
|
||||||
|
"--database",
|
||||||
|
text(database)?,
|
||||||
|
"--generation",
|
||||||
|
text(generation)?,
|
||||||
|
"--pin",
|
||||||
|
pin,
|
||||||
|
"--decision",
|
||||||
|
text(&path)?,
|
||||||
|
"--signature",
|
||||||
|
text(&signature)?,
|
||||||
|
"--allowed-reviewers",
|
||||||
|
text(&root.join("allowed_signers"))?,
|
||||||
|
"--identity",
|
||||||
|
"fixture",
|
||||||
|
])
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn import_sources(root: &Path, database: &Path) -> anyhow::Result<()> {
|
fn import_sources(root: &Path, database: &Path) -> anyhow::Result<()> {
|
||||||
let sources = [
|
let sources = [
|
||||||
(
|
(
|
||||||
|
|
@ -174,31 +301,11 @@ fn release_lifecycle(
|
||||||
mut decision: Value,
|
mut decision: Value,
|
||||||
decision_path: &Path,
|
decision_path: &Path,
|
||||||
) -> anyhow::Result<(std::path::PathBuf, String)> {
|
) -> anyhow::Result<(std::path::PathBuf, String)> {
|
||||||
let key = root.join("signer");
|
|
||||||
let status = Command::new("ssh-keygen")
|
|
||||||
.args(["-q", "-t", "ed25519", "-N", "", "-f"])
|
|
||||||
.arg(&key)
|
|
||||||
.status()?;
|
|
||||||
ensure!(status.success(), "generate test key");
|
|
||||||
let allowed = root.join("allowed_signers");
|
|
||||||
fs::write(
|
|
||||||
&allowed,
|
|
||||||
format!("fixture {}", fs::read_to_string(key.with_extension("pub"))?),
|
|
||||||
)?;
|
|
||||||
sign_and_activate(root, approved, approved_pin)?;
|
sign_and_activate(root, approved, approved_pin)?;
|
||||||
decision["decision"] = json!("revoke");
|
decision["decision"] = json!("revoke");
|
||||||
fs::write(decision_path, serde_json::to_vec(&decision)?)?;
|
let revocation_path = decision_path.with_file_name("revocation.json");
|
||||||
run(&[
|
fs::write(&revocation_path, serde_json::to_vec(&decision)?)?;
|
||||||
"review",
|
record_review(root, database, approved, approved_pin, &revocation_path)?;
|
||||||
"--database",
|
|
||||||
text(database)?,
|
|
||||||
"--generation",
|
|
||||||
text(approved)?,
|
|
||||||
"--pin",
|
|
||||||
approved_pin,
|
|
||||||
"--decision",
|
|
||||||
text(decision_path)?,
|
|
||||||
])?;
|
|
||||||
let revoked = root.join("revoked");
|
let revoked = root.join("revoked");
|
||||||
let built = run(&[
|
let built = run(&[
|
||||||
"build",
|
"build",
|
||||||
|
|
@ -241,6 +348,15 @@ fn release_lifecycle(
|
||||||
fn sign_and_activate(root: &Path, approved: &Path, approved_pin: &str) -> anyhow::Result<()> {
|
fn sign_and_activate(root: &Path, approved: &Path, approved_pin: &str) -> anyhow::Result<()> {
|
||||||
let key = root.join("signer");
|
let key = root.join("signer");
|
||||||
let allowed = root.join("allowed_signers");
|
let allowed = root.join("allowed_signers");
|
||||||
|
let wrong_reviewers = root.join("wrong_reviewers");
|
||||||
|
fs::write(
|
||||||
|
&wrong_reviewers,
|
||||||
|
format!(
|
||||||
|
"untrusted {}",
|
||||||
|
fs::read_to_string(key.with_extension("pub"))?
|
||||||
|
),
|
||||||
|
)?;
|
||||||
|
assert!(
|
||||||
run(&[
|
run(&[
|
||||||
"sign",
|
"sign",
|
||||||
"--generation",
|
"--generation",
|
||||||
|
|
@ -249,6 +365,21 @@ fn sign_and_activate(root: &Path, approved: &Path, approved_pin: &str) -> anyhow
|
||||||
approved_pin,
|
approved_pin,
|
||||||
"--key",
|
"--key",
|
||||||
text(&key)?,
|
text(&key)?,
|
||||||
|
"--allowed-reviewers",
|
||||||
|
text(&wrong_reviewers)?,
|
||||||
|
])
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
run(&[
|
||||||
|
"sign",
|
||||||
|
"--generation",
|
||||||
|
text(approved)?,
|
||||||
|
"--pin",
|
||||||
|
approved_pin,
|
||||||
|
"--key",
|
||||||
|
text(&key)?,
|
||||||
|
"--allowed-reviewers",
|
||||||
|
text(&allowed)?,
|
||||||
])?;
|
])?;
|
||||||
let current = root.join("current.json");
|
let current = root.join("current.json");
|
||||||
assert!(
|
assert!(
|
||||||
|
|
@ -328,11 +459,86 @@ fn review_identity(
|
||||||
let path = root.join("identity-review.json");
|
let path = root.join("identity-review.json");
|
||||||
fs::write(&path, serde_json::to_vec(&decision)?)?;
|
fs::write(&path, serde_json::to_vec(&decision)?)?;
|
||||||
let mut args = args.to_vec();
|
let mut args = args.to_vec();
|
||||||
args.extend(["--database", text(database)?, "--decision", text(&path)?]);
|
let signature = sign_review(root, &path)?;
|
||||||
|
let allowed = root.join("allowed_signers");
|
||||||
|
args.extend([
|
||||||
|
"--database",
|
||||||
|
text(database)?,
|
||||||
|
"--decision",
|
||||||
|
text(&path)?,
|
||||||
|
"--signature",
|
||||||
|
text(&signature)?,
|
||||||
|
"--allowed-reviewers",
|
||||||
|
text(&allowed)?,
|
||||||
|
"--identity",
|
||||||
|
"fixture",
|
||||||
|
]);
|
||||||
run(&args)?;
|
run(&args)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn prepare_signer(root: &Path) -> anyhow::Result<()> {
|
||||||
|
let key = root.join("signer");
|
||||||
|
let status = Command::new("ssh-keygen")
|
||||||
|
.args(["-q", "-t", "ed25519", "-N", "", "-f"])
|
||||||
|
.arg(&key)
|
||||||
|
.status()?;
|
||||||
|
ensure!(status.success(), "generate test key");
|
||||||
|
fs::write(
|
||||||
|
root.join("allowed_signers"),
|
||||||
|
format!("fixture {}", fs::read_to_string(key.with_extension("pub"))?),
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sign_review(root: &Path, decision: &Path) -> anyhow::Result<std::path::PathBuf> {
|
||||||
|
let status = Command::new("ssh-keygen")
|
||||||
|
.args([
|
||||||
|
"-Y",
|
||||||
|
"sign",
|
||||||
|
"-n",
|
||||||
|
argand_site_registry::review::SIGNATURE_NAMESPACE,
|
||||||
|
"-f",
|
||||||
|
])
|
||||||
|
.arg(root.join("signer"))
|
||||||
|
.arg(decision)
|
||||||
|
.status()?;
|
||||||
|
ensure!(status.success(), "sign fixture review");
|
||||||
|
Ok(std::path::PathBuf::from(format!(
|
||||||
|
"{}.sig",
|
||||||
|
decision.display()
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_review(
|
||||||
|
root: &Path,
|
||||||
|
database: &Path,
|
||||||
|
generation: &Path,
|
||||||
|
pin: &str,
|
||||||
|
decision: &Path,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let signature = sign_review(root, decision)?;
|
||||||
|
let allowed = root.join("allowed_signers");
|
||||||
|
run(&[
|
||||||
|
"review",
|
||||||
|
"--database",
|
||||||
|
text(database)?,
|
||||||
|
"--generation",
|
||||||
|
text(generation)?,
|
||||||
|
"--pin",
|
||||||
|
pin,
|
||||||
|
"--decision",
|
||||||
|
text(decision)?,
|
||||||
|
"--signature",
|
||||||
|
text(&signature)?,
|
||||||
|
"--allowed-reviewers",
|
||||||
|
text(&allowed)?,
|
||||||
|
"--identity",
|
||||||
|
"fixture",
|
||||||
|
])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn export_fixture(root: &Path, revoked: &Path, revoked_pin: &str) -> anyhow::Result<()> {
|
fn export_fixture(root: &Path, revoked: &Path, revoked_pin: &str) -> anyhow::Result<()> {
|
||||||
let export = root.join("registry.jsonl");
|
let export = root.join("registry.jsonl");
|
||||||
run(&[
|
run(&[
|
||||||
|
|
|
||||||
105
crates/argand-site-registry/tests/evaluation.rs
Normal file
105
crates/argand-site-registry/tests/evaluation.rs
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
// By Nic Weyand!
|
||||||
|
//! Multilingual and adversarial resolver judgments over authored fixtures.
|
||||||
|
#[allow(dead_code)]
|
||||||
|
mod common;
|
||||||
|
|
||||||
|
use argand_site_registry::{ResolutionStatus, evaluation};
|
||||||
|
use serde_json::json;
|
||||||
|
use std::io::Cursor;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn multilingual_regional_and_deceptive_queries_are_replayed() -> anyhow::Result<()> {
|
||||||
|
let root = tempfile::tempdir()?;
|
||||||
|
let db = common::fixture(root.path())?;
|
||||||
|
let candidate = common::build(&db, root.path(), "candidate")?;
|
||||||
|
common::approve(
|
||||||
|
&db,
|
||||||
|
&candidate,
|
||||||
|
"FB",
|
||||||
|
"https://facebook.com/",
|
||||||
|
"primary",
|
||||||
|
"",
|
||||||
|
)?;
|
||||||
|
common::approve(
|
||||||
|
&db,
|
||||||
|
&candidate,
|
||||||
|
"Atlas",
|
||||||
|
"https://atlas.example.co.uk/",
|
||||||
|
"regional",
|
||||||
|
"GB",
|
||||||
|
)?;
|
||||||
|
common::approve(
|
||||||
|
&db,
|
||||||
|
&candidate,
|
||||||
|
"Atlas",
|
||||||
|
"https://atlas.example.de/",
|
||||||
|
"regional",
|
||||||
|
"DE",
|
||||||
|
)?;
|
||||||
|
let registry = common::build(&db, root.path(), "reviewed")?;
|
||||||
|
let facebook = registry.lookup("FB", 1)?.candidates.remove(0).entity_id;
|
||||||
|
let rows = [
|
||||||
|
json!({"id":"canonical","query":"Facebook","expected_status":"resolved","expected_entity_id":facebook,"expected_url":"https://facebook.com/"}),
|
||||||
|
json!({"id":"casefold-alias","query":"fb","expected_status":"resolved"}),
|
||||||
|
json!({"id":"combining-accent","query":"Cafe\u{301} Atlas","country":"GB","expected_status":"resolved","expected_url":"https://atlas.example.co.uk/"}),
|
||||||
|
json!({"id":"gb-region","query":"Atlas","country":"GB","expected_status":"resolved","expected_url":"https://atlas.example.co.uk/"}),
|
||||||
|
json!({"id":"de-region","query":"Atlas","country":"DE","expected_status":"resolved","expected_url":"https://atlas.example.de/"}),
|
||||||
|
json!({"id":"wrong-region","query":"Atlas","country":"US","expected_status":"region_mismatch"}),
|
||||||
|
json!({"id":"cyrillic-lookalike","query":"Fасebook","expected_status":"no_name_match"}),
|
||||||
|
json!({"id":"domain-is-not-an-identity","query":"facebook.com","expected_status":"no_name_match"}),
|
||||||
|
];
|
||||||
|
let input = rows
|
||||||
|
.iter()
|
||||||
|
.map(serde_json::to_string)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?
|
||||||
|
.join("\n");
|
||||||
|
let report = evaluation::run(®istry, Cursor::new(input), 100, common::timestamp()?)?;
|
||||||
|
assert_eq!(report.total, 8);
|
||||||
|
assert_eq!(report.passed, 8);
|
||||||
|
assert_eq!(report.failed, 0);
|
||||||
|
assert_eq!(report.evaluated_at, common::timestamp()?);
|
||||||
|
assert!(report.failures.is_empty());
|
||||||
|
|
||||||
|
let incorrect = serde_json::to_string(&json!({
|
||||||
|
"id":"wrong-judgment",
|
||||||
|
"query":"FB",
|
||||||
|
"expected_status":"no_name_match"
|
||||||
|
}))?;
|
||||||
|
let report = evaluation::run(®istry, Cursor::new(incorrect), 1, common::timestamp()?)?;
|
||||||
|
assert_eq!(report.failed, 1);
|
||||||
|
assert_eq!(report.failures[0].actual_status, ResolutionStatus::Resolved);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_duplicate_and_over_limit_cases_fail_closed() -> anyhow::Result<()> {
|
||||||
|
let root = tempfile::tempdir()?;
|
||||||
|
let db = common::fixture(root.path())?;
|
||||||
|
let registry = common::build(&db, root.path(), "generation")?;
|
||||||
|
let duplicate_key =
|
||||||
|
r#"{"id":"one","id":"two","query":"FB","expected_status":"no_active_review"}"#;
|
||||||
|
assert!(
|
||||||
|
evaluation::run(
|
||||||
|
®istry,
|
||||||
|
Cursor::new(duplicate_key),
|
||||||
|
1,
|
||||||
|
common::timestamp()?
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let duplicate_id = r#"{"id":"one","query":"FB","expected_status":"no_active_review"}
|
||||||
|
{"id":"one","query":"Atlas","expected_status":"no_active_review"}"#;
|
||||||
|
assert!(
|
||||||
|
evaluation::run(
|
||||||
|
®istry,
|
||||||
|
Cursor::new(duplicate_id),
|
||||||
|
2,
|
||||||
|
common::timestamp()?
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let two = r#"{"id":"one","query":"FB","expected_status":"no_active_review"}
|
||||||
|
{"id":"two","query":"Atlas","expected_status":"no_active_review"}"#;
|
||||||
|
assert!(evaluation::run(®istry, Cursor::new(two), 1, common::timestamp()?).is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -9,7 +9,119 @@ use argand_site_registry::{
|
||||||
store,
|
store,
|
||||||
};
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use std::{fs, io::Write};
|
use std::{fs, io::Write, process::Command};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn version_one_store_migrates_without_losing_review_history() -> anyhow::Result<()> {
|
||||||
|
let root = tempfile::tempdir()?;
|
||||||
|
let path = root.path().join("v1.sqlite");
|
||||||
|
let db = rusqlite::Connection::open(&path)?;
|
||||||
|
db.execute_batch(include_str!("../migrations/001.sql"))?;
|
||||||
|
db.execute("INSERT INTO reviews VALUES(1,?1,'revoke','legacy','reason','evidence','2026-01-01T00:00:00Z','2026-01-01T00:00:00Z','unspecified','','')", ["0".repeat(64)])?;
|
||||||
|
drop(db);
|
||||||
|
|
||||||
|
let migrated = store::open(&path)?;
|
||||||
|
assert_eq!(
|
||||||
|
migrated.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
|
||||||
|
2
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
migrated.query_row("SELECT rules FROM registry_metadata", [], |row| row
|
||||||
|
.get::<_, String>(0))?,
|
||||||
|
store::RULE_VERSION
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
migrated.query_row("SELECT count(*) FROM reviews", [], |row| row
|
||||||
|
.get::<_, i64>(0))?,
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
migrated.query_row("SELECT count(*) FROM review_auth", [], |row| row
|
||||||
|
.get::<_, i64>(0))?,
|
||||||
|
0
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn externally_signed_unauthenticated_reviews_are_rejected() -> anyhow::Result<()> {
|
||||||
|
let root = tempfile::tempdir()?;
|
||||||
|
let db = common::fixture(root.path())?;
|
||||||
|
let initial = common::build(&db, root.path(), "initial")?;
|
||||||
|
common::approve(&db, &initial, "FB", "https://facebook.com/", "primary", "")?;
|
||||||
|
let generation = common::build(&db, root.path(), "generation")?;
|
||||||
|
let key = root.path().join("key");
|
||||||
|
assert!(
|
||||||
|
Command::new("ssh-keygen")
|
||||||
|
.args(["-q", "-t", "ed25519", "-N", "", "-f"])
|
||||||
|
.arg(&key)
|
||||||
|
.status()?
|
||||||
|
.success()
|
||||||
|
);
|
||||||
|
fs::write(
|
||||||
|
root.path().join("allowed"),
|
||||||
|
format!("fixture {}", fs::read_to_string(key.with_extension("pub"))?),
|
||||||
|
)?;
|
||||||
|
assert!(
|
||||||
|
argand_site_registry::release::sign(
|
||||||
|
&root.path().join("generation"),
|
||||||
|
&key,
|
||||||
|
&generation.identity,
|
||||||
|
&root.path().join("allowed")
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
Command::new("ssh-keygen")
|
||||||
|
.args(["-Y", "sign", "-n", "argand-site-registry", "-f"])
|
||||||
|
.arg(&key)
|
||||||
|
.arg(root.path().join("generation/COMPLETE.json"))
|
||||||
|
.status()?
|
||||||
|
.success()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
argand_site_registry::release::verify_signed(
|
||||||
|
&root.path().join("generation"),
|
||||||
|
&root.path().join("allowed"),
|
||||||
|
"fixture"
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unsigned_sqlite_sidecars_and_generation_symlinks_are_rejected() -> anyhow::Result<()> {
|
||||||
|
let root = tempfile::tempdir()?;
|
||||||
|
let db = common::fixture(root.path())?;
|
||||||
|
let generation = common::build(&db, root.path(), "sealed")?;
|
||||||
|
let scratch = root.path().join("scratch.sqlite");
|
||||||
|
fs::copy(root.path().join("sealed/registry.sqlite"), &scratch)?;
|
||||||
|
let attacker = rusqlite::Connection::open(&scratch)?;
|
||||||
|
attacker.execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;")?;
|
||||||
|
attacker.execute(
|
||||||
|
"UPDATE properties SET url='https://unsigned.example.org/' WHERE url='https://facebook.com/'",
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
fs::copy(
|
||||||
|
scratch.with_extension("sqlite-wal"),
|
||||||
|
root.path().join("sealed/registry.sqlite-wal"),
|
||||||
|
)?;
|
||||||
|
let error = Registry::open(&root.path().join("sealed"), &generation.identity)
|
||||||
|
.err()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("unsigned sidecar was accepted"))?;
|
||||||
|
assert!(error.to_string().contains("unexpected generation entry"));
|
||||||
|
drop(attacker);
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
let link = root.path().join("generation-link");
|
||||||
|
symlink(root.path().join("sealed"), &link)?;
|
||||||
|
assert!(Registry::open(&link, &generation.identity).is_err());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn removed_entity_websites_retire_the_previous_selection() -> anyhow::Result<()> {
|
fn removed_entity_websites_retire_the_previous_selection() -> anyhow::Result<()> {
|
||||||
|
|
@ -70,8 +182,9 @@ fn import_order_does_not_change_generation_and_psl_refresh_preserves_review() ->
|
||||||
assert_eq!(first.identity, second.identity);
|
assert_eq!(first.identity, second.identity);
|
||||||
common::approve(&b, &second, "FB", "https://facebook.com/", "primary", "")?;
|
common::approve(&b, &second, "FB", "https://facebook.com/", "primary", "")?;
|
||||||
let input = root.path().join("psl");
|
let input = root.path().join("psl");
|
||||||
fs::write(&input, common::PSL)?;
|
let comment_only = format!("{}// Comment-only refresh\n", common::PSL);
|
||||||
let mut manifest = common::manifest(Source::Psl, Format::PslText, common::PSL.as_bytes())?;
|
fs::write(&input, &comment_only)?;
|
||||||
|
let mut manifest = common::manifest(Source::Psl, Format::PslText, comment_only.as_bytes())?;
|
||||||
manifest.retrieved_at += chrono::Duration::days(1);
|
manifest.retrieved_at += chrono::Duration::days(1);
|
||||||
store::import(&mut b, &manifest, &input)?;
|
store::import(&mut b, &manifest, &input)?;
|
||||||
let refreshed = common::build(&b, root.path(), "refresh")?;
|
let refreshed = common::build(&b, root.path(), "refresh")?;
|
||||||
|
|
@ -80,6 +193,77 @@ fn import_order_does_not_change_generation_and_psl_refresh_preserves_review() ->
|
||||||
.resolve("FB", None, None, common::timestamp()?)?
|
.resolve("FB", None, None, common::timestamp()?)?
|
||||||
.is_some()
|
.is_some()
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
second.lookup("FB", 1)?.candidates[0].fingerprint,
|
||||||
|
refreshed.lookup("FB", 1)?.candidates[0].fingerprint
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn psl_semantic_change_invalidates_affected_review_only() -> anyhow::Result<()> {
|
||||||
|
let root = tempfile::tempdir()?;
|
||||||
|
let mut db = common::fixture(root.path())?;
|
||||||
|
let original = common::build(&db, root.path(), "original")?;
|
||||||
|
common::approve(
|
||||||
|
&db,
|
||||||
|
&original,
|
||||||
|
"Atlas",
|
||||||
|
"https://atlas.example.co.uk/",
|
||||||
|
"regional",
|
||||||
|
"GB",
|
||||||
|
)?;
|
||||||
|
let original_fingerprint = original
|
||||||
|
.lookup("Atlas", 10)?
|
||||||
|
.candidates
|
||||||
|
.into_iter()
|
||||||
|
.find(|candidate| candidate.url == "https://atlas.example.co.uk/")
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("fixture regional property missing"))?
|
||||||
|
.fingerprint;
|
||||||
|
|
||||||
|
let changed = common::PSL.replace("co.uk\n", "");
|
||||||
|
let input = root.path().join("changed-psl");
|
||||||
|
fs::write(&input, &changed)?;
|
||||||
|
let mut manifest = common::manifest(Source::Psl, Format::PslText, changed.as_bytes())?;
|
||||||
|
manifest.retrieved_at += chrono::Duration::days(1);
|
||||||
|
store::import(&mut db, &manifest, &input)?;
|
||||||
|
let rebuilt = common::build(&db, root.path(), "changed")?;
|
||||||
|
let changed_candidate = rebuilt
|
||||||
|
.lookup("Atlas", 10)?
|
||||||
|
.candidates
|
||||||
|
.into_iter()
|
||||||
|
.find(|candidate| candidate.url == "https://atlas.example.co.uk/")
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("changed property missing"))?;
|
||||||
|
assert_ne!(original_fingerprint, changed_candidate.fingerprint);
|
||||||
|
assert_eq!(
|
||||||
|
changed_candidate.web_property["domain"]["registrable_domain"],
|
||||||
|
"co.uk"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
rebuilt
|
||||||
|
.resolve("Atlas", None, Some("GB"), common::timestamp()?)?
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn typed_diff_reports_review_only_changes() -> anyhow::Result<()> {
|
||||||
|
let root = tempfile::tempdir()?;
|
||||||
|
let db = common::fixture(root.path())?;
|
||||||
|
let before = common::build(&db, root.path(), "before")?;
|
||||||
|
common::approve(&db, &before, "FB", "https://facebook.com/", "primary", "")?;
|
||||||
|
let after = common::build(&db, root.path(), "after")?;
|
||||||
|
let mut output = Vec::new();
|
||||||
|
argand_site_registry::release::diff(&before, &after, &mut output)?;
|
||||||
|
let rows = String::from_utf8(output)?
|
||||||
|
.lines()
|
||||||
|
.map(serde_json::from_str::<serde_json::Value>)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
assert!(rows.iter().any(|row| {
|
||||||
|
row["type"] == "change" && row["subject"] == "review" && row["change"] == "added"
|
||||||
|
}));
|
||||||
|
assert_eq!(rows.last().and_then(|row| row["changes"].as_u64()), Some(1));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
60
crates/argand-site-registry/tests/observation.rs
Normal file
60
crates/argand-site-registry/tests/observation.rs
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
// By Nic Weyand!
|
||||||
|
//! Future crawler evidence remains normalized, source-bound and non-authoritative.
|
||||||
|
#[allow(dead_code)]
|
||||||
|
mod common;
|
||||||
|
|
||||||
|
use argand_site_registry::{
|
||||||
|
normalize::Normalizer,
|
||||||
|
observation::{Observation, ObservationKind},
|
||||||
|
};
|
||||||
|
|
||||||
|
fn fixture(relation: ObservationKind) -> anyhow::Result<Observation> {
|
||||||
|
Ok(Observation {
|
||||||
|
relation,
|
||||||
|
from_url: "HTTPS://Example.COM:443/start/../region".into(),
|
||||||
|
to_url: "https://example.co.uk/shop".into(),
|
||||||
|
source: "synthetic-crawler".into(),
|
||||||
|
source_identifier: "capture:1".into(),
|
||||||
|
license: "CC0-1.0".into(),
|
||||||
|
license_url: "https://creativecommons.org/publicdomain/zero/1.0/".into(),
|
||||||
|
retrieved_at: common::timestamp()?,
|
||||||
|
content_sha256: "a".repeat(64),
|
||||||
|
selector: "HTTP Location header".into(),
|
||||||
|
confidence: 9000,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn observation_normalization_is_deterministic_and_retains_rights() -> anyhow::Result<()> {
|
||||||
|
let normalizer = Normalizer::new(common::PSL.as_bytes(), "fixture-psl".into())?;
|
||||||
|
let observation = fixture(ObservationKind::Redirect { status: 308 })?;
|
||||||
|
let first = observation.normalize(&normalizer)?;
|
||||||
|
let second = observation.normalize(&normalizer)?;
|
||||||
|
assert_eq!(first.fingerprint, second.fingerprint);
|
||||||
|
assert_eq!(first.from.url, "https://example.com/region");
|
||||||
|
assert_eq!(first.to.domain.registrable_domain, "example.co.uk");
|
||||||
|
assert_eq!(first.source_identifier, "capture:1");
|
||||||
|
assert_eq!(first.license, "CC0-1.0");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn observation_validation_rejects_false_redirects_and_bad_scopes() -> anyhow::Result<()> {
|
||||||
|
let normalizer = Normalizer::new(common::PSL.as_bytes(), "fixture-psl".into())?;
|
||||||
|
assert!(
|
||||||
|
fixture(ObservationKind::Redirect { status: 200 })?
|
||||||
|
.normalize(&normalizer)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture(ObservationKind::CountrySelector {
|
||||||
|
country: "usa".into()
|
||||||
|
})?
|
||||||
|
.normalize(&normalizer)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let mut bad_rights = fixture(ObservationKind::Canonical)?;
|
||||||
|
bad_rights.license_url = "file:///tmp/claim".into();
|
||||||
|
assert!(bad_rights.normalize(&normalizer).is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -5,7 +5,7 @@ use anyhow::{Context, ensure};
|
||||||
use argand_site_registry::{
|
use argand_site_registry::{
|
||||||
model::{Compression, Format, Source},
|
model::{Compression, Format, Source},
|
||||||
normalize::{Normalizer, name_key},
|
normalize::{Normalizer, name_key},
|
||||||
query::Registry,
|
query::{Registry, ResolutionStatus},
|
||||||
review::{self, Review},
|
review::{self, Review},
|
||||||
store,
|
store,
|
||||||
};
|
};
|
||||||
|
|
@ -94,6 +94,35 @@ fn aliases_deduplication_provenance_and_separate_popularity() -> anyhow::Result<
|
||||||
assert_eq!(r.receipt.properties, 4); // Curlie Facebook shares the same property.
|
assert_eq!(r.receipt.properties, 4); // Curlie Facebook shares the same property.
|
||||||
assert_eq!(r.receipt.entities, 3); // Curlie is not automatically the Wikidata entity.
|
assert_eq!(r.receipt.entities, 3); // Curlie is not automatically the Wikidata entity.
|
||||||
assert!(r.resolve("Facebook", None, None, timestamp()?)?.is_none());
|
assert!(r.resolve("Facebook", None, None, timestamp()?)?.is_none());
|
||||||
|
let unresolved = r.resolve_explained("Facebook", None, None, timestamp()?)?;
|
||||||
|
assert_eq!(unresolved.status, ResolutionStatus::NoActiveReview);
|
||||||
|
assert_eq!(unresolved.counts.missing_review, 1);
|
||||||
|
let reverse = r.lookup_web("facebook.com", 10)?;
|
||||||
|
assert_eq!(reverse.total_properties, 1);
|
||||||
|
assert_eq!(reverse.total_edges, 2);
|
||||||
|
assert!(
|
||||||
|
reverse
|
||||||
|
.matches
|
||||||
|
.iter()
|
||||||
|
.any(|item| item.candidate.canonical_name == "Facebook")
|
||||||
|
);
|
||||||
|
let popularity = r.popularity("https://facebook.com/", 10)?;
|
||||||
|
assert_eq!(popularity.total, 2);
|
||||||
|
assert_eq!(
|
||||||
|
popularity
|
||||||
|
.observations
|
||||||
|
.iter()
|
||||||
|
.map(|item| item.source.as_str())
|
||||||
|
.collect::<std::collections::BTreeSet<_>>(),
|
||||||
|
std::collections::BTreeSet::from(["crux", "majestic"])
|
||||||
|
);
|
||||||
|
let entity = r
|
||||||
|
.entity_by_id(&c.entity_id, 10)?
|
||||||
|
.context("entity lookup missing")?;
|
||||||
|
assert_eq!(entity.entity.canonical_name, "Facebook");
|
||||||
|
let category = r.category("42", 10)?;
|
||||||
|
assert_eq!(category.total_members, 1);
|
||||||
|
assert!(!serde_json::to_string(&category)?.contains("Synthetic category description"));
|
||||||
let export = dir.path().join("export.jsonl");
|
let export = dir.path().join("export.jsonl");
|
||||||
argand_site_registry::release::export(&r, &export, false)?;
|
argand_site_registry::release::export(&r, &export, false)?;
|
||||||
let text = std::fs::read_to_string(&export)?;
|
let text = std::fs::read_to_string(&export)?;
|
||||||
|
|
@ -150,6 +179,17 @@ fn regional_review_expiry_and_revocation() -> anyhow::Result<()> {
|
||||||
)?
|
)?
|
||||||
.is_none()
|
.is_none()
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
approved
|
||||||
|
.resolve_explained(
|
||||||
|
"Atlas",
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
timestamp()? + chrono::Duration::days(8)
|
||||||
|
)?
|
||||||
|
.status,
|
||||||
|
ResolutionStatus::NoActiveReview
|
||||||
|
);
|
||||||
let fingerprint = approved
|
let fingerprint = approved
|
||||||
.lookup("Atlas", 10)?
|
.lookup("Atlas", 10)?
|
||||||
.candidates
|
.candidates
|
||||||
|
|
@ -201,6 +241,16 @@ fn ambiguity_survives_limits_and_same_named_domains_do_not_merge() -> anyhow::Re
|
||||||
assert_eq!(result.total_edges, 4);
|
assert_eq!(result.total_edges, 4);
|
||||||
assert!(result.truncated);
|
assert!(result.truncated);
|
||||||
assert!(r.resolve("Atlas", None, None, timestamp()?)?.is_none());
|
assert!(r.resolve("Atlas", None, None, timestamp()?)?.is_none());
|
||||||
|
assert_eq!(
|
||||||
|
r.resolve_explained("Atlas", None, None, timestamp()?)?
|
||||||
|
.status,
|
||||||
|
ResolutionStatus::AmbiguousIdentity
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
r.resolve_explained("absent", None, None, timestamp()?)?
|
||||||
|
.status,
|
||||||
|
ResolutionStatus::NoNameMatch
|
||||||
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,10 +4,11 @@
|
||||||
|
|
||||||
Use `argand_site_registry::query::Registry::open(generation, trusted_pin)` once per
|
Use `argand_site_registry::query::Registry::open(generation, trusted_pin)` once per
|
||||||
immutable generation and reuse the reader. `lookup(query, limit)` returns evidence
|
immutable generation and reuse the reader. `lookup(query, limit)` returns evidence
|
||||||
and complete ambiguity counts; `resolve(query, locale, country, now)` returns an
|
and complete ambiguity counts; `resolve_explained` returns a reviewed candidate or
|
||||||
optional reviewed candidate. Check the compiled example and API docs for exact
|
a typed abstention reason with counts. Exact reverse views cover entity IDs,
|
||||||
types. `selection_context` binds the full alternative set for downstream query
|
URLs/domains, popularity and Curlie categories. Check the compiled example and API
|
||||||
review. Preserve the returned provenance, scopes, counts and attribution.
|
docs for exact types. `selection_context` binds the full alternative set for
|
||||||
|
downstream query review. Preserve returned provenance, scopes, counts and attribution.
|
||||||
|
|
||||||
For local integration, point a Cargo dependency at
|
For local integration, point a Cargo dependency at
|
||||||
`crates/argand-site-registry` inside an extracted standalone source tree. Once an
|
`crates/argand-site-registry` inside an extracted standalone source tree. Once an
|
||||||
|
|
@ -17,7 +18,8 @@ Both crates remain in this workspace; the atomic helper is a relative dependency
|
||||||
|
|
||||||
## CLI and other languages
|
## CLI and other languages
|
||||||
|
|
||||||
`lookup`, `resolve`, `verify` and the other commands emit JSON. The Python example
|
`lookup`, `resolve`, `entity`, `lookup-web`, `popularity`, `category`, `stats`,
|
||||||
|
`evaluate`, `verify` and the other commands emit JSON. The Python example
|
||||||
passes arguments directly to the native executable, preserving query text and the
|
passes arguments directly to the native executable, preserving query text and the
|
||||||
entire response. A nonzero exit is an error. `destination: null` is a successful
|
entire response. A nonzero exit is an error. `destination: null` is a successful
|
||||||
abstention, not a request to pick the first lookup candidate. Render source names
|
abstention, not a request to pick the first lookup candidate. Render source names
|
||||||
|
|
@ -37,8 +39,10 @@ omits descriptions and includes fact provenance plus an attribution envelope.
|
||||||
It is an assertion export, not a self-contained signed list of admitted routes.
|
It is an assertion export, not a self-contained signed list of admitted routes.
|
||||||
Raw SQL inspection is useful for audit; it does not implement resolution policy.
|
Raw SQL inspection is useful for audit; it does not implement resolution policy.
|
||||||
|
|
||||||
Code version 0.1.0 is an initial interface. Schema/rule contracts are versioned
|
Code version 0.2.0 uses schema version 2 and `argand.site-rules/v2`.
|
||||||
independently in receipts. Unsupported contracts fail closed. Pin source releases,
|
It adds authenticated reviewer proofs, exact immutable SQLite opening, typed diffs
|
||||||
|
and audit/evaluation APIs. Schema/rule contracts remain versioned independently in
|
||||||
|
receipts. Unsupported contracts fail closed. Pin source releases,
|
||||||
compile consumers and replay fixed fixtures before upgrades. Preserve import and
|
compile consumers and replay fixed fixtures before upgrades. Preserve import and
|
||||||
review history; never mutate complete generations to migrate them.
|
review history; never mutate complete generations to migrate them.
|
||||||
|
|
||||||
|
|
|
||||||
35
docs/EVALUATION.md
Normal file
35
docs/EVALUATION.md
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
# Resolver evaluation
|
||||||
|
|
||||||
|
`evaluate` replays authored judgments through the same native resolver used by
|
||||||
|
consumers. It opens one externally pinned immutable generation, streams up to the
|
||||||
|
configured case limit and reports correctness plus native p50/p95 query latency.
|
||||||
|
Pass an explicit `--at` time when the report must be replayable across approval
|
||||||
|
expiry boundaries; the chosen clock is included in the report.
|
||||||
|
The input is bounded to 16 MiB, each line to 64 KiB and case IDs must be unique.
|
||||||
|
|
||||||
|
Each nonempty JSONL line has this form:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"id":"facebook-primary","query":"facebook","locale":null,"country":null,"expected_status":"resolved","expected_entity_id":"argand:entity:SOURCE:ID","expected_url":"https://www.facebook.com/"}
|
||||||
|
```
|
||||||
|
|
||||||
|
`locale`, `country`, `expected_entity_id` and `expected_url` are optional. Status
|
||||||
|
is required and is one of `resolved`, `no_name_match`, `ambiguous_identity`,
|
||||||
|
`safety_limit_exceeded`, `no_eligible_destination`, `no_active_review`,
|
||||||
|
`region_mismatch` or `ambiguous_destination`. Expected URLs must be copied from a
|
||||||
|
pinned registry, including normalization such as a trailing slash.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
argand-site-registry evaluate --generation /data/registry/reviewed \
|
||||||
|
--pin "$REVIEWED_PIN" --cases /data/evaluation/navigation.jsonl \
|
||||||
|
--maximum-cases 10000 --at 2026-09-13T00:00:00Z \
|
||||||
|
> /data/evaluation/report.json
|
||||||
|
jq -e '.failed == 0 and .passed == .total' /data/evaluation/report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Keep the corpus version and digest with the report. Include canonical names,
|
||||||
|
aliases, Unicode normalization, multiple scripts, every served country/locale,
|
||||||
|
unknown names, deceptive lookalikes, ambiguous entities, expired/revoked reviews
|
||||||
|
and tied destinations. Use synthetic or authorized query material; do not commit
|
||||||
|
private user logs. Latency values are process and hardware measurements. Compare
|
||||||
|
them only with an equivalent environment and sufficient sample size.
|
||||||
|
|
@ -6,6 +6,8 @@
|
||||||
- [Source licenses](../crates/argand-site-registry/LICENSE_SOURCES.md): exact terms and attribution.
|
- [Source licenses](../crates/argand-site-registry/LICENSE_SOURCES.md): exact terms and attribution.
|
||||||
- [Consumers](CONSUMERS.md): Rust, Python/CLI, data distribution and Argand transition.
|
- [Consumers](CONSUMERS.md): Rust, Python/CLI, data distribution and Argand transition.
|
||||||
- [Trust](TRUST.md): enforced checks and publisher/consumer responsibilities.
|
- [Trust](TRUST.md): enforced checks and publisher/consumer responsibilities.
|
||||||
|
- [Publishing](PUBLISHING.md): reviewer keys, candidate acceptance and activation.
|
||||||
|
- [Evaluation](EVALUATION.md): bounded JSONL judgments and result interpretation.
|
||||||
- [Releasing](RELEASING.md): CI, source signing and archive verification.
|
- [Releasing](RELEASING.md): CI, source signing and archive verification.
|
||||||
- [Validation](VALIDATION.md): independent builds and native acceptance evidence.
|
- [Validation](VALIDATION.md): independent builds and native acceptance evidence.
|
||||||
- [Contributing](../CONTRIBUTING.md), [governance](../GOVERNANCE.md),
|
- [Contributing](../CONTRIBUTING.md), [governance](../GOVERNANCE.md),
|
||||||
|
|
|
||||||
75
docs/PUBLISHING.md
Normal file
75
docs/PUBLISHING.md
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
# Dataset publisher runbook
|
||||||
|
|
||||||
|
This runbook creates a reviewable candidate and a signed activation. Source update,
|
||||||
|
review and release keys are separate authorities. Scheduled jobs may acquire,
|
||||||
|
import and build candidates; they do not approve, sign or activate destinations.
|
||||||
|
|
||||||
|
## Trust roots and local state
|
||||||
|
|
||||||
|
Keep cache, mutable store, immutable generations, reviewer keys, release keys and
|
||||||
|
consumer trust files outside the checkout. An OpenSSH reviewer trust file contains
|
||||||
|
one accepted principal and public key per line:
|
||||||
|
|
||||||
|
```text
|
||||||
|
operator@example.org ssh-ed25519 REVIEWER_PUBLIC_KEY
|
||||||
|
```
|
||||||
|
|
||||||
|
Distribute the release publisher public key to consumers through an independent
|
||||||
|
authenticated channel. Do not put private keys, production trust files or source
|
||||||
|
datasets in Git or CI. The isolated source CI runner has none of these files.
|
||||||
|
|
||||||
|
## Candidate acceptance
|
||||||
|
|
||||||
|
Run `update` or the explicit download/import/build commands from the operator guide.
|
||||||
|
For every candidate generation:
|
||||||
|
|
||||||
|
1. Verify its externally recorded receipt pin with `verify`.
|
||||||
|
2. Run `stats` and compare source snapshots, selected sources, facts, rejections,
|
||||||
|
reviews, database bytes and upcoming expiry with the prior accepted generation.
|
||||||
|
3. Run `diff` against the prior pin. Investigate every source-selection, identity,
|
||||||
|
name, property, edge, popularity, review and equivalence change.
|
||||||
|
4. Use `lookup`, `entity`, `lookup-web`, `popularity` and `category` to inspect exact
|
||||||
|
source evidence and conflicts. Popularity never proves ownership.
|
||||||
|
5. Replay the maintained evaluation corpus with `evaluate`; require zero judgment
|
||||||
|
mismatches and compare latency with a documented hardware/process baseline.
|
||||||
|
|
||||||
|
## Authenticated decisions
|
||||||
|
|
||||||
|
Create a bounded review JSON from the exact candidate fingerprint and evidence.
|
||||||
|
Sign its exact bytes and append it through the CLI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh-keygen -Y sign -n argand-site-registry-review \
|
||||||
|
-f /secure/reviewer-key review.json
|
||||||
|
argand-site-registry review --database /data/registry/import.sqlite \
|
||||||
|
--generation /data/registry/candidate --pin "$CANDIDATE_PIN" \
|
||||||
|
--decision review.json --signature review.json.sig \
|
||||||
|
--allowed-reviewers /secure/reviewer-allowed-signers \
|
||||||
|
--identity operator@example.org
|
||||||
|
```
|
||||||
|
|
||||||
|
Identity equivalence decisions use the same signed JSON and reviewer namespace.
|
||||||
|
Rebuild after appending decisions, then repeat the complete diff and evaluation.
|
||||||
|
The release command re-verifies every retained reviewer signature against the
|
||||||
|
current reviewer trust file. Missing, altered or no-longer-trusted proofs stop it.
|
||||||
|
|
||||||
|
## Sign and activate
|
||||||
|
|
||||||
|
```bash
|
||||||
|
argand-site-registry sign --generation /data/registry/reviewed \
|
||||||
|
--pin "$REVIEWED_PIN" --key /secure/release-key \
|
||||||
|
--allowed-reviewers /secure/reviewer-allowed-signers
|
||||||
|
argand-site-registry activate --generation /data/registry/reviewed \
|
||||||
|
--current /data/registry/current.json \
|
||||||
|
--allowed-signers /secure/release-allowed-signers \
|
||||||
|
--identity registry-publisher
|
||||||
|
```
|
||||||
|
|
||||||
|
Record the source commit, candidate and accepted receipt pins, typed diff, evaluation
|
||||||
|
report, reviewer trust-file digest, release signer identity and activation receipt
|
||||||
|
in an immutable operator log. Activation refuses a rollback that drops a distributed
|
||||||
|
revocation. Deliver a new current pointer/pin to every consumer and bound their caches.
|
||||||
|
|
||||||
|
For an incident, append a signed revocation, rebuild with full history, inspect,
|
||||||
|
evaluate, sign and activate. Preserve the suspect source bytes, generation and proofs.
|
||||||
|
Follow [SECURITY.md](../SECURITY.md) for private reporting and key compromise.
|
||||||
|
|
@ -14,7 +14,8 @@ define the build behavior. A lockfile pins dependencies, not the host compiler.
|
||||||
The Forgejo workflow uses the documented [workflow and context syntax](https://forgejo.org/docs/latest/user/actions/reference/).
|
The Forgejo workflow uses the documented [workflow and context syntax](https://forgejo.org/docs/latest/user/actions/reference/).
|
||||||
Register `site-registry-isolated` only on a disposable, repository-scoped runner
|
Register `site-registry-isolated` only on a disposable, repository-scoped runner
|
||||||
with the above tools, two build jobs and at least 4 GiB memory. Use a pinned,
|
with the above tools, two build jobs and at least 4 GiB memory. Use a pinned,
|
||||||
reviewed runner image. Do not mount production directories, share signing keys,
|
reviewed runner image; the exact local build is in [ci/Dockerfile](../ci/Dockerfile).
|
||||||
|
Do not mount production directories, share signing keys,
|
||||||
or use Argand's host runners. Follow Forgejo's [runner security guidance](https://forgejo.org/docs/latest/user/actions/security/).
|
or use Argand's host runners. Follow Forgejo's [runner security guidance](https://forgejo.org/docs/latest/user/actions/security/).
|
||||||
The label is a deployment requirement, not a provisioned runner supplied by this
|
The label is a deployment requirement, not a provisioned runner supplied by this
|
||||||
repository. The workflow runs on trusted main pushes or manual dispatch, fetches
|
repository. The workflow runs on trusted main pushes or manual dispatch, fetches
|
||||||
|
|
@ -85,6 +86,8 @@ receipt; the initial release tool packages source only.
|
||||||
|
|
||||||
Source releases contain no provider datasets or real approvals. Dataset publishers
|
Source releases contain no provider datasets or real approvals. Dataset publishers
|
||||||
follow the operator guide: import, inspect, review, build, diff, sign and activate.
|
follow the operator guide: import, inspect, review, build, diff, sign and activate.
|
||||||
|
Follow the [publisher runbook](PUBLISHING.md); release signing requires the external
|
||||||
|
reviewer trust file and re-verifies every stored decision signature.
|
||||||
The dataset namespace `argand-site-registry` is distinct from the source namespace
|
The dataset namespace `argand-site-registry` is distinct from the source namespace
|
||||||
above. Keep `LICENSE_SOURCES.md` and `ATTRIBUTION.json` with the database and receipt.
|
above. Keep `LICENSE_SOURCES.md` and `ATTRIBUTION.json` with the database and receipt.
|
||||||
The weekly update example creates candidates. It never approves, renews, signs or
|
The weekly update example creates candidates. It never approves, renews, signs or
|
||||||
|
|
|
||||||
|
|
@ -12,18 +12,22 @@ Destination and identity decisions bind exact evidence fingerprints, including
|
||||||
names, assertions and normalization context. Reviews expire within 90 days.
|
names, assertions and normalization context. Reviews expire within 90 days.
|
||||||
Changed evidence invalidates earlier approvals. Resolution abstains on ambiguity,
|
Changed evidence invalidates earlier approvals. Resolution abstains on ambiguity,
|
||||||
ties, missing approval or ineligible claims; regional scopes must explicitly
|
ties, missing approval or ineligible claims; regional scopes must explicitly
|
||||||
match. The local writer owns the append-only review log.
|
match. Reviewer decisions are signed under a dedicated SSH namespace and the local
|
||||||
|
writer retains their exact decision/signature bytes in the append-only review log.
|
||||||
|
|
||||||
Generations bind the database, license document and attribution to a completion
|
Generations bind the database, license document and attribution to a completion
|
||||||
receipt. Consumers provide a trusted hash or verify an external publisher key.
|
receipt. Consumers provide a trusted hash or verify an external publisher key.
|
||||||
Activation checks signatures and refuses rollback that loses distributed
|
Release signing re-verifies every stored decision against an external reviewer
|
||||||
revocations. Updates build candidates and cannot approve, sign or activate them.
|
trust file. Activation checks publisher signatures, rejects structurally incomplete
|
||||||
|
review proofs and refuses rollback that loses distributed revocations. Updates
|
||||||
|
build candidates and cannot approve, sign or activate them.
|
||||||
|
|
||||||
## What a publisher must establish
|
## What a publisher must establish
|
||||||
|
|
||||||
The reviewer name and evidence locator in a decision are operator assertions.
|
The reviewer name and evidence locator in a decision are operator assertions.
|
||||||
The CLI validates their structure and evidence binding; it does not independently
|
The CLI authenticates exact decision bytes to an allowed SSH signer and validates
|
||||||
authenticate the reviewer, retrieve their evidence or prove website ownership.
|
their structure and evidence binding; it does not retrieve the cited evidence or
|
||||||
|
prove website ownership.
|
||||||
Protect the writer database and signing key with separate operating permissions.
|
Protect the writer database and signing key with separate operating permissions.
|
||||||
Restrict who can author decisions and require human review before release signing.
|
Restrict who can author decisions and require human review before release signing.
|
||||||
|
|
||||||
|
|
@ -37,8 +41,9 @@ to review. Confidence values are assertion scores, not calibrated probabilities.
|
||||||
Choose expiry based on volatility, within the enforced maximum. Do not renew
|
Choose expiry based on volatility, within the enforced maximum. Do not renew
|
||||||
blindly on a timer. Expired approval should lead to abstention until evidence is
|
blindly on a timer. Expired approval should lead to abstention until evidence is
|
||||||
reviewed. Disclose editorial conflicts and use an independent reviewer for a
|
reviewed. Disclose editorial conflicts and use an independent reviewer for a
|
||||||
disputed claim when possible. The initial implementation is a local single-writer
|
disputed claim when possible. The implementation is a local single-writer tool
|
||||||
tool; it does not provide authenticated reviewer accounts or an enforced quorum.
|
with externally authenticated reviewer keys; it does not provide accounts or an
|
||||||
|
enforced quorum.
|
||||||
|
|
||||||
## What consumers must preserve
|
## What consumers must preserve
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -57,7 +57,7 @@ def safe_path(name):
|
||||||
raise ValueError("unsafe archive path")
|
raise ValueError("unsafe archive path")
|
||||||
allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml",
|
allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml",
|
||||||
".service", ".timer"}
|
".service", ".timer"}
|
||||||
if path.suffix not in allowed and path.name != ".gitignore" and name not in (
|
if path.suffix not in allowed and path.name not in (".gitignore", "Dockerfile") and name not in (
|
||||||
"LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore"):
|
"LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore"):
|
||||||
raise ValueError(f"file is outside the source release allowlist: {name}")
|
raise ValueError(f"file is outside the source release allowlist: {name}")
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue