feat: establish standalone Argand Site Registry
This commit is contained in:
commit
2a0fe1714b
SSH key fingerprint:
SHA256:2te+ycJIQON/Wo/dH6+ZkFSQ4HnHWpetV2azx9E65dQ
60 changed files with 10494 additions and 0 deletions
16
crates/argand-atomic/Cargo.toml
Normal file
16
crates/argand-atomic/Cargo.toml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
# engine/crates/argand-atomic/Cargo.toml
|
||||
# By Nic Weyand!
|
||||
|
||||
[package]
|
||||
name = "argand-atomic"
|
||||
version.workspace = true
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
211
crates/argand-atomic/src/lib.rs
Normal file
211
crates/argand-atomic/src/lib.rs
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
// engine/crates/argand-atomic/src/lib.rs
|
||||
// By Nic Weyand!
|
||||
|
||||
//! Power-loss-durable atomic publication of small files.
|
||||
|
||||
use std::{
|
||||
fs::{File, OpenOptions},
|
||||
io::{self, Write},
|
||||
path::{Path, PathBuf},
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
|
||||
static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Replaces `path` atomically after its new contents and parent entry are durable.
|
||||
///
|
||||
/// A process-unique sibling prevents concurrent writers from corrupting each
|
||||
/// other's temporary file. Publication remains last-writer-wins.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns the underlying filesystem error when the temporary file cannot be
|
||||
/// created, written, synchronized, renamed, or made durable in its directory.
|
||||
pub fn replace_durable(path: &Path, bytes: &[u8]) -> io::Result<()> {
|
||||
replace_durable_with(path, |file| file.write_all(bytes))
|
||||
}
|
||||
|
||||
/// Replaces `path` atomically with contents written incrementally by `write`.
|
||||
///
|
||||
/// This preserves the same power-loss boundary as [`replace_durable`] without
|
||||
/// requiring a large generated artifact to exist twice in memory.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns the writer's error or an underlying filesystem error when the
|
||||
/// temporary file cannot be created, synchronized, renamed, or made durable.
|
||||
pub fn replace_durable_with(
|
||||
path: &Path,
|
||||
write: impl FnOnce(&mut File) -> io::Result<()>,
|
||||
) -> io::Result<()> {
|
||||
let temporary = temporary_sibling(path);
|
||||
let mut guard = TemporaryFile::create(&temporary)?;
|
||||
let file = guard
|
||||
.file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::other("atomic publication temporary file closed unexpectedly"))?;
|
||||
write(file)?;
|
||||
file.sync_all()?;
|
||||
drop(guard.file.take());
|
||||
std::fs::rename(&temporary, path)?;
|
||||
guard.published = true;
|
||||
sync_parent(path)
|
||||
}
|
||||
|
||||
/// Creates `path` atomically after its contents are durable, refusing replacement.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`io::ErrorKind::AlreadyExists`] if `path` already exists. Other
|
||||
/// errors describe temporary-file creation, writing, synchronization,
|
||||
/// publication, or parent-directory synchronization failures.
|
||||
pub fn create_durable(path: &Path, bytes: &[u8]) -> io::Result<()> {
|
||||
create_durable_with(path, |file| file.write_all(bytes))
|
||||
}
|
||||
|
||||
/// Creates `path` atomically from incrementally written contents, without clobbering.
|
||||
///
|
||||
/// The complete synchronized sibling is published with a hard link, so readers
|
||||
/// can never observe partial contents and an existing destination wins.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`io::ErrorKind::AlreadyExists`] if `path` already exists. Other
|
||||
/// errors describe temporary-file creation, writing, synchronization,
|
||||
/// publication, or parent-directory synchronization failures.
|
||||
pub fn create_durable_with(
|
||||
path: &Path,
|
||||
write: impl FnOnce(&mut File) -> io::Result<()>,
|
||||
) -> io::Result<()> {
|
||||
let temporary = temporary_sibling(path);
|
||||
let mut guard = TemporaryFile::create(&temporary)?;
|
||||
let file = guard
|
||||
.file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::other("atomic publication temporary file closed unexpectedly"))?;
|
||||
write(file)?;
|
||||
file.sync_all()?;
|
||||
drop(guard.file.take());
|
||||
std::fs::hard_link(&temporary, path)?;
|
||||
std::fs::remove_file(&temporary)?;
|
||||
guard.published = true;
|
||||
sync_parent(path)
|
||||
}
|
||||
|
||||
fn temporary_sibling(path: &Path) -> PathBuf {
|
||||
let sequence = TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed);
|
||||
let mut name = path.file_name().unwrap_or_default().to_os_string();
|
||||
name.push(format!(".{}.{}.tmp", std::process::id(), sequence));
|
||||
path.with_file_name(name)
|
||||
}
|
||||
|
||||
fn sync_parent(path: &Path) -> io::Result<()> {
|
||||
let parent = path
|
||||
.parent()
|
||||
.filter(|parent| !parent.as_os_str().is_empty())
|
||||
.unwrap_or_else(|| Path::new("."));
|
||||
File::open(parent)?.sync_all()
|
||||
}
|
||||
|
||||
struct TemporaryFile {
|
||||
file: Option<File>,
|
||||
path: PathBuf,
|
||||
published: bool,
|
||||
}
|
||||
|
||||
impl TemporaryFile {
|
||||
fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new() // atomic-writes: allow canonical atomic publication primitive
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(path)?;
|
||||
Ok(Self {
|
||||
file: Some(file),
|
||||
path: path.to_owned(),
|
||||
published: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TemporaryFile {
|
||||
fn drop(&mut self) {
|
||||
if !self.published {
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::io::Write;
|
||||
|
||||
use super::{create_durable, create_durable_with, replace_durable, replace_durable_with};
|
||||
|
||||
#[test]
|
||||
fn creates_complete_contents_without_clobbering() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let destination = directory.path().join("receipt.json");
|
||||
|
||||
create_durable(&destination, b"first")?;
|
||||
let error = match create_durable(&destination, b"second") {
|
||||
Ok(()) => return Err(std::io::Error::other("existing destination was clobbered")),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists);
|
||||
assert_eq!(std::fs::read(&destination)?, b"first");
|
||||
assert_eq!(std::fs::read_dir(directory.path())?.count(), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_create_removes_unpublished_temporary_file() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let destination = directory.path().join("receipt.json");
|
||||
|
||||
let error = match create_durable_with(&destination, |file| {
|
||||
file.write_all(b"partial")?;
|
||||
Err(std::io::Error::other("fixture failure"))
|
||||
}) {
|
||||
Ok(()) => {
|
||||
return Err(std::io::Error::other(
|
||||
"fixture writer unexpectedly succeeded",
|
||||
));
|
||||
}
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::Other);
|
||||
assert!(!destination.exists());
|
||||
assert_eq!(std::fs::read_dir(directory.path())?.count(), 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaces_complete_contents_without_temporary_debris() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let destination = directory.path().join("checkpoint.json");
|
||||
|
||||
replace_durable(&destination, b"old")?;
|
||||
replace_durable(&destination, b"complete-new-value")?;
|
||||
|
||||
assert_eq!(std::fs::read(&destination)?, b"complete-new-value");
|
||||
assert_eq!(std::fs::read_dir(directory.path())?.count(), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streams_complete_contents_without_temporary_debris() -> std::io::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let destination = directory.path().join("streamed");
|
||||
|
||||
replace_durable_with(&destination, |file| {
|
||||
file.write_all(b"first-")?;
|
||||
file.write_all(b"second")
|
||||
})?;
|
||||
|
||||
assert_eq!(std::fs::read(&destination)?, b"first-second");
|
||||
assert_eq!(std::fs::read_dir(directory.path())?.count(), 1);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
8
crates/argand-site-registry/.gitignore
vendored
Normal file
8
crates/argand-site-registry/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# By Nic Weyand! Local source bytes, databases, generations and credentials.
|
||||
/data/
|
||||
/cache/
|
||||
/generations/
|
||||
*.sqlite
|
||||
*.sqlite-*
|
||||
*.part
|
||||
*.sig
|
||||
35
crates/argand-site-registry/Cargo.toml
Normal file
35
crates/argand-site-registry/Cargo.toml
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
# By Nic Weyand!
|
||||
[package]
|
||||
name = "argand-site-registry"
|
||||
version.workspace = true
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
argand-atomic = { path = "../argand-atomic" }
|
||||
bzip2 = "0.6.1"
|
||||
chrono.workspace = true
|
||||
clap.workspace = true
|
||||
csv = "1.4.0"
|
||||
flate2.workspace = true
|
||||
publicsuffix = "=2.3.0"
|
||||
reqwest.workspace = true
|
||||
rusqlite = { version = "=0.40.2", features = ["bundled"] }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
tar = "0.4.46"
|
||||
tokio.workspace = true
|
||||
toml.workspace = true
|
||||
unicode-normalization.workspace = true
|
||||
url.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
http.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
60
crates/argand-site-registry/LICENSE_SOURCES.md
Normal file
60
crates/argand-site-registry/LICENSE_SOURCES.md
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
# Site Registry source licenses
|
||||
|
||||
Reviewed against the primary distribution and licensing pages on 2026-09-12.
|
||||
The Rust code uses the workspace's **AGPL-3.0-or-later** license. Imported data keeps
|
||||
its own licenses; neither the code license nor a merged export relicenses it.
|
||||
Commercial reuse is supported subject to the following obligations. A provider's
|
||||
listing is evidence of an assertion, not a guarantee of ownership or safety.
|
||||
|
||||
| Source | Exact data license and evidence | Distribution and consumed fields |
|
||||
| --- | --- | --- |
|
||||
| Wikidata | [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/), [`CC0-1.0`](https://www.wikidata.org/wiki/Wikidata:Licensing) | [JSON dumps](https://www.wikidata.org/wiki/Wikidata:Database_download), [entity JSON](https://www.wikidata.org/wiki/Special:EntityData/Q355.json). Entity ID, revision, all labels/aliases, full P856 statements including ranks, qualifiers and references; P17, P159, P407, P1001 and country/language code mappings P297/P218/P219/P220. Relevant raw entity records remain available for audit. |
|
||||
| Majestic Million | [CC BY 3.0 Unported](https://creativecommons.org/licenses/by/3.0/), [`CC-BY-3.0`](https://majestic.com/reports/majestic-million) | [Official CSV](https://downloads.majestic.com/majestic_million.csv). Domain/IDN/TLD, global/TLD ranks, referring subnet/IP counts and their previous values. Ranks remain source-specific signals; no entity ownership is inferred. |
|
||||
| Chrome UX Report (CrUX), Google | [CC BY 4.0 International](https://creativecommons.org/licenses/by/4.0/), [`CC-BY-4.0`](https://developer.chrome.com/docs/crux/methodology) | [Monthly BigQuery dataset](https://developer.chrome.com/docs/crux/bigquery/): `origin`, `experimental.popularity.rank`, observation month, optional audience-country dataset code. The adapter produces `origin,rank,yyyymm,country_code` CSV. Rank is a coarse bucket, not a precise visit count. Audience country is not website jurisdiction. No API key or OAuth token is retained. |
|
||||
| Curlie | [CC BY 3.0 Unported](https://creativecommons.org/licenses/by/3.0/), [`CC-BY-3.0`](https://curlie.org/docs/en/license.html), including the attribution placement prescribed on that page | [Format documentation](https://curlie.org/docs/en/rdf.html), [official download redirect](https://curlie.org/directory-dl), currently [Passau-hosted archive](https://share.innkube.fim.uni-passau.de/curlie-rdf/curlie-rdf-all.tar.gz). Despite its RDF name, the current archive contains **literal TSV**. Content: URL, title, description, category ID. Structure: category ID, full category path, entry count, description, latitude, longitude. Archive notices are retained. |
|
||||
| Public Suffix List contributors | [Mozilla Public License 2.0](https://mozilla.org/MPL/2.0/), [`MPL-2.0`](https://publicsuffix.org/list/public_suffix_list.dat) | [Official list](https://publicsuffix.org/list/public_suffix_list.dat). All ICANN and PRIVATE rules, wildcard/exception rules, version/commit comments and notices. Used for hostname, registrable-domain and public-suffix derivations. Download at most once per day. |
|
||||
|
||||
## Attribution and distribution
|
||||
|
||||
* **Wikidata:** CC0 imposes no attribution condition. Keep Wikidata IDs, source
|
||||
links and revision evidence for traceability. CC0 structured data does not
|
||||
extend to unrelated Wikipedia prose, images or linked websites.
|
||||
* **Majestic:** credit “Majestic Million, Majestic”, link the source and CC BY 3.0,
|
||||
retain supplied notices, and identify Argand's changes. The current distribution
|
||||
page governs this import; older blog posts describe different historic terms.
|
||||
* **CrUX:** credit “Chrome UX Report, Google”, link the source and CC BY 4.0,
|
||||
retain notices and indicate the projection/normalization. BigQuery access and
|
||||
billing are separate from the data license. This adapter requires an explicit
|
||||
project and positive maximum-bytes-billed limit.
|
||||
* **Curlie:** the requirement applies to **names, categories and descriptions**.
|
||||
Every public page using Curlie content must include its prescribed HTML credit:
|
||||
|
||||
```html
|
||||
<div title="Curlie Directory Attribution">
|
||||
With content from <a style="text-decoration:none; color:#cd4932;" href="https://curlie.org/" title="Curlie - the largest human-edited directory of the web">Curlie.org</a> - the largest human-edited directory of the web. Contribute by submitting a website or becoming an editor.
|
||||
</div>
|
||||
```
|
||||
|
||||
Also retain the source and license links and indicate modifications. Imports
|
||||
retain descriptions as audit data. Lookup/resolve never expose descriptions;
|
||||
JSONL export omits them by default. `--include-descriptions` is an explicit
|
||||
distribution choice: receiving applications must satisfy these obligations
|
||||
before displaying the text. Treat directory descriptions as untrusted text,
|
||||
never as executable HTML. A JSON attribution object alone does not satisfy
|
||||
Curlie's public-page placement requirement.
|
||||
* **PSL:** retain the list's notices, MPL license and access to its source form
|
||||
when redistributing it. Generations retain the unmodified list in `records`
|
||||
and `facts`; exports include the PSL fact and original download locator.
|
||||
Changes to covered PSL source files must remain available under MPL 2.0.
|
||||
The Rust `publicsuffix` parser is MIT/Apache-2.0; that is separate from the list.
|
||||
|
||||
Every generation contains this document and `ATTRIBUTION.json`, both hash-bound
|
||||
by its signed completion receipt. Exports carry source manifests, licenses,
|
||||
retrieval times, assertion selectors and confidence, plus a required attribution
|
||||
envelope. Consumers must preserve these when extracting subsets or redistributing
|
||||
derived facts. Source credits do not imply endorsement by any provider.
|
||||
|
||||
Cloudflare Radar, default Tranco, Cisco Umbrella, arbitrary mirrors and any other
|
||||
unreviewed source are not supported. Adding a source requires verified commercial
|
||||
reuse rights, its own adapter, provenance and attribution policy. A source format
|
||||
or distribution-host change fails validation until the adapter is reviewed.
|
||||
348
crates/argand-site-registry/README.md
Normal file
348
crates/argand-site-registry/README.md
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
# Argand Site Registry
|
||||
|
||||
A Rust library and CLI for an entity ↔ website/domain dataset. SQLite stores
|
||||
source assertions separately and builds indexed, immutable registry generations.
|
||||
The code follows the engine workspace's **AGPL-3.0-or-later** license; see the
|
||||
[GNU AGPL](https://www.gnu.org/licenses/agpl-3.0.html). Data licenses and required
|
||||
credits are in [LICENSE_SOURCES.md](LICENSE_SOURCES.md).
|
||||
|
||||
`facebook → Facebook → facebook.com` is a name-to-entity-to-registrable-domain
|
||||
lookup. The actual retained Wikidata destination is `https://www.facebook.com/`;
|
||||
normalization does not silently replace it with an apex URL. A mobile website is
|
||||
a separate property. The current retained Amazon entity, Q3884, has 13 P856
|
||||
properties, including `amazon.com`, `amazon.co.uk`, and `amazon.de`.
|
||||
|
||||
Imported assertions enter the review queue. Only explicit, unexpired reviews
|
||||
can produce a `resolve` destination. Automatic updates build candidates; signing
|
||||
and activation are separate operator actions. This crate is a dataset component;
|
||||
existing Argand Navigate policy and collection admission still apply when a
|
||||
consumer integrates it into public search.
|
||||
|
||||
## Install and run the offline acceptance example
|
||||
|
||||
Rust 1.97+ and OpenSSH (`ssh-keygen`) are required. From the standalone repository root:
|
||||
|
||||
```bash
|
||||
cargo install --path crates/argand-site-registry --locked
|
||||
argand-site-registry --help
|
||||
cargo test -p argand-site-registry --all-targets --locked --offline
|
||||
```
|
||||
|
||||
The native CLI test imports small source-shaped fixtures for **all five sources**,
|
||||
repeats the imports, resolves aliases, signs and activates an approved generation,
|
||||
revokes the destination, and rejects rollback past the revocation. Synthetic
|
||||
fixtures are authored in Rust test code; no provider datasets or signing keys
|
||||
are committed. To retain a local example for inspection, choose a **new** path:
|
||||
|
||||
```bash
|
||||
ARGAND_REGISTRY_E2E_OUTPUT=/tmp/argand-site-registry-example \
|
||||
cargo test -p argand-site-registry --test cli --locked --offline -- --nocapture
|
||||
```
|
||||
|
||||
The example contains `candidate/`, `approved/`, `revoked/`, JSON manifests, an
|
||||
attributed export and a disposable test key. Do not use that test key or those
|
||||
synthetic approvals for a real release.
|
||||
|
||||
## Acquire and import sources
|
||||
|
||||
All paths are explicit. These commands use `jq` only to read CLI JSON output.
|
||||
They create data outside the checkout. Byte caps are upper bounds, not estimates
|
||||
of current source sizes. Increase a cap only after checking available storage.
|
||||
|
||||
```bash
|
||||
export ARGAND_SITE_DATA="$HOME/.local/share/argand-site-registry"
|
||||
mkdir -p "$ARGAND_SITE_DATA"
|
||||
|
||||
argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \
|
||||
--source psl --format psl-text \
|
||||
--url https://publicsuffix.org/list/public_suffix_list.dat \
|
||||
--snapshot "$(date -u +%F)" --scope full --maximum-bytes 1000000 \
|
||||
> "$ARGAND_SITE_DATA/psl-download.json"
|
||||
|
||||
argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \
|
||||
--source wikidata --format wikidata-entities \
|
||||
--url 'https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q355%7CQ3884&format=json&maxlag=5' \
|
||||
--snapshot "$(date -u +%F)" --scope selection:facebook-amazon \
|
||||
--maximum-bytes 5000000 > "$ARGAND_SITE_DATA/wikidata-download.json"
|
||||
|
||||
argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \
|
||||
--source majestic --format majestic-csv \
|
||||
--url https://downloads.majestic.com/majestic_million.csv \
|
||||
--snapshot "$(date -u +%F)" --scope full --maximum-bytes 250000000 \
|
||||
> "$ARGAND_SITE_DATA/majestic-download.json"
|
||||
|
||||
argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \
|
||||
--source curlie --format curlie-tar-gz \
|
||||
--url https://curlie.org/directory-dl \
|
||||
--snapshot "$(date -u +%F)" --scope full --maximum-bytes 1000000000 \
|
||||
> "$ARGAND_SITE_DATA/curlie-download.json"
|
||||
|
||||
for source in psl wikidata majestic curlie; do
|
||||
argand-site-registry import --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||
--input "$(jq -r .input "$ARGAND_SITE_DATA/$source-download.json")" \
|
||||
--manifest "$(jq -r .manifest "$ARGAND_SITE_DATA/$source-download.json")"
|
||||
done
|
||||
```
|
||||
|
||||
For a full Wikidata dump, select a real dump URL from the official
|
||||
[download index](https://dumps.wikimedia.org/wikidatawiki/entities/), then use
|
||||
`--format wikidata-dump --compression gzip` (or `bzip2`) and `--scope full`.
|
||||
The parser handles the documented one-entity-per-line JSON array and concatenated
|
||||
compressed streams. Do not use truthy RDF: it loses statement evidence. Full
|
||||
dumps need substantial disk space and a long sequential scan even though memory
|
||||
is bounded. A small entity selection is useful on limited hardware.
|
||||
|
||||
To reuse an already acquired file, retain its **original** retrieval time,
|
||||
source URL and snapshot/revision. First verify its acquisition receipt, then:
|
||||
|
||||
```bash
|
||||
argand-site-registry manifest --input /data/Q355.json \
|
||||
--output /data/Q355.source.json --source wikidata --format wikidata-entities \
|
||||
--source-url https://www.wikidata.org/wiki/Special:EntityData/Q355.json \
|
||||
--snapshot retained-Q355-revision --scope selection:Q355 \
|
||||
--retrieved-at 2026-09-10T13:40:20.446514Z
|
||||
argand-site-registry import --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||
--input /data/Q355.json --manifest /data/Q355.source.json
|
||||
```
|
||||
|
||||
Replace paths, snapshot and time with the actual acquisition details. A manifest
|
||||
declares provenance; making one does not authenticate arbitrary file contents.
|
||||
|
||||
### CrUX
|
||||
|
||||
The adapter queries the documented monthly BigQuery table and streams paginated
|
||||
results into this exact CSV projection:
|
||||
|
||||
```sql
|
||||
SELECT DISTINCT origin, experimental.popularity.rank AS rank,
|
||||
'202608' AS yyyymm, '' AS country_code
|
||||
FROM `chrome-ux-report.all.202608`
|
||||
WHERE experimental.popularity.rank IS NOT NULL
|
||||
ORDER BY origin, rank
|
||||
```
|
||||
|
||||
The month above is an example of the documented table naming. Confirm that the
|
||||
desired month exists. For an audience-country dataset, use `country: "GB"` in
|
||||
the request; the adapter selects `chrome-ux-report.country_gb.202608` and emits
|
||||
`GB`. The rank is a bucket; do not mix it numerically with Majestic's exact rank.
|
||||
|
||||
Create `crux-request.json` with your project and explicit limits:
|
||||
|
||||
```json
|
||||
{
|
||||
"project": "your-billing-project",
|
||||
"month": "202608",
|
||||
"country": null,
|
||||
"maximum_bytes_billed": 1000000000,
|
||||
"maximum_output_bytes": 500000000
|
||||
}
|
||||
```
|
||||
|
||||
Supply an authorized OAuth access token through `GOOGLE_OAUTH_ACCESS_TOKEN`
|
||||
using your credential manager, then run:
|
||||
|
||||
```bash
|
||||
argand-site-registry crux-download --cache "$ARGAND_SITE_DATA/cache" \
|
||||
--request crux-request.json > "$ARGAND_SITE_DATA/crux-download.json"
|
||||
argand-site-registry import --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||
--input "$(jq -r .input "$ARGAND_SITE_DATA/crux-download.json")" \
|
||||
--manifest "$(jq -r .manifest "$ARGAND_SITE_DATA/crux-download.json")"
|
||||
```
|
||||
|
||||
No default billing project or unbounded query is provided. An interrupted job
|
||||
reuses its content-derived BigQuery job ID; result pages replay from the same
|
||||
query result. Keep `job.json` with the acquisition records. Expired server results
|
||||
require an operator to inspect the existing job. Pinned local exports of the
|
||||
exact CSV projection can instead use `manifest --source crux --format crux-csv
|
||||
--source-url https://developer.chrome.com/docs/crux/bigquery/` with their actual
|
||||
retrieval time, query/snapshot identity and appropriate `monthly:YYYYMM:country`
|
||||
scope. The token is never written into a manifest.
|
||||
|
||||
## Build, look up and review
|
||||
|
||||
```bash
|
||||
argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||
--output "$ARGAND_SITE_DATA/generation-1" > "$ARGAND_SITE_DATA/build-1.json"
|
||||
export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-1.json")"
|
||||
argand-site-registry lookup --generation "$ARGAND_SITE_DATA/generation-1" \
|
||||
--pin "$ARGAND_SITE_PIN" --query facebook
|
||||
argand-site-registry lookup --generation "$ARGAND_SITE_DATA/generation-1" \
|
||||
--pin "$ARGAND_SITE_PIN" --query amazon --limit 100
|
||||
```
|
||||
|
||||
The real-source acceptance run produced:
|
||||
|
||||
| Query | Entity | Example properties | Registrable domains |
|
||||
| --- | --- | --- | --- |
|
||||
| `facebook` | Facebook (Q355) | `https://www.facebook.com/`, `https://m.facebook.com/` | `facebook.com` |
|
||||
| `amazon` | Amazon (Q3884) | `https://www.amazon.com/`, `https://www.amazon.co.uk/`, `https://www.amazon.de/` | `amazon.com`, `amazon.co.uk`, `amazon.de` |
|
||||
|
||||
These properties were asserted on the **same Wikidata entity**. Hostname
|
||||
resemblance did not establish the relationship. Imported qualifiers remain in
|
||||
`evidence` and `property_scopes`; unknown locale/country remains null. Names,
|
||||
aliases and entity metadata carry their own fact-level source declarations.
|
||||
Regional locale/country and role are explicit reviewed assertions. They are
|
||||
separate from entity headquarters, ccTLD spelling and CrUX audience country.
|
||||
|
||||
Inspect the full statements, references, names/aliases, hostname spelling and
|
||||
independent current ownership/role evidence. A review JSON has this shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"fingerprint": "COPY_THE_EXACT_64_CHARACTER_FINGERPRINT_FROM_LOOKUP",
|
||||
"decision": "approve",
|
||||
"reviewer": "operator identity",
|
||||
"reason": "How entity ownership and this exact destination role were verified",
|
||||
"evidence": "An immutable capture identifier or evidence digest",
|
||||
"reviewed_at": "2026-09-12T12:00:00Z",
|
||||
"expires_at": "2026-10-12T12:00:00Z",
|
||||
"role": "regional",
|
||||
"locale": "",
|
||||
"country": "GB"
|
||||
}
|
||||
```
|
||||
|
||||
Replace the example evidence and dates; approvals expire within 90 days. Use
|
||||
`role: "primary"` for the independently verified default and `country: "DE"`
|
||||
for a separately verified German regional property. A country-scoped review
|
||||
can leave locale empty. If both are specified, both must match the request.
|
||||
|
||||
```bash
|
||||
argand-site-registry review --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||
--generation "$ARGAND_SITE_DATA/generation-1" --pin "$ARGAND_SITE_PIN" \
|
||||
--decision review.json
|
||||
argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \
|
||||
--output "$ARGAND_SITE_DATA/generation-2" > "$ARGAND_SITE_DATA/build-2.json"
|
||||
export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-2.json")"
|
||||
argand-site-registry resolve --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||
--pin "$ARGAND_SITE_PIN" --query amazon --country GB
|
||||
```
|
||||
|
||||
After the corresponding real reviews, GB selects the reviewed UK property;
|
||||
DE selects the reviewed German property; otherwise an explicitly reviewed
|
||||
primary may be used. Unknown, expired, tied or entity-ambiguous requests return
|
||||
`"destination": null`. Result limits never hide ambiguity. Name/alias changes,
|
||||
changed statements/revisions, URLs or normalization evidence invalidate reviews.
|
||||
Deprecated, end-dated and non-value statements remain audit evidence and cannot
|
||||
be admitted. An unchanged PSL file with a new retrieval time preserves reviews.
|
||||
|
||||
## Release, export, update and recovery
|
||||
|
||||
When two providers describe the same navigational entity, `lookup` deliberately
|
||||
shows both source IDs. Connect them only after reviewing their identities:
|
||||
|
||||
```bash
|
||||
argand-site-registry equivalence --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||
--pin "$ARGAND_SITE_PIN" --left SOURCE_ENTITY_ID --right OTHER_SOURCE_ENTITY_ID
|
||||
```
|
||||
|
||||
Use the returned fingerprint in a review JSON with `role: "unspecified"`, empty
|
||||
locale/country, a reason, immutable identity evidence and an expiry. Then repeat
|
||||
the command with `--database "$ARGAND_SITE_DATA/import.sqlite" --decision
|
||||
identity-review.json` and rebuild. `resolve` follows only active, explicitly
|
||||
reviewed equivalences and includes their provenance. Each destination still
|
||||
needs its own review. The original IDs, raw ambiguity counts and conflicting
|
||||
assertions remain visible. Changed names or website assertions invalidate the
|
||||
identity decision; identity revocations use the same append-only release log.
|
||||
Operator-authored decisions are published under CC0-1.0, separately from source
|
||||
data licenses.
|
||||
|
||||
Each generation contains `registry.sqlite`, `LICENSE_SOURCES.md`,
|
||||
`ATTRIBUTION.json` and a hash-binding `COMPLETE.json`. Distribute all four together.
|
||||
Keep the import database, review history and cached source bytes for recovery.
|
||||
The SQLite file includes raw relevant records and descriptions for audit; public
|
||||
consumers must obey the source attribution requirements. `export` streams
|
||||
source-bearing JSONL and omits descriptions by default:
|
||||
|
||||
```bash
|
||||
argand-site-registry export --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||
--pin "$ARGAND_SITE_PIN" --output "$ARGAND_SITE_DATA/assertions.jsonl"
|
||||
argand-site-registry sign --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||
--pin "$ARGAND_SITE_PIN" --key /secure/registry-signing-key
|
||||
argand-site-registry activate --generation "$ARGAND_SITE_DATA/generation-2" \
|
||||
--current "$ARGAND_SITE_DATA/current.json" \
|
||||
--allowed-signers /secure/registry-allowed-signers --identity registry-publisher
|
||||
```
|
||||
|
||||
Use an existing operator-controlled SSH signing key. The external allowed-signers
|
||||
file follows OpenSSH syntax: `registry-publisher ssh-ed25519 PUBLIC_KEY`. Neither
|
||||
keys nor the trust file should come from the downloaded dataset. Consumers can
|
||||
open `Registry::open(path, trusted_receipt_sha256)` once and reuse its indexed
|
||||
queries, or verify a publisher with `release::verify_signed` first. A hash proves
|
||||
integrity only relative to a trusted pin. Signature verification authenticates
|
||||
the publisher, not the truth of a source assertion.
|
||||
|
||||
`diff --old PATH --old-pin HASH --new PATH --new-pin HASH` streams added/removed
|
||||
edge fingerprints. `verify --generation PATH --pin HASH` checks every artifact
|
||||
bound by the receipt. To revoke, append a review with `decision: "revoke"`, then
|
||||
rebuild, sign and activate. Re-activating an older signed generation supports
|
||||
rollback **only if it retains every distributed revocation**. Otherwise rebuild
|
||||
the older source selection with the current review log; never edit generations.
|
||||
|
||||
The [update configuration](examples/update.toml) and [systemd service/timer](examples/)
|
||||
provide weekly candidate refreshes without a resident daemon. Set absolute paths
|
||||
and an installed executable path. TOML paths do not expand environment variables.
|
||||
`update --config /etc/argand-site-registry.toml` downloads/imports all configured
|
||||
sources and builds only after they succeed. The same inputs and review log reuse
|
||||
the same generation. A nonzero exit is a failed refresh; the active pointer stays
|
||||
intact. Failed `pending-*` builds can be inspected before explicitly removing
|
||||
that incomplete directory. Acquisition, import and update operations take local
|
||||
locks; use one writer and keep old complete generations for rollback.
|
||||
|
||||
Downloads permit only the reviewed HTTPS source endpoints, validate each
|
||||
redirect, bound bytes and bind range resumes to strong ETags. Chunked/validatorless
|
||||
responses safely restart on interruption. PSL network attempts are limited to
|
||||
once per 24 hours per cache. CrUX is opt-in and may use `{previous_month}` in
|
||||
update configuration; monthly data may not yet be published on the first day.
|
||||
Wikidata source snapshot labels support `{date}` and `{month}` in scheduled
|
||||
downloads. No scheduled job signs, approves, renews approvals or activates links.
|
||||
|
||||
## Storage and operating limits
|
||||
|
||||
Migration `migrations/001.sql` owns schema version 1. `sources`, `records` and
|
||||
`facts` preserve snapshot/native IDs, licenses, retrieval times and confidence;
|
||||
`reviews` is append-only. Complete source selection is latest retrieval time per
|
||||
provider/scope, with digest as the deterministic tie break. Use the **same scope**
|
||||
for a replacement snapshot, and separate scopes for deliberate independent
|
||||
selections. History and conflicts remain stored. A failed source cannot replace
|
||||
a complete one. Avoid overlapping full/partial scopes unless both evidences are
|
||||
intended to remain active.
|
||||
|
||||
Derived tables are `selected_sources`, `entities`, `names`, `properties`, `edges`,
|
||||
`popularity` and `rejected`. Entity IDs derive from source/native IDs; URL IDs
|
||||
derive from strict normalized URLs. Equal source entities merge across snapshots
|
||||
and equal URLs share a property. Explicit `equivalences` connect reviewed
|
||||
cross-source identities while preserving both IDs. Names are never an identity
|
||||
join. The canonical label rule prefers labels, then English,
|
||||
then language/text order. Original labels/aliases are kept. Popularity has its
|
||||
own source, target, observation period and audience scope and never creates an
|
||||
ownership edge. All derivations bind input fact IDs, PSL identity and the
|
||||
`argand.site-rules/v1` contract through their generation receipt.
|
||||
|
||||
Imports use transactions of 256 relevant records with durable replay checkpoints.
|
||||
Restart replays the compressed stream and skips committed records. Large source
|
||||
records are capped at 16 MiB; SQLite has an 8 MiB page cache and disk-backed sorts.
|
||||
Builds stream a canonical sorted copy and never load the full registry into RAM.
|
||||
Names and entity metadata exposed by lookup are capped at 256 facts each, with
|
||||
uncapped totals; the complete assertions remain available in the database/export.
|
||||
Lookup returns at most 100 edges and reports all pre-limit ambiguity counts.
|
||||
The full store/history and each generation consume disk; there is no automatic
|
||||
pruning. The original compressed source is hashed before/after import, so expect
|
||||
extra sequential disk reads. These bounds are not a full-dump throughput claim.
|
||||
|
||||
The `observation` module defines future crawler evidence for redirects, canonical
|
||||
links, hreflang, JSON-LD sameAs, sitemaps and country selectors, with capture IDs,
|
||||
hashes, rights and confidence. It does not crawl or automatically infer ownership.
|
||||
|
||||
Source vandalism, compromised publishers and a domain changing ownership cannot
|
||||
be eliminated by hashes or popularity. Review expiration, exact evidence binding,
|
||||
signed releases, explicit revocations and conservative abstention contain those
|
||||
risks. Protect the writer database, signing key and consumer trust configuration.
|
||||
Do not feed raw `lookup` candidates straight into an automatic redirect consumer.
|
||||
|
||||
If an import fails, fix the input/format or reuse the matching original source
|
||||
manifest, then rerun the same import. Do not edit digests to make corrupted data
|
||||
pass. A source host/schema change needs an adapter review. HTTP 403/429 is a
|
||||
source-access failure; reuse an authorized retained snapshot or retry according
|
||||
to the provider's policy. A null resolution means evidence/review is missing,
|
||||
expired or ambiguous; `lookup` explains which assertions are involved.
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# By Nic Weyand! Install the executable/config and create this service user first.
|
||||
[Unit]
|
||||
Description=Build an Argand Site Registry candidate
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=argand-site-registry
|
||||
Group=argand-site-registry
|
||||
StateDirectory=argand-site-registry
|
||||
ExecStart=/usr/local/bin/argand-site-registry update --config /etc/argand-site-registry.toml
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/var/lib/argand-site-registry
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||
TimeoutStartSec=infinity
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
# By Nic Weyand! Creates review candidates; does not sign or activate them.
|
||||
[Unit]
|
||||
Description=Refresh the Argand Site Registry weekly
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Sun *-*-* 03:00:00 UTC
|
||||
RandomizedDelaySec=30m
|
||||
Persistent=true
|
||||
Unit=argand-site-registry.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
24
crates/argand-site-registry/examples/lookup.rs
Normal file
24
crates/argand-site-registry/examples/lookup.rs
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// By Nic Weyand!
|
||||
//! Open an externally pinned registry once and emit the complete lookup envelope.
|
||||
|
||||
use argand_site_registry::query::Registry;
|
||||
use clap::Parser;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
query: String,
|
||||
}
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let args = Args::parse();
|
||||
let registry = Registry::open(&args.generation, &args.pin)?;
|
||||
let result = registry.lookup(&args.query, 100)?;
|
||||
println!("{}", serde_json::to_string(&result)?);
|
||||
Ok(())
|
||||
}
|
||||
50
crates/argand-site-registry/examples/update.toml
Normal file
50
crates/argand-site-registry/examples/update.toml
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# By Nic Weyand! Operator example; use absolute writable paths.
|
||||
cache = "/var/lib/argand-site-registry/cache"
|
||||
database = "/var/lib/argand-site-registry/import.sqlite"
|
||||
generations = "/var/lib/argand-site-registry/generations"
|
||||
|
||||
[[downloads]]
|
||||
source = "psl"
|
||||
format = "psl_text"
|
||||
url = "https://publicsuffix.org/list/public_suffix_list.dat"
|
||||
snapshot = "{date}"
|
||||
scope = "full"
|
||||
maximum_bytes = 1000000
|
||||
|
||||
[[downloads]]
|
||||
source = "wikidata"
|
||||
format = "wikidata_entities"
|
||||
url = "https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q355%7CQ3884&format=json&maxlag=5"
|
||||
snapshot = "{date}"
|
||||
scope = "selection:facebook-amazon"
|
||||
maximum_bytes = 5000000
|
||||
|
||||
[[downloads]]
|
||||
source = "majestic"
|
||||
format = "majestic_csv"
|
||||
url = "https://downloads.majestic.com/majestic_million.csv"
|
||||
snapshot = "{date}"
|
||||
scope = "full"
|
||||
maximum_bytes = 250000000
|
||||
|
||||
[[downloads]]
|
||||
source = "curlie"
|
||||
format = "curlie_tar_gz"
|
||||
url = "https://curlie.org/directory-dl"
|
||||
snapshot = "{date}"
|
||||
scope = "full"
|
||||
maximum_bytes = 1000000000
|
||||
|
||||
# Optional pinned acquisitions; repeat [[inputs]] for each source.
|
||||
# [[inputs]]
|
||||
# input = "/data/source-object.gz"
|
||||
# manifest = "/data/source.json"
|
||||
|
||||
# CrUX is deliberately opt-in: provide your authorized project, positive billing
|
||||
# cap, and GOOGLE_OAUTH_ACCESS_TOKEN through the scheduler credential environment.
|
||||
# [[crux]]
|
||||
# project = "your-billing-project"
|
||||
# month = "{previous_month}"
|
||||
# country = "GB"
|
||||
# maximum_bytes_billed = 1000000000
|
||||
# maximum_output_bytes = 500000000
|
||||
43
crates/argand-site-registry/migrations/001.sql
Normal file
43
crates/argand-site-registry/migrations/001.sql
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
-- By Nic Weyand! Immutable assertions; only import progress and append-only review grow.
|
||||
CREATE TABLE registry_metadata (singleton INTEGER PRIMARY KEY CHECK(singleton=1), rules TEXT NOT NULL) STRICT;
|
||||
INSERT INTO registry_metadata VALUES(1,'argand.site-rules/v1');
|
||||
CREATE TABLE sources (
|
||||
id TEXT PRIMARY KEY, source TEXT NOT NULL, scope TEXT NOT NULL,
|
||||
retrieved_at TEXT NOT NULL, manifest TEXT NOT NULL,
|
||||
checkpoint INTEGER NOT NULL DEFAULT 0 CHECK(checkpoint >= 0),
|
||||
complete INTEGER NOT NULL DEFAULT 0 CHECK(complete IN (0,1))
|
||||
) STRICT;
|
||||
CREATE INDEX source_latest ON sources(source,scope,complete,retrieved_at,id);
|
||||
CREATE TABLE records (
|
||||
source_id TEXT NOT NULL REFERENCES sources(id), ordinal INTEGER NOT NULL,
|
||||
native_id TEXT NOT NULL, raw_json TEXT NOT NULL,
|
||||
PRIMARY KEY(source_id,ordinal)
|
||||
) STRICT;
|
||||
CREATE TABLE facts (
|
||||
id TEXT PRIMARY KEY, source_id TEXT NOT NULL, ordinal INTEGER NOT NULL,
|
||||
subject TEXT NOT NULL, predicate TEXT NOT NULL, value TEXT NOT NULL,
|
||||
selector TEXT NOT NULL, confidence INTEGER NOT NULL CHECK(confidence BETWEEN 0 AND 10000),
|
||||
FOREIGN KEY(source_id,ordinal) REFERENCES records(source_id,ordinal)
|
||||
) STRICT;
|
||||
CREATE INDEX fact_source ON facts(source_id,ordinal);
|
||||
CREATE INDEX fact_subject ON facts(subject,predicate);
|
||||
CREATE TABLE reviews (
|
||||
sequence INTEGER PRIMARY KEY, fingerprint TEXT NOT NULL,
|
||||
decision TEXT NOT NULL CHECK(decision IN ('approve','revoke')),
|
||||
reviewer TEXT NOT NULL, reason TEXT NOT NULL, evidence TEXT NOT NULL,
|
||||
reviewed_at TEXT NOT NULL, expires_at TEXT NOT NULL,
|
||||
role TEXT NOT NULL CHECK(role IN ('primary','regional','unspecified')),
|
||||
locale TEXT NOT NULL, country TEXT NOT NULL
|
||||
) STRICT;
|
||||
CREATE INDEX review_fingerprint ON reviews(fingerprint,sequence DESC);
|
||||
CREATE TABLE equivalences (
|
||||
fingerprint TEXT PRIMARY KEY, left_entity TEXT NOT NULL, right_entity TEXT NOT NULL,
|
||||
left_signature TEXT NOT NULL, right_signature TEXT NOT NULL
|
||||
) STRICT;
|
||||
CREATE INDEX equivalence_left ON equivalences(left_entity);
|
||||
CREATE INDEX equivalence_right ON equivalences(right_entity);
|
||||
CREATE TRIGGER equivalence_no_update BEFORE UPDATE ON equivalences BEGIN SELECT RAISE(ABORT,'equivalences are immutable'); END;
|
||||
CREATE TRIGGER equivalence_no_delete BEFORE DELETE ON equivalences BEGIN SELECT RAISE(ABORT,'equivalences are immutable'); END;
|
||||
CREATE TRIGGER review_no_update BEFORE UPDATE ON reviews BEGIN SELECT RAISE(ABORT,'reviews are append-only'); END;
|
||||
CREATE TRIGGER review_no_delete BEFORE DELETE ON reviews BEGIN SELECT RAISE(ABORT,'reviews are append-only'); END;
|
||||
PRAGMA user_version=1;
|
||||
139
crates/argand-site-registry/src/adapters/csv_sources.rs
Normal file
139
crates/argand-site-registry/src/adapters/csv_sources.rs
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
// By Nic Weyand!
|
||||
//! Source-specific CSV projections; popularity never creates an entity edge.
|
||||
|
||||
use super::{RecordSink, SourceAdapter, bounded_line};
|
||||
use crate::model::{Fact, Record};
|
||||
use anyhow::{Context, ensure};
|
||||
use serde_json::{Value, json};
|
||||
use std::io::BufRead;
|
||||
|
||||
pub(super) struct CsvSource {
|
||||
pub crux: bool,
|
||||
}
|
||||
|
||||
impl SourceAdapter for CsvSource {
|
||||
fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> {
|
||||
let mut line = String::new();
|
||||
ensure!(bounded_line(input, &mut line)? > 0, "empty CSV");
|
||||
let headers = parse_line(&line)?;
|
||||
let required = if self.crux {
|
||||
vec!["origin", "rank", "yyyymm", "country_code"]
|
||||
} else {
|
||||
vec![
|
||||
"GlobalRank",
|
||||
"TldRank",
|
||||
"Domain",
|
||||
"TLD",
|
||||
"RefSubNets",
|
||||
"RefIPs",
|
||||
"IDN_Domain",
|
||||
"IDN_TLD",
|
||||
"PrevGlobalRank",
|
||||
"PrevTldRank",
|
||||
"PrevRefSubNets",
|
||||
"PrevRefIPs",
|
||||
]
|
||||
};
|
||||
ensure!(
|
||||
headers.iter().map(String::as_str).collect::<Vec<_>>() == required,
|
||||
"CSV schema differs from documented projection"
|
||||
);
|
||||
let mut ordinal = 0;
|
||||
while bounded_line(input, &mut line)? > 0 {
|
||||
ensure!(!line.trim().is_empty(), "blank CSV row");
|
||||
let cells = parse_line(&line)?;
|
||||
ensure!(cells.len() == headers.len(), "CSV column count changed");
|
||||
let raw: Value = headers
|
||||
.iter()
|
||||
.zip(&cells)
|
||||
.map(|(k, v)| (k.clone(), json!(v)))
|
||||
.collect();
|
||||
let value = if self.crux {
|
||||
crux(&raw)?
|
||||
} else {
|
||||
majestic(&raw)?
|
||||
};
|
||||
ordinal += 1;
|
||||
let subject = value["target"]
|
||||
.as_str()
|
||||
.context("missing popularity target")?
|
||||
.to_owned();
|
||||
sink.emit(Record {
|
||||
native_id: format!("row:{ordinal}"),
|
||||
raw,
|
||||
facts: vec![Fact {
|
||||
subject,
|
||||
predicate: "popularity".into(),
|
||||
value,
|
||||
selector: format!("row:{ordinal}"),
|
||||
confidence: 10000,
|
||||
}],
|
||||
})?;
|
||||
}
|
||||
ensure!(ordinal > 0, "empty CSV dataset");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_line(line: &str) -> anyhow::Result<Vec<String>> {
|
||||
let mut reader = csv::ReaderBuilder::new()
|
||||
.has_headers(false)
|
||||
.from_reader(line.as_bytes());
|
||||
let row = reader.records().next().context("missing CSV record")??;
|
||||
ensure!(
|
||||
reader.records().next().is_none(),
|
||||
"multiline CSV unsupported by these source contracts"
|
||||
);
|
||||
Ok(row.iter().map(str::to_owned).collect())
|
||||
}
|
||||
|
||||
fn integer(raw: &Value, key: &str) -> anyhow::Result<u64> {
|
||||
Ok(raw[key].as_str().context("CSV field missing")?.parse()?)
|
||||
}
|
||||
|
||||
fn majestic(raw: &Value) -> anyhow::Result<Value> {
|
||||
let rank = integer(raw, "GlobalRank")?;
|
||||
ensure!(rank > 0, "rank must be positive");
|
||||
Ok(
|
||||
json!({"target":raw["Domain"],"target_kind":"hostname","rank":rank,
|
||||
"tld_rank":integer(raw,"TldRank")?,"referring_subnets":integer(raw,"RefSubNets")?,
|
||||
"referring_ips":integer(raw,"RefIPs")?,"previous_global_rank":integer(raw,"PrevGlobalRank")?,
|
||||
"previous_tld_rank":integer(raw,"PrevTldRank")?,"previous_referring_subnets":integer(raw,"PrevRefSubNets")?,
|
||||
"previous_referring_ips":integer(raw,"PrevRefIPs")?,"country_code":null,"period":null}),
|
||||
)
|
||||
}
|
||||
|
||||
fn crux(raw: &Value) -> anyhow::Result<Value> {
|
||||
let month = raw["yyyymm"].as_str().context("missing month")?;
|
||||
validate_month(month)?;
|
||||
let country = raw["country_code"]
|
||||
.as_str()
|
||||
.context("missing audience country")?;
|
||||
ensure!(
|
||||
country.is_empty()
|
||||
|| (country.len() == 2 && country.bytes().all(|b| b.is_ascii_alphabetic())),
|
||||
"invalid CrUX audience country"
|
||||
);
|
||||
let rank = integer(raw, "rank")?;
|
||||
ensure!(rank > 0, "rank must be positive");
|
||||
let url = url::Url::parse(raw["origin"].as_str().context("missing origin")?)?;
|
||||
ensure!(
|
||||
url.path() == "/" && url.query().is_none() && url.fragment().is_none(),
|
||||
"CrUX target must be origin"
|
||||
);
|
||||
Ok(
|
||||
json!({"target":raw["origin"],"target_kind":"origin","rank":rank,"coarse_rank":true,"period":month,"country_code":country.to_ascii_uppercase()}),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_month(month: &str) -> anyhow::Result<()> {
|
||||
ensure!(
|
||||
month.len() == 6 && month.bytes().all(|b| b.is_ascii_digit()),
|
||||
"month must be YYYYMM"
|
||||
);
|
||||
ensure!(
|
||||
(1..=12).contains(&month[4..].parse::<u8>()?),
|
||||
"invalid month"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
171
crates/argand-site-registry/src/adapters/curlie.rs
Normal file
171
crates/argand-site-registry/src/adapters/curlie.rs
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
// By Nic Weyand!
|
||||
//! Curlie v2 (2025-05): unquoted 4-column content and 6-column structure TSV.
|
||||
|
||||
use super::{RecordSink, SourceAdapter, bounded_line};
|
||||
use crate::model::{Fact, Record, Source, entity_id};
|
||||
use anyhow::{Context, ensure};
|
||||
use serde_json::json;
|
||||
use std::{
|
||||
io::{BufRead, BufReader, Read},
|
||||
path::Component,
|
||||
};
|
||||
|
||||
pub(super) struct Curlie;
|
||||
|
||||
impl SourceAdapter for Curlie {
|
||||
fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> {
|
||||
let gzip = flate2::read::MultiGzDecoder::new(input);
|
||||
let mut archive = tar::Archive::new(gzip);
|
||||
let mut entries = 0_u64;
|
||||
for entry in archive.entries()? {
|
||||
let mut entry = entry?;
|
||||
let path = entry.path()?.into_owned();
|
||||
ensure!(
|
||||
path.components().all(|c| matches!(c, Component::Normal(_))),
|
||||
"unsafe archive path"
|
||||
);
|
||||
let name = path.to_str().context("non-UTF8 archive member")?.to_owned();
|
||||
ensure!(
|
||||
name.starts_with("curlie-rdf/") || name == "curlie-rdf",
|
||||
"unexpected Curlie archive root"
|
||||
);
|
||||
if entry.header().entry_type().is_dir() {
|
||||
continue;
|
||||
}
|
||||
ensure!(
|
||||
entry.header().entry_type().is_file(),
|
||||
"archive links and special members rejected"
|
||||
);
|
||||
ensure!(
|
||||
entry.size() <= 4 * 1024 * 1024 * 1024,
|
||||
"archive member exceeds 4 GiB"
|
||||
);
|
||||
let content = name.ends_with("-c.tsv");
|
||||
let structure = name.ends_with("-s.tsv");
|
||||
if !content && !structure {
|
||||
ensure!(
|
||||
path.extension()
|
||||
.is_some_and(|e| e.eq_ignore_ascii_case("txt")),
|
||||
"unrecognized archive member"
|
||||
);
|
||||
let mut text = String::new();
|
||||
entry.take(1024 * 1024 + 1).read_to_string(&mut text)?;
|
||||
ensure!(text.len() <= 1024 * 1024, "Curlie metadata too large");
|
||||
sink.emit(Record {
|
||||
native_id: name,
|
||||
raw: json!(text),
|
||||
facts: vec![],
|
||||
})?;
|
||||
continue;
|
||||
}
|
||||
let mut reader = BufReader::new(&mut entry);
|
||||
let mut line = String::new();
|
||||
let mut ordinal = 0;
|
||||
while bounded_line(&mut reader, &mut line)? > 0 {
|
||||
let fields: Vec<&str> = line.trim_end_matches(['\n', '\r']).split('\t').collect();
|
||||
ensure!(
|
||||
fields.len() == if content { 4 } else { 6 },
|
||||
"Curlie TSV schema changed: {name}"
|
||||
);
|
||||
ordinal += 1;
|
||||
entries += 1;
|
||||
let locator = format!("{name}:{ordinal}");
|
||||
let facts = if content {
|
||||
content_facts(&fields)?
|
||||
} else {
|
||||
category_facts(&fields)?
|
||||
};
|
||||
sink.emit(Record {
|
||||
native_id: locator,
|
||||
raw: json!(fields),
|
||||
facts,
|
||||
})?;
|
||||
}
|
||||
}
|
||||
// Read through the gzip trailer: tar EOF alone must not hide corruption.
|
||||
let mut gzip = archive.into_inner();
|
||||
let mut tail = [0; 8192];
|
||||
let mut trailing = 0usize;
|
||||
loop {
|
||||
let n = gzip.read(&mut tail)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
trailing += n;
|
||||
ensure!(
|
||||
trailing <= 1024 * 1024 && tail[..n].iter().all(|b| *b == 0),
|
||||
"unexpected archive trailing data"
|
||||
);
|
||||
}
|
||||
ensure!(entries > 0, "Curlie archive has no entries");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn content_facts(fields: &[&str]) -> anyhow::Result<Vec<Fact>> {
|
||||
ensure!(fields[3].parse::<u64>()? > 0, "invalid category ID");
|
||||
// Curlie identifies a site, not the legal organization operating it.
|
||||
let subject = entity_id(Source::Curlie, fields[0]);
|
||||
let values = [
|
||||
(
|
||||
"name",
|
||||
json!({"text":fields[1],"language":"und","kind":"label"}),
|
||||
"column:2",
|
||||
),
|
||||
(
|
||||
"website",
|
||||
json!({"url":fields[0],"relation":"directory_listing","category_id":fields[3],"statement":null}),
|
||||
"column:1",
|
||||
),
|
||||
(
|
||||
"description",
|
||||
json!({"text":fields[2],"category_id":fields[3]}),
|
||||
"column:3",
|
||||
),
|
||||
(
|
||||
"category_membership",
|
||||
json!({"category_id":fields[3]}),
|
||||
"column:4",
|
||||
),
|
||||
];
|
||||
Ok(values
|
||||
.into_iter()
|
||||
.map(|(p, v, s)| Fact {
|
||||
subject: subject.clone(),
|
||||
predicate: p.into(),
|
||||
value: v,
|
||||
selector: s.into(),
|
||||
confidence: 5000,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn category_facts(fields: &[&str]) -> anyhow::Result<Vec<Fact>> {
|
||||
ensure!(fields[0].parse::<u64>()? > 0, "invalid category ID");
|
||||
let count: u64 = fields[2].parse()?;
|
||||
let latitude = coordinate(fields[4], 90.0)?;
|
||||
let longitude = coordinate(fields[5], 180.0)?;
|
||||
ensure!(
|
||||
latitude.is_some() == longitude.is_some(),
|
||||
"incomplete coordinates"
|
||||
);
|
||||
Ok(vec![Fact {
|
||||
subject: format!("curlie:category:{}", fields[0]),
|
||||
predicate: "category".into(),
|
||||
value: json!({"category_id":fields[0],"path":fields[1],"entry_count":count,"description":fields[3],"latitude":latitude,"longitude":longitude}),
|
||||
selector: "columns:1-6".into(),
|
||||
confidence: 5000,
|
||||
}])
|
||||
}
|
||||
|
||||
fn coordinate(value: &str, maximum: f64) -> anyhow::Result<Option<f64>> {
|
||||
if value.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let coordinate: f64 = value.parse()?;
|
||||
ensure!(
|
||||
coordinate.is_finite() && coordinate.abs() <= maximum,
|
||||
"invalid geographic coordinate"
|
||||
);
|
||||
Ok(Some(coordinate))
|
||||
}
|
||||
76
crates/argand-site-registry/src/adapters/mod.rs
Normal file
76
crates/argand-site-registry/src/adapters/mod.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
// By Nic Weyand!
|
||||
//! Bounded source-specific readers behind one ingestion interface.
|
||||
|
||||
use crate::model::{Fact, Format, Record};
|
||||
use anyhow::ensure;
|
||||
use serde_json::json;
|
||||
use std::io::{BufRead, Read};
|
||||
pub(crate) mod csv_sources;
|
||||
mod curlie;
|
||||
mod wikidata;
|
||||
|
||||
/// Maximum decompressed size of one entity, row, or metadata document.
|
||||
pub const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// Transactional sink; an emitted record and all its facts share one checkpoint.
|
||||
pub trait RecordSink {
|
||||
/// Persists one source record.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns storage or record-validation errors.
|
||||
fn emit(&mut self, record: Record) -> anyhow::Result<()>;
|
||||
}
|
||||
|
||||
/// Common streaming import interface. Readers never fetch website assertions.
|
||||
pub trait SourceAdapter {
|
||||
/// Reads the complete input, rejecting malformed or truncated records.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns parser, input, or sink errors.
|
||||
fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()>;
|
||||
}
|
||||
|
||||
/// Chooses the explicit source format adapter.
|
||||
#[must_use]
|
||||
pub fn adapter(format: Format) -> Box<dyn SourceAdapter> {
|
||||
match format {
|
||||
Format::WikidataDump => Box::new(wikidata::Wikidata { dump: true }),
|
||||
Format::WikidataEntities => Box::new(wikidata::Wikidata { dump: false }),
|
||||
Format::MajesticCsv => Box::new(csv_sources::CsvSource { crux: false }),
|
||||
Format::CruxCsv => Box::new(csv_sources::CsvSource { crux: true }),
|
||||
Format::CurlieTarGz => Box::new(curlie::Curlie),
|
||||
Format::PslText => Box::new(PslAdapter),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn bounded_line(input: &mut dyn BufRead, buffer: &mut String) -> anyhow::Result<usize> {
|
||||
buffer.clear();
|
||||
let n = input
|
||||
.take((MAX_RECORD_BYTES + 1) as u64)
|
||||
.read_line(buffer)?;
|
||||
ensure!(n <= MAX_RECORD_BYTES, "record exceeds 16 MiB");
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
struct PslAdapter;
|
||||
impl SourceAdapter for PslAdapter {
|
||||
fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> {
|
||||
let mut text = String::new();
|
||||
input
|
||||
.take((MAX_RECORD_BYTES + 1) as u64)
|
||||
.read_to_string(&mut text)?;
|
||||
ensure!(text.len() <= MAX_RECORD_BYTES, "PSL exceeds bound");
|
||||
crate::normalize::Normalizer::new(text.as_bytes(), String::new())?;
|
||||
sink.emit(Record {
|
||||
native_id: "public_suffix_list.dat".into(),
|
||||
raw: json!(text),
|
||||
facts: vec![Fact {
|
||||
subject: "psl".into(),
|
||||
predicate: "psl".into(),
|
||||
value: json!(text),
|
||||
selector: String::new(),
|
||||
confidence: 10000,
|
||||
}],
|
||||
})
|
||||
}
|
||||
}
|
||||
186
crates/argand-site-registry/src/adapters/wikidata.rs
Normal file
186
crates/argand-site-registry/src/adapters/wikidata.rs
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
// By Nic Weyand!
|
||||
//! Wikibase JSON, retaining complete statement qualifiers, references, and rank.
|
||||
|
||||
use super::{MAX_RECORD_BYTES, RecordSink, SourceAdapter, bounded_line};
|
||||
use crate::model::{Fact, Record, Source, entity_id};
|
||||
use anyhow::{Context, ensure};
|
||||
use serde_json::{Value, json};
|
||||
use std::io::{BufRead, Read};
|
||||
|
||||
pub(super) struct Wikidata {
|
||||
pub dump: bool,
|
||||
}
|
||||
|
||||
impl SourceAdapter for Wikidata {
|
||||
fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> {
|
||||
if !self.dump {
|
||||
let mut raw = Vec::new();
|
||||
input
|
||||
.take((MAX_RECORD_BYTES + 1) as u64)
|
||||
.read_to_end(&mut raw)?;
|
||||
ensure!(
|
||||
raw.len() <= MAX_RECORD_BYTES,
|
||||
"entity response exceeds 16 MiB; use dump format"
|
||||
);
|
||||
let value = crate::json::parse(&raw)?;
|
||||
for (key, entity) in value["entities"]
|
||||
.as_object()
|
||||
.context("missing entities object")?
|
||||
{
|
||||
ensure!(
|
||||
entity["id"].as_str() == Some(key),
|
||||
"entity map key disagrees with native ID"
|
||||
);
|
||||
project(entity, sink, true)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let mut line = String::new();
|
||||
bounded_line(input, &mut line)?;
|
||||
ensure!(line.trim() == "[", "dump must begin with [");
|
||||
let mut seen = false;
|
||||
let mut comma = false;
|
||||
loop {
|
||||
ensure!(
|
||||
bounded_line(input, &mut line)? > 0,
|
||||
"truncated Wikidata dump"
|
||||
);
|
||||
let row = line.trim();
|
||||
if row.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if row == "]" {
|
||||
ensure!(!comma && seen, "empty dump or trailing comma");
|
||||
while bounded_line(input, &mut line)? > 0 {
|
||||
ensure!(line.trim().is_empty(), "data after dump");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(!seen || comma, "missing entity separator");
|
||||
comma = row.ends_with(',');
|
||||
let entity = crate::json::parse(row.strip_suffix(',').unwrap_or(row).as_bytes())?;
|
||||
project(&entity, sink, false)?;
|
||||
seen = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn project(raw: &Value, sink: &mut dyn RecordSink, explicit_selection: bool) -> anyhow::Result<()> {
|
||||
let id = raw["id"].as_str().context("missing Wikidata ID")?;
|
||||
ensure!(
|
||||
id.len() > 1
|
||||
&& matches!(id.as_bytes()[0], b'Q' | b'P' | b'L')
|
||||
&& id.as_bytes()[1] != b'0'
|
||||
&& id[1..].bytes().all(|b| b.is_ascii_digit()),
|
||||
"invalid Wikidata ID"
|
||||
);
|
||||
let claims = raw.get("claims").and_then(Value::as_object);
|
||||
let mut facts = Vec::new();
|
||||
let subject = entity_id(Source::Wikidata, id);
|
||||
// Preserve all names of relevant entities and all small code-mapping records.
|
||||
let relevant = claims.is_some_and(|c| {
|
||||
["P856", "P297", "P218", "P219", "P220"]
|
||||
.iter()
|
||||
.any(|p| c.contains_key(*p))
|
||||
});
|
||||
if !relevant {
|
||||
// A valid entity response with removed websites/deleted entity is a
|
||||
// retirement observation. It must supersede the old selection instead
|
||||
// of looking like an empty failed download that keeps old links active.
|
||||
if explicit_selection {
|
||||
sink.emit(Record {
|
||||
native_id: id.into(),
|
||||
raw: raw.clone(),
|
||||
facts: Vec::new(),
|
||||
})?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
for (field, kind) in [("labels", "label"), ("aliases", "alias")] {
|
||||
if let Some(names) = raw.get(field).and_then(Value::as_object) {
|
||||
for (language, values) in names {
|
||||
let names: Vec<&Value> = if kind == "alias" {
|
||||
values
|
||||
.as_array()
|
||||
.context("aliases must be arrays")?
|
||||
.iter()
|
||||
.collect()
|
||||
} else {
|
||||
vec![values]
|
||||
};
|
||||
for (i, value) in names.into_iter().enumerate() {
|
||||
ensure!(
|
||||
value["language"].as_str() == Some(language),
|
||||
"name language mismatch"
|
||||
);
|
||||
let text = value["value"].as_str().context("name must be text")?;
|
||||
let pointer_language = language.replace('~', "~0").replace('/', "~1");
|
||||
facts.push(Fact {
|
||||
subject: subject.clone(),
|
||||
predicate: "name".into(),
|
||||
value: json!({"text":text,"language":language,"kind":kind,"native_id":id}),
|
||||
selector: if kind == "alias" {
|
||||
format!("/{field}/{pointer_language}/{i}")
|
||||
} else {
|
||||
format!("/{field}/{pointer_language}")
|
||||
},
|
||||
confidence: 8000,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(claims) = claims {
|
||||
for property in [
|
||||
"P856", "P17", "P159", "P407", "P1001", "P297", "P218", "P219", "P220",
|
||||
] {
|
||||
if let Some(statements) = claims.get(property) {
|
||||
for (i, statement) in statements
|
||||
.as_array()
|
||||
.context("claims must be arrays")?
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let predicate = if property == "P856" {
|
||||
"website"
|
||||
} else {
|
||||
property
|
||||
};
|
||||
let value = if property == "P856" {
|
||||
website_value(statement, id, raw)?
|
||||
} else {
|
||||
json!({"native_id":id,"statement":statement})
|
||||
};
|
||||
facts.push(Fact {
|
||||
subject: subject.clone(),
|
||||
predicate: predicate.into(),
|
||||
value,
|
||||
selector: format!("/claims/{property}/{i}"),
|
||||
confidence: 5000,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sink.emit(Record {
|
||||
native_id: id.into(),
|
||||
raw: raw.clone(),
|
||||
facts,
|
||||
})
|
||||
}
|
||||
|
||||
fn website_value(statement: &Value, id: &str, raw: &Value) -> anyhow::Result<Value> {
|
||||
let snak = &statement["mainsnak"];
|
||||
ensure!(
|
||||
snak["property"].as_str() == Some("P856"),
|
||||
"mismatched website property"
|
||||
);
|
||||
let url = if snak["snaktype"] == "value"
|
||||
&& snak.pointer("/datavalue/type").and_then(Value::as_str) == Some("string")
|
||||
{
|
||||
snak.pointer("/datavalue/value")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(json!({"url":url,"native_id":id,"statement":statement,"revision":raw.get("lastrevid")}))
|
||||
}
|
||||
359
crates/argand-site-registry/src/build.rs
Normal file
359
crates/argand-site-registry/src/build.rs
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
// By Nic Weyand!
|
||||
//! Immutable SQLite projections, with source evidence left intact.
|
||||
|
||||
use crate::{
|
||||
model::SourceManifest,
|
||||
normalize::{Normalizer, name_key},
|
||||
store,
|
||||
};
|
||||
use anyhow::{Context, ensure};
|
||||
use rusqlite::{Connection, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
fs::{self, File},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
const PROJECTIONS: &str = "
|
||||
CREATE TABLE selected_sources(id TEXT PRIMARY KEY REFERENCES sources(id)) STRICT;
|
||||
INSERT INTO selected_sources SELECT id FROM (
|
||||
SELECT id,row_number() OVER(PARTITION BY source,scope ORDER BY retrieved_at DESC,id DESC) AS position
|
||||
FROM sources WHERE complete=1) WHERE position=1;
|
||||
CREATE TABLE names(entity TEXT NOT NULL,key TEXT NOT NULL,text TEXT NOT NULL,language TEXT NOT NULL,kind TEXT NOT NULL,fact TEXT NOT NULL REFERENCES facts(id),PRIMARY KEY(entity,fact)) STRICT;
|
||||
CREATE INDEX name_lookup ON names(key,entity);
|
||||
CREATE TABLE entities(id TEXT PRIMARY KEY,canonical_name TEXT NOT NULL,names_fingerprint TEXT NOT NULL) STRICT;
|
||||
CREATE TABLE properties(id TEXT PRIMARY KEY,url TEXT NOT NULL UNIQUE,hostname TEXT NOT NULL,domain TEXT NOT NULL,suffix TEXT NOT NULL,derived_json TEXT NOT NULL) STRICT;
|
||||
CREATE INDEX property_host ON properties(hostname);
|
||||
CREATE TABLE edges(fingerprint TEXT PRIMARY KEY,entity TEXT NOT NULL REFERENCES entities(id),property TEXT NOT NULL REFERENCES properties(id),relation TEXT NOT NULL,facts TEXT NOT NULL,evidence TEXT NOT NULL,eligible INTEGER NOT NULL CHECK(eligible IN(0,1))) STRICT;
|
||||
CREATE INDEX edge_entity ON edges(entity,property);
|
||||
CREATE TABLE popularity(fact TEXT PRIMARY KEY REFERENCES facts(id),source TEXT NOT NULL,target TEXT NOT NULL,hostname TEXT NOT NULL,domain TEXT NOT NULL,value TEXT NOT NULL,derived_json TEXT NOT NULL) STRICT;
|
||||
CREATE INDEX popularity_host ON popularity(hostname,source);
|
||||
CREATE TABLE rejected(fact TEXT PRIMARY KEY REFERENCES facts(id),reason TEXT NOT NULL) STRICT;
|
||||
";
|
||||
|
||||
/// Completion receipt. Authenticity needs an external digest or trusted signature.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Receipt {
|
||||
/// `argand.site-registry/v1`.
|
||||
pub schema: String,
|
||||
/// Parser/derivation contract.
|
||||
pub rules: String,
|
||||
/// Database content digest.
|
||||
pub database_sha256: String,
|
||||
/// Digest of the shipped source-license document.
|
||||
pub licenses_sha256: String,
|
||||
/// Digest of the shipped machine-readable attribution envelope.
|
||||
pub attribution_sha256: String,
|
||||
/// PSL source manifest identity.
|
||||
pub psl_source: String,
|
||||
/// Active source snapshot declarations.
|
||||
pub sources: Vec<SourceManifest>,
|
||||
/// Distinct entity count.
|
||||
pub entities: u64,
|
||||
/// Strict URL identity count.
|
||||
pub properties: u64,
|
||||
/// Edge assertion groups, including ineligible evidence.
|
||||
pub edges: u64,
|
||||
/// Rejected projection facts, retained in the assertion tables.
|
||||
pub rejected: u64,
|
||||
}
|
||||
|
||||
/// Builds a fresh generation from a consistent snapshot of committed imports.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects existing destinations, incomplete PSL, corrupt stores, and I/O failures.
|
||||
pub fn build(db: &Connection, output: &Path) -> anyhow::Result<Receipt> {
|
||||
fs::create_dir(output).context("generation destination must not exist")?;
|
||||
let path = output.join("registry.sqlite");
|
||||
let snapshot = store::open(&path)?;
|
||||
copy_canonical(db, &snapshot)?;
|
||||
snapshot.execute_batch(PROJECTIONS)?;
|
||||
let (psl_id,text): (String,String)=snapshot.query_row("SELECT f.source_id,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl' ORDER BY f.source_id LIMIT 1",[],|r| Ok((r.get(0)?,r.get(1)?))).context("import a complete PSL snapshot first")?;
|
||||
let psl_count:u64=snapshot.query_row("SELECT count(*) FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl'",[],|r|store::unsigned(r,0))?;
|
||||
ensure!(psl_count == 1, "exactly one active PSL snapshot required");
|
||||
let psl_text: String = serde_json::from_str(&text)?;
|
||||
let normalizer = Normalizer::new(psl_text.as_bytes(), psl_id.clone())?;
|
||||
snapshot.execute_batch("BEGIN")?;
|
||||
project_names(&snapshot)?;
|
||||
project_facts(&snapshot, &normalizer)?;
|
||||
snapshot.execute_batch("COMMIT; ANALYZE;")?;
|
||||
let check: String = snapshot.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "registry integrity failed");
|
||||
let foreign_count: u64 =
|
||||
snapshot.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| {
|
||||
store::unsigned(r, 0)
|
||||
})?;
|
||||
ensure!(foreign_count == 0, "registry foreign keys failed");
|
||||
let mut statement = snapshot
|
||||
.prepare("SELECT manifest FROM sources JOIN selected_sources USING(id) ORDER BY id")?;
|
||||
let sources = statement
|
||||
.query_map([], |r| r.get::<_, String>(0))?
|
||||
.map(|s| Ok(serde_json::from_str(&s?)?))
|
||||
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||
drop(statement);
|
||||
let mut receipt = Receipt {
|
||||
schema: "argand.site-registry/v1".into(),
|
||||
rules: store::RULE_VERSION.into(),
|
||||
database_sha256: String::new(),
|
||||
licenses_sha256: crate::digest(crate::release::LICENSES.as_bytes()),
|
||||
attribution_sha256: crate::digest(&serde_json::to_vec_pretty(
|
||||
&crate::release::attribution(),
|
||||
)?),
|
||||
psl_source: psl_id,
|
||||
sources,
|
||||
entities: count(&snapshot, "entities")?,
|
||||
properties: count(&snapshot, "properties")?,
|
||||
edges: count(&snapshot, "edges")?,
|
||||
rejected: count(&snapshot, "rejected")?,
|
||||
};
|
||||
snapshot.close().map_err(|(_, e)| e)?;
|
||||
File::open(&path)?.sync_all()?;
|
||||
receipt.database_sha256 = crate::file_digest(&path)?;
|
||||
argand_atomic::create_durable(
|
||||
&output.join("LICENSE_SOURCES.md"),
|
||||
crate::release::LICENSES.as_bytes(),
|
||||
)?;
|
||||
argand_atomic::create_durable(
|
||||
&output.join("ATTRIBUTION.json"),
|
||||
&serde_json::to_vec_pretty(&crate::release::attribution())?,
|
||||
)?;
|
||||
argand_atomic::create_durable(
|
||||
&output.join("COMPLETE.json"),
|
||||
&serde_json::to_vec_pretty(&receipt)?,
|
||||
)?;
|
||||
Ok(receipt)
|
||||
}
|
||||
|
||||
fn copy_canonical(source: &Connection, destination: &Connection) -> anyhow::Result<()> {
|
||||
// Sorted logical copy prevents physical insertion order or failed imports
|
||||
// from changing generation bytes. Hold one read transaction across tables.
|
||||
let source_transaction = source.unchecked_transaction()?;
|
||||
let destination_transaction = destination.unchecked_transaction()?;
|
||||
let tables = [
|
||||
(
|
||||
"sources",
|
||||
"SELECT * FROM sources WHERE complete=1 ORDER BY id",
|
||||
7,
|
||||
),
|
||||
(
|
||||
"records",
|
||||
"SELECT r.* FROM records r JOIN sources s ON s.id=r.source_id WHERE s.complete=1 ORDER BY r.source_id,r.ordinal",
|
||||
4,
|
||||
),
|
||||
(
|
||||
"facts",
|
||||
"SELECT f.* FROM facts f JOIN sources s ON s.id=f.source_id WHERE s.complete=1 ORDER BY f.id",
|
||||
8,
|
||||
),
|
||||
("reviews", "SELECT * FROM reviews ORDER BY sequence", 11),
|
||||
(
|
||||
"equivalences",
|
||||
"SELECT * FROM equivalences ORDER BY fingerprint",
|
||||
5,
|
||||
),
|
||||
];
|
||||
for (table, select, columns) in tables {
|
||||
let placeholders = (1..=columns)
|
||||
.map(|i| format!("?{i}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let mut insert =
|
||||
destination.prepare(&format!("INSERT INTO {table} VALUES({placeholders})"))?;
|
||||
let mut select = source.prepare(select)?;
|
||||
let mut rows = select.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let values = (0..columns)
|
||||
.map(|i| row.get::<_, rusqlite::types::Value>(i))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
insert.execute(rusqlite::params_from_iter(values))?;
|
||||
}
|
||||
}
|
||||
destination_transaction.commit()?;
|
||||
source_transaction.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn count(db: &Connection, table: &str) -> anyhow::Result<u64> {
|
||||
Ok(
|
||||
db.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| {
|
||||
store::unsigned(r, 0)
|
||||
})?,
|
||||
)
|
||||
}
|
||||
|
||||
fn project_names(db: &Connection) -> anyhow::Result<()> {
|
||||
let mut stmt=db.prepare("SELECT f.id,f.subject,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='name' ORDER BY f.subject,f.id")?;
|
||||
let mut rows = stmt.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let (id, subject, raw): (String, String, String) = (row.get(0)?, row.get(1)?, row.get(2)?);
|
||||
let value: Value = serde_json::from_str(&raw)?;
|
||||
let text = value["text"].as_str().context("name text missing")?;
|
||||
match name_key(text) {
|
||||
Ok(key) => {
|
||||
db.execute(
|
||||
"INSERT INTO names VALUES(?1,?2,?3,?4,?5,?6)",
|
||||
params![
|
||||
subject,
|
||||
key,
|
||||
text,
|
||||
value["language"].as_str().unwrap_or("und"),
|
||||
value["kind"].as_str().unwrap_or("label"),
|
||||
id
|
||||
],
|
||||
)?;
|
||||
}
|
||||
Err(error) => {
|
||||
db.execute(
|
||||
"INSERT INTO rejected VALUES(?1,?2)",
|
||||
params![id, error.to_string()],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
// All entities with website assertions exist even when names are absent.
|
||||
let mut entities=db.prepare("SELECT DISTINCT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate IN('website','name') ORDER BY f.subject")?;
|
||||
for subject in entities.query_map([], |r| r.get::<_, String>(0))? {
|
||||
let subject = subject?;
|
||||
let mut names=db.prepare("SELECT DISTINCT text,language,kind FROM names WHERE entity=?1 ORDER BY CASE WHEN kind='label' THEN 0 ELSE 1 END,CASE WHEN language='en' THEN 0 ELSE 1 END,language,text")?;
|
||||
let values = names
|
||||
.query_map([&subject], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, String>(2)?,
|
||||
))
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let canonical = values.first().map_or(subject.as_str(), |v| v.0.as_str());
|
||||
let fingerprint = crate::digest(&serde_json::to_vec(&values)?);
|
||||
db.execute(
|
||||
"INSERT INTO entities VALUES(?1,?2,?3)",
|
||||
params![subject, canonical, fingerprint],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn project_facts(db: &Connection, normalizer: &Normalizer) -> anyhow::Result<()> {
|
||||
let mut stmt=db.prepare("SELECT f.id,f.subject,f.predicate,f.value,s.source,f.selector,r.native_id FROM facts f JOIN sources s ON s.id=f.source_id JOIN selected_sources a ON a.id=s.id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE f.predicate IN('website','popularity') ORDER BY f.subject,f.id")?;
|
||||
let mut rows = stmt.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let id: String = row.get(0)?;
|
||||
let subject: String = row.get(1)?;
|
||||
let predicate: String = row.get(2)?;
|
||||
let raw: String = row.get(3)?;
|
||||
let value: Value = serde_json::from_str(&raw)?;
|
||||
let source: String = row.get(4)?;
|
||||
let result = if predicate == "website" {
|
||||
project_edge(
|
||||
db,
|
||||
normalizer,
|
||||
&id,
|
||||
&subject,
|
||||
&value,
|
||||
&source,
|
||||
&row.get::<_, String>(5)?,
|
||||
&row.get::<_, String>(6)?,
|
||||
)
|
||||
} else {
|
||||
project_popularity(db, normalizer, &id, &source, &value)
|
||||
};
|
||||
if let Err(error) = result {
|
||||
// SQL errors are operational failures, never silently quarantined data.
|
||||
if error.downcast_ref::<rusqlite::Error>().is_some() {
|
||||
return Err(error);
|
||||
}
|
||||
db.execute(
|
||||
"INSERT INTO rejected VALUES(?1,?2)",
|
||||
params![id, error.to_string()],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn project_popularity(
|
||||
db: &Connection,
|
||||
n: &Normalizer,
|
||||
id: &str,
|
||||
source: &str,
|
||||
value: &Value,
|
||||
) -> anyhow::Result<()> {
|
||||
let target = value["target"]
|
||||
.as_str()
|
||||
.context("missing popularity target")?;
|
||||
let domain = if value["target_kind"] == "origin" {
|
||||
n.url(target)?.domain
|
||||
} else {
|
||||
n.domain(target)?
|
||||
};
|
||||
db.execute(
|
||||
"INSERT INTO popularity VALUES(?1,?2,?3,?4,?5,?6,?7)",
|
||||
params![
|
||||
id,
|
||||
source,
|
||||
target,
|
||||
domain.hostname,
|
||||
domain.registrable_domain,
|
||||
serde_json::to_string(value)?,
|
||||
serde_json::to_string(&domain)?
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)] // One complete source assertion, kept explicit for auditability.
|
||||
fn project_edge(
|
||||
db: &Connection,
|
||||
n: &Normalizer,
|
||||
id: &str,
|
||||
entity: &str,
|
||||
value: &Value,
|
||||
source: &str,
|
||||
selector: &str,
|
||||
native: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let url = value["url"]
|
||||
.as_str()
|
||||
.context("website statement has no concrete URL")?;
|
||||
let property = n.url(url)?;
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO properties VALUES(?1,?2,?3,?4,?5,?6)",
|
||||
params![
|
||||
property.id,
|
||||
property.url,
|
||||
property.domain.hostname,
|
||||
property.domain.registrable_domain,
|
||||
property.domain.public_suffix,
|
||||
serde_json::to_string(&property)?
|
||||
],
|
||||
)?;
|
||||
let names: String = db.query_row(
|
||||
"SELECT names_fingerprint FROM entities WHERE id=?1",
|
||||
[entity],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let relation = if source == "wikidata" {
|
||||
"asserted_official"
|
||||
} else {
|
||||
"directory_listing"
|
||||
};
|
||||
let evidence = json!({"source":source,"native_id":native,"selector":selector,"assertion":value,"names_fingerprint":names,"normalization":store::RULE_VERSION});
|
||||
let mut identity_property = property.clone();
|
||||
// Bind review to actual PSL bytes and derived fields, not a fresh timestamp
|
||||
// for an otherwise identical list. Full retrieval provenance stays on property.
|
||||
identity_property.domain.psl_source.clear();
|
||||
let fingerprint = crate::digest(&serde_json::to_vec(&(
|
||||
entity,
|
||||
&identity_property,
|
||||
&evidence,
|
||||
))?);
|
||||
// End-dated assertions stay as historical evidence, never current destinations.
|
||||
// Future/partial starts require the operator to inspect the retained qualifiers.
|
||||
let eligible = value.pointer("/statement/rank").and_then(Value::as_str) != Some("deprecated")
|
||||
&& value.pointer("/statement/qualifiers/P582").is_none();
|
||||
db.execute("INSERT INTO edges VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(fingerprint) DO UPDATE SET facts=json_insert(edges.facts,'$[#]',?8)",params![fingerprint,entity,property.id,relation,serde_json::to_string(&vec![id])?,serde_json::to_string(&evidence)?,eligible,id])?;
|
||||
Ok(())
|
||||
}
|
||||
380
crates/argand-site-registry/src/cli.rs
Normal file
380
crates/argand-site-registry/src/cli.rs
Normal file
|
|
@ -0,0 +1,380 @@
|
|||
// By Nic Weyand!
|
||||
//! Explicit source acquisition, import, review, and immutable generation commands.
|
||||
|
||||
use anyhow::ensure;
|
||||
use argand_site_registry as registry;
|
||||
use clap::{Parser, Subcommand};
|
||||
use registry::{
|
||||
model::{Compression, Format, Source, SourceManifest},
|
||||
query::Registry,
|
||||
};
|
||||
use std::{io::Write, path::PathBuf};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about = "Provenance-preserving, reviewed entity ↔ website registry"
|
||||
)]
|
||||
struct Args {
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Command {
|
||||
/// Preview an explicit entity equivalence, or record its exact review JSON.
|
||||
Equivalence {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
left: String,
|
||||
#[arg(long)]
|
||||
right: String,
|
||||
#[arg(long, requires = "database")]
|
||||
decision: Option<PathBuf>,
|
||||
#[arg(long)]
|
||||
database: Option<PathBuf>,
|
||||
},
|
||||
/// Download one allowlisted source into an immutable local cache.
|
||||
Download {
|
||||
#[arg(long)]
|
||||
cache: PathBuf,
|
||||
#[arg(long, value_enum)]
|
||||
source: Source,
|
||||
#[arg(long, value_enum)]
|
||||
format: Format,
|
||||
#[arg(long, value_enum, default_value = "none")]
|
||||
compression: Compression,
|
||||
#[arg(long)]
|
||||
url: String,
|
||||
#[arg(long)]
|
||||
snapshot: String,
|
||||
#[arg(long)]
|
||||
scope: String,
|
||||
#[arg(long)]
|
||||
maximum_bytes: u64,
|
||||
},
|
||||
/// Download paginated `CrUX` data using an explicit billing configuration JSON.
|
||||
CruxDownload {
|
||||
#[arg(long)]
|
||||
cache: PathBuf,
|
||||
#[arg(long)]
|
||||
request: PathBuf,
|
||||
},
|
||||
/// Declare a pinned local source; does not certify ownership or publisher trust.
|
||||
Manifest {
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
#[arg(long)]
|
||||
output: PathBuf,
|
||||
#[arg(long, value_enum)]
|
||||
source: Source,
|
||||
#[arg(long, value_enum)]
|
||||
format: Format,
|
||||
#[arg(long, value_enum, default_value = "none")]
|
||||
compression: Compression,
|
||||
#[arg(long)]
|
||||
source_url: String,
|
||||
#[arg(long)]
|
||||
snapshot: String,
|
||||
#[arg(long)]
|
||||
scope: String,
|
||||
#[arg(long)]
|
||||
retrieved_at: chrono::DateTime<chrono::Utc>,
|
||||
},
|
||||
/// Import a complete pinned source, resuming committed record batches.
|
||||
Import {
|
||||
#[arg(long)]
|
||||
database: PathBuf,
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
#[arg(long)]
|
||||
manifest: PathBuf,
|
||||
},
|
||||
/// Build a new immutable generation; output must not exist.
|
||||
Build {
|
||||
#[arg(long)]
|
||||
database: PathBuf,
|
||||
#[arg(long)]
|
||||
output: PathBuf,
|
||||
},
|
||||
/// Audit an exact name or alias; includes ambiguity counts and attribution.
|
||||
Lookup {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
query: String,
|
||||
#[arg(long, default_value_t = 20)]
|
||||
limit: u32,
|
||||
},
|
||||
/// Resolve only an unambiguous, explicitly reviewed, unexpired property.
|
||||
Resolve {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
query: String,
|
||||
#[arg(long)]
|
||||
locale: Option<String>,
|
||||
#[arg(long)]
|
||||
country: Option<String>,
|
||||
},
|
||||
/// Append an exact assertion approval or revocation from a review JSON file.
|
||||
Review {
|
||||
#[arg(long)]
|
||||
database: PathBuf,
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
decision: PathBuf,
|
||||
},
|
||||
/// Export facts as streaming JSONL with source licenses and attribution.
|
||||
Export {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
output: PathBuf,
|
||||
#[arg(long)]
|
||||
include_descriptions: bool,
|
||||
},
|
||||
/// Show added/removed evidence fingerprints without loading either registry.
|
||||
Diff {
|
||||
#[arg(long)]
|
||||
old: PathBuf,
|
||||
#[arg(long)]
|
||||
old_pin: String,
|
||||
#[arg(long)]
|
||||
new: PathBuf,
|
||||
#[arg(long)]
|
||||
new_pin: String,
|
||||
},
|
||||
/// Verify complete database bytes against an external receipt pin.
|
||||
Verify {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
},
|
||||
/// Sign an independently reviewed generation with an SSH signing key.
|
||||
Sign {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
pin: String,
|
||||
#[arg(long)]
|
||||
key: PathBuf,
|
||||
},
|
||||
/// Verify publisher signature and atomically activate (also supports rollback).
|
||||
Activate {
|
||||
#[arg(long)]
|
||||
generation: PathBuf,
|
||||
#[arg(long)]
|
||||
current: PathBuf,
|
||||
#[arg(long)]
|
||||
allowed_signers: PathBuf,
|
||||
#[arg(long)]
|
||||
identity: String,
|
||||
},
|
||||
/// Run configured acquisition/import/build; never approves or activates.
|
||||
Update {
|
||||
#[arg(long)]
|
||||
config: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
pub(super) async fn run() -> anyhow::Result<()> {
|
||||
let value = match Args::parse().command {
|
||||
command @ (Command::Download { .. }
|
||||
| Command::CruxDownload { .. }
|
||||
| Command::Manifest { .. }
|
||||
| Command::Equivalence { .. }) => acquire(command).await?,
|
||||
Command::Import {
|
||||
database,
|
||||
input,
|
||||
manifest,
|
||||
} => {
|
||||
serde_json::json!({"source_id":registry::store::import(&mut registry::store::open(&database)?,®istry::read_json(&manifest)?,&input)?})
|
||||
}
|
||||
Command::Build { database, output } => {
|
||||
ensure!(database.is_file(), "import database does not exist");
|
||||
let receipt = registry::build::build(®istry::store::open(&database)?, &output)?;
|
||||
serde_json::json!({"receipt":receipt,"pin":registry::file_digest(&output.join("COMPLETE.json"))?,"generation":output})
|
||||
}
|
||||
Command::Lookup {
|
||||
generation,
|
||||
pin,
|
||||
query,
|
||||
limit,
|
||||
} => serde_json::to_value(Registry::open(&generation, &pin)?.lookup(&query, limit)?)?,
|
||||
Command::Resolve {
|
||||
generation,
|
||||
pin,
|
||||
query,
|
||||
locale,
|
||||
country,
|
||||
} => {
|
||||
serde_json::json!({"destination":Registry::open(&generation,&pin)?.resolve(&query,locale.as_deref(),country.as_deref(),chrono::Utc::now())?,"attribution":registry::release::attribution()})
|
||||
}
|
||||
Command::Review {
|
||||
database,
|
||||
generation,
|
||||
pin,
|
||||
decision,
|
||||
} => {
|
||||
serde_json::json!({"review_sequence":registry::review::record(®istry::store::open(&database)?,&Registry::open(&generation,&pin)?,®istry::read_json(&decision)?)?,"rebuild_required":true})
|
||||
}
|
||||
Command::Export {
|
||||
generation,
|
||||
pin,
|
||||
output,
|
||||
include_descriptions,
|
||||
} => {
|
||||
registry::release::export(
|
||||
&Registry::open(&generation, &pin)?,
|
||||
&output,
|
||||
include_descriptions,
|
||||
)?;
|
||||
serde_json::json!({"export":output})
|
||||
}
|
||||
Command::Diff {
|
||||
old,
|
||||
old_pin,
|
||||
new,
|
||||
new_pin,
|
||||
} => {
|
||||
registry::release::diff(
|
||||
&Registry::open(&old, &old_pin)?,
|
||||
&Registry::open(&new, &new_pin)?,
|
||||
&mut std::io::stdout().lock(),
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
Command::Verify { generation, pin } => {
|
||||
serde_json::to_value(Registry::open(&generation, &pin)?.receipt)?
|
||||
}
|
||||
Command::Sign {
|
||||
generation,
|
||||
pin,
|
||||
key,
|
||||
} => {
|
||||
registry::release::sign(&generation, &key, &pin)?;
|
||||
serde_json::json!({"signed":generation})
|
||||
}
|
||||
Command::Activate {
|
||||
generation,
|
||||
current,
|
||||
allowed_signers,
|
||||
identity,
|
||||
} => {
|
||||
registry::release::activate(&generation, ¤t, &allowed_signers, &identity)?;
|
||||
serde_json::json!({"current":current})
|
||||
}
|
||||
Command::Update { config } => {
|
||||
let config = read_config(&config)?;
|
||||
serde_json::json!({"candidate":registry::update::run(&config).await?})
|
||||
}
|
||||
};
|
||||
writeln!(
|
||||
std::io::stdout().lock(),
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&value)?
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn acquire(command: Command) -> anyhow::Result<serde_json::Value> {
|
||||
Ok(match command {
|
||||
Command::Equivalence {
|
||||
generation,
|
||||
pin,
|
||||
left,
|
||||
right,
|
||||
decision,
|
||||
database,
|
||||
} => {
|
||||
let registry = Registry::open(&generation, &pin)?;
|
||||
let pair = registry::identity::propose(®istry, &left, &right)?;
|
||||
if let Some(decision) = decision {
|
||||
let database = database
|
||||
.ok_or_else(|| anyhow::anyhow!("identity review needs a writer database"))?;
|
||||
serde_json::json!({"review_sequence":registry::identity::record(®istry::store::open(&database)?,®istry,&pair,®istry::read_json(&decision)?)?,"rebuild_required":true})
|
||||
} else {
|
||||
serde_json::to_value(pair)?
|
||||
}
|
||||
}
|
||||
Command::Download {
|
||||
cache,
|
||||
source,
|
||||
format,
|
||||
compression,
|
||||
url,
|
||||
snapshot,
|
||||
scope,
|
||||
maximum_bytes,
|
||||
} => serde_json::to_value(
|
||||
registry::download::download(
|
||||
&cache,
|
||||
®istry::download::Download {
|
||||
source,
|
||||
format,
|
||||
compression,
|
||||
url,
|
||||
snapshot,
|
||||
scope,
|
||||
maximum_bytes,
|
||||
},
|
||||
)
|
||||
.await?,
|
||||
)?,
|
||||
Command::CruxDownload { cache, request } => serde_json::to_value(
|
||||
registry::crux::download(&cache, ®istry::read_json(&request)?).await?,
|
||||
)?,
|
||||
Command::Manifest {
|
||||
input,
|
||||
output,
|
||||
source,
|
||||
format,
|
||||
compression,
|
||||
source_url,
|
||||
snapshot,
|
||||
scope,
|
||||
retrieved_at,
|
||||
} => {
|
||||
let manifest = SourceManifest {
|
||||
schema: "argand.site-source/v1".into(),
|
||||
source,
|
||||
format,
|
||||
compression,
|
||||
source_url,
|
||||
snapshot,
|
||||
scope,
|
||||
retrieved_at,
|
||||
license: source.license().into(),
|
||||
license_url: source.license_url().into(),
|
||||
sha256: registry::file_digest(&input)?,
|
||||
bytes: input.metadata()?.len(),
|
||||
};
|
||||
manifest.validate()?;
|
||||
argand_atomic::create_durable(&output, &serde_json::to_vec_pretty(&manifest)?)?;
|
||||
serde_json::to_value(manifest)?
|
||||
}
|
||||
_ => anyhow::bail!("expected an acquisition command"),
|
||||
})
|
||||
}
|
||||
|
||||
fn read_config(path: &std::path::Path) -> anyhow::Result<registry::update::Config> {
|
||||
ensure!(
|
||||
path.metadata()?.len() <= 1024 * 1024,
|
||||
"config exceeds 1 MiB"
|
||||
);
|
||||
Ok(toml::from_str(&std::fs::read_to_string(path)?)?)
|
||||
}
|
||||
353
crates/argand-site-registry/src/crux.rs
Normal file
353
crates/argand-site-registry/src/crux.rs
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
// By Nic Weyand!
|
||||
//! Official `BigQuery` `CrUX` projection, bounded pagination and idempotent job IDs.
|
||||
|
||||
use crate::{
|
||||
download::{CachedSource, client},
|
||||
model::{Compression, Format, Source, SourceManifest},
|
||||
};
|
||||
use anyhow::{Context, ensure};
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::Write,
|
||||
path::Path,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// Explicit authenticated `BigQuery` request. Never run without a billing cap.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CruxDownload {
|
||||
/// Operator's billing project.
|
||||
pub project: String,
|
||||
/// Observation month, YYYYMM.
|
||||
pub month: String,
|
||||
/// Optional two-letter audience country.
|
||||
pub country: Option<String>,
|
||||
/// Maximum bytes billed by this query.
|
||||
pub maximum_bytes_billed: u64,
|
||||
/// Maximum exported CSV bytes.
|
||||
pub maximum_output_bytes: u64,
|
||||
}
|
||||
|
||||
/// Acquires a complete `CrUX` projection using `GOOGLE_OAUTH_ACCESS_TOKEN`.
|
||||
/// The token is never persisted, logged, or sent outside bigquery.googleapis.com.
|
||||
/// A durable job ID avoids duplicate submissions on interruption. Completed
|
||||
/// pages replay from the same server-side result; output is published atomically.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns authentication, billing, schema, size, expiry, or transport failures.
|
||||
pub async fn download(cache: &Path, request: &CruxDownload) -> anyhow::Result<CachedSource> {
|
||||
let query = query(request)?;
|
||||
let key = crate::digest(&serde_json::to_vec(request)?);
|
||||
let root = cache.join("crux").join(&key);
|
||||
fs::create_dir_all(&root)?;
|
||||
let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(root.join("download.lock"))?;
|
||||
lock.try_lock()
|
||||
.context("CrUX acquisition already running")?;
|
||||
let manifest_path = root.join("source.json");
|
||||
if manifest_path.exists() {
|
||||
let manifest: SourceManifest = crate::read_json(&manifest_path)?;
|
||||
manifest.validate()?;
|
||||
let input = root.join(&manifest.sha256);
|
||||
ensure!(
|
||||
crate::file_digest(&input)? == manifest.sha256
|
||||
&& input.metadata()?.len() == manifest.bytes
|
||||
&& manifest.source == Source::Crux
|
||||
&& manifest.bytes <= request.maximum_output_bytes,
|
||||
"cached CrUX corruption"
|
||||
);
|
||||
return Ok(CachedSource {
|
||||
input,
|
||||
manifest: manifest_path,
|
||||
});
|
||||
}
|
||||
let token = std::env::var("GOOGLE_OAUTH_ACCESS_TOKEN")
|
||||
.context("set GOOGLE_OAUTH_ACCESS_TOKEN for the explicitly configured billing project")?;
|
||||
let client = client()?;
|
||||
let job_id = format!("argand_site_registry_{key}");
|
||||
ensure_job(&client, &token, request, &job_id, &query, &root).await?;
|
||||
let part = root.join("projection.part");
|
||||
download_pages(&client, &token, request, &job_id, &part).await?;
|
||||
let sha256 = crate::file_digest(&part)?;
|
||||
let input = root.join(&sha256);
|
||||
let manifest = SourceManifest {
|
||||
schema: "argand.site-source/v1".into(),
|
||||
source: Source::Crux,
|
||||
format: Format::CruxCsv,
|
||||
compression: Compression::None,
|
||||
snapshot: format!(
|
||||
"{}:{}:{job_id}",
|
||||
request.month,
|
||||
request.country.as_deref().unwrap_or("global")
|
||||
),
|
||||
scope: format!(
|
||||
"monthly:{}:{}",
|
||||
request.month,
|
||||
request.country.as_deref().unwrap_or("global")
|
||||
),
|
||||
source_url: "https://developer.chrome.com/docs/crux/bigquery/".into(),
|
||||
license: Source::Crux.license().into(),
|
||||
license_url: Source::Crux.license_url().into(),
|
||||
retrieved_at: Utc::now(),
|
||||
sha256,
|
||||
bytes: part.metadata()?.len(),
|
||||
};
|
||||
fs::rename(part, &input)?;
|
||||
File::open(&root)?.sync_all()?;
|
||||
argand_atomic::create_durable(&manifest_path, &serde_json::to_vec_pretty(&manifest)?)?;
|
||||
Ok(CachedSource {
|
||||
input,
|
||||
manifest: manifest_path,
|
||||
})
|
||||
}
|
||||
|
||||
async fn ensure_job(
|
||||
client: &reqwest::Client,
|
||||
token: &str,
|
||||
request: &CruxDownload,
|
||||
job_id: &str,
|
||||
query: &str,
|
||||
root: &Path,
|
||||
) -> anyhow::Result<()> {
|
||||
let job = json!({"jobReference":{"projectId":request.project,"jobId":job_id,"location":"US"},"configuration":{"query":{"query":query,"useLegacySql":false,"maximumBytesBilled":request.maximum_bytes_billed.to_string()}}});
|
||||
let job_path = root.join("job.json");
|
||||
if !job_path.exists() {
|
||||
argand_atomic::create_durable(&job_path, &serde_json::to_vec_pretty(&job)?)?;
|
||||
}
|
||||
let endpoint = format!(
|
||||
"https://bigquery.googleapis.com/bigquery/v2/projects/{}/jobs",
|
||||
request.project
|
||||
);
|
||||
let response = client
|
||||
.post(&endpoint)
|
||||
.bearer_auth(token)
|
||||
.json(&job)
|
||||
.send()
|
||||
.await?;
|
||||
ensure!(
|
||||
response.status().is_success() || response.status() == reqwest::StatusCode::CONFLICT,
|
||||
"BigQuery job submission HTTP {}",
|
||||
response.status()
|
||||
);
|
||||
// A conflict must be our exact query and budget, not an unrelated prior job.
|
||||
let remote = bounded_json(
|
||||
client
|
||||
.get(format!("{endpoint}/{job_id}"))
|
||||
.query(&[("location", "US")])
|
||||
.bearer_auth(token)
|
||||
.send()
|
||||
.await?,
|
||||
)
|
||||
.await?;
|
||||
ensure!(
|
||||
remote
|
||||
.pointer("/configuration/query/query")
|
||||
.and_then(Value::as_str)
|
||||
== Some(query),
|
||||
"existing BigQuery job query differs"
|
||||
);
|
||||
ensure!(
|
||||
remote
|
||||
.pointer("/configuration/query/maximumBytesBilled")
|
||||
.and_then(Value::as_str)
|
||||
== Some(&request.maximum_bytes_billed.to_string()),
|
||||
"existing BigQuery job billing limit differs"
|
||||
);
|
||||
ensure!(
|
||||
remote.pointer("/status/errorResult").is_none(),
|
||||
"BigQuery job failed; inspect job metadata using your account"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_pages(
|
||||
client: &reqwest::Client,
|
||||
token: &str,
|
||||
request: &CruxDownload,
|
||||
job_id: &str,
|
||||
part: &Path,
|
||||
) -> anyhow::Result<()> {
|
||||
let result_url = format!(
|
||||
"https://bigquery.googleapis.com/bigquery/v2/projects/{}/queries/{job_id}",
|
||||
request.project
|
||||
);
|
||||
let mut output = File::create(part)?; // atomic-writes: allow unpublished replay file, synced and renamed before manifest publication
|
||||
output.write_all(b"origin,rank,yyyymm,country_code\n")?;
|
||||
let mut written = 29_u64;
|
||||
let mut count = 0_u64;
|
||||
let mut page_token = String::new();
|
||||
let mut pending = 0_u32;
|
||||
loop {
|
||||
let mut get = client.get(&result_url).bearer_auth(token).query(&[
|
||||
("location", "US"),
|
||||
("maxResults", "10000"),
|
||||
("timeoutMs", "10000"),
|
||||
]);
|
||||
if !page_token.is_empty() {
|
||||
get = get.query(&[("pageToken", &page_token)]);
|
||||
}
|
||||
let page = bounded_json(get.send().await?).await?;
|
||||
if page["jobComplete"] != true {
|
||||
pending += 1;
|
||||
ensure!(
|
||||
pending <= 180,
|
||||
"BigQuery job still pending; rerun to resume the same job"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
continue;
|
||||
}
|
||||
ensure!(page.get("errors").is_none(), "BigQuery returned job errors");
|
||||
let bytes = page_csv(&page)?;
|
||||
written += u64::try_from(bytes.len())?;
|
||||
ensure!(
|
||||
written <= request.maximum_output_bytes,
|
||||
"CrUX output byte cap exceeded"
|
||||
);
|
||||
output.write_all(&bytes)?;
|
||||
count += u64::try_from(
|
||||
page.get("rows")
|
||||
.and_then(Value::as_array)
|
||||
.map_or(0, Vec::len),
|
||||
)?;
|
||||
let next = page
|
||||
.get("pageToken")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
if next.is_empty() {
|
||||
let total: u64 = page["totalRows"]
|
||||
.as_str()
|
||||
.context("BigQuery totalRows missing")?
|
||||
.parse()?;
|
||||
ensure!(
|
||||
count == total && count > 0,
|
||||
"incomplete or empty CrUX result"
|
||||
);
|
||||
break;
|
||||
}
|
||||
ensure!(next != page_token, "BigQuery repeated pagination token");
|
||||
page_token = next.into();
|
||||
}
|
||||
output.sync_all()?;
|
||||
drop(output);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn query(request: &CruxDownload) -> anyhow::Result<String> {
|
||||
crate::adapters::csv_sources::validate_month(&request.month)?;
|
||||
ensure!(
|
||||
!request.project.is_empty()
|
||||
&& request.project.len() <= 63
|
||||
&& request
|
||||
.project
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'),
|
||||
"invalid Google project ID"
|
||||
);
|
||||
ensure!(
|
||||
request.maximum_bytes_billed > 0 && request.maximum_output_bytes > 0,
|
||||
"explicit positive query and output byte caps required"
|
||||
);
|
||||
let (table, country) = if let Some(country) = &request.country {
|
||||
ensure!(
|
||||
country.len() == 2 && country.bytes().all(|b| b.is_ascii_alphabetic()),
|
||||
"invalid country"
|
||||
);
|
||||
(
|
||||
format!(
|
||||
"chrome-ux-report.country_{}.{}",
|
||||
country.to_ascii_lowercase(),
|
||||
request.month
|
||||
),
|
||||
country.to_ascii_uppercase(),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
format!("chrome-ux-report.all.{}", request.month),
|
||||
String::new(),
|
||||
)
|
||||
};
|
||||
Ok(format!(
|
||||
"SELECT DISTINCT origin, experimental.popularity.rank AS rank, '{}' AS yyyymm, '{}' AS country_code FROM `{table}` WHERE experimental.popularity.rank IS NOT NULL ORDER BY origin, rank",
|
||||
request.month, country
|
||||
))
|
||||
}
|
||||
|
||||
async fn bounded_json(mut response: reqwest::Response) -> anyhow::Result<Value> {
|
||||
ensure!(
|
||||
response.status().is_success(),
|
||||
"BigQuery HTTP {}",
|
||||
response.status()
|
||||
);
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
ensure!(
|
||||
bytes.len() + chunk.len() <= 24 * 1024 * 1024,
|
||||
"BigQuery page exceeds 24 MiB"
|
||||
);
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
crate::json::parse(&bytes)
|
||||
}
|
||||
|
||||
fn page_csv(page: &Value) -> anyhow::Result<Vec<u8>> {
|
||||
let schema = page
|
||||
.pointer("/schema/fields")
|
||||
.and_then(Value::as_array)
|
||||
.context("missing BigQuery schema")?;
|
||||
ensure!(
|
||||
schema
|
||||
.iter()
|
||||
.filter_map(|f| f["name"].as_str())
|
||||
.collect::<Vec<_>>()
|
||||
== ["origin", "rank", "yyyymm", "country_code"],
|
||||
"BigQuery projection schema changed"
|
||||
);
|
||||
let mut writer = csv::Writer::from_writer(Vec::new());
|
||||
if let Some(rows) = page.get("rows").and_then(Value::as_array) {
|
||||
for row in rows {
|
||||
let fields = row["f"].as_array().context("missing BigQuery row fields")?;
|
||||
ensure!(fields.len() == 4, "BigQuery field count mismatch");
|
||||
let values = fields
|
||||
.iter()
|
||||
.map(|f| f["v"].as_str().context("null or non-string BigQuery cell"))
|
||||
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||
writer.write_record(values)?;
|
||||
}
|
||||
}
|
||||
Ok(writer.into_inner()?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn documented_projection_billing_bound_and_page_schema() -> anyhow::Result<()> {
|
||||
let mut request = CruxDownload {
|
||||
project: "example-project".into(),
|
||||
month: "202608".into(),
|
||||
country: Some("GB".into()),
|
||||
maximum_bytes_billed: 1_000_000,
|
||||
maximum_output_bytes: 1_000_000,
|
||||
};
|
||||
let sql = query(&request)?;
|
||||
assert!(sql.contains("`chrome-ux-report.country_gb.202608`"));
|
||||
assert!(sql.contains("experimental.popularity.rank"));
|
||||
request.maximum_bytes_billed = 0;
|
||||
assert!(query(&request).is_err());
|
||||
let mut page = json!({"schema":{"fields":[{"name":"origin"},{"name":"rank"},{"name":"yyyymm"},{"name":"country_code"}]},"rows":[{"f":[{"v":"https://example.co.uk"},{"v":"1000"},{"v":"202608"},{"v":"GB"}]}]});
|
||||
assert_eq!(page_csv(&page)?, b"https://example.co.uk,1000,202608,GB\n");
|
||||
page["rows"][0]["f"][1]["v"] = Value::Null;
|
||||
assert!(page_csv(&page).is_err());
|
||||
page["schema"]["fields"][0]["name"] = json!("changed");
|
||||
assert!(page_csv(&page).is_err());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
430
crates/argand-site-registry/src/download.rs
Normal file
430
crates/argand-site-registry/src/download.rs
Normal file
|
|
@ -0,0 +1,430 @@
|
|||
// By Nic Weyand!
|
||||
//! Allowlisted source acquisition with validator-bound range resume.
|
||||
|
||||
use crate::model::{Compression, Format, Source, SourceManifest};
|
||||
use anyhow::{Context, ensure};
|
||||
use chrono::Utc;
|
||||
use reqwest::{Client, StatusCode, header};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// Explicit download request. The byte cap is mandatory for large objects.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Download {
|
||||
/// Provider whose license and transport allowlist apply.
|
||||
pub source: Source,
|
||||
/// Supported source format.
|
||||
pub format: Format,
|
||||
/// Outer compression.
|
||||
#[serde(default)]
|
||||
pub compression: Compression,
|
||||
/// Official distribution URL.
|
||||
pub url: String,
|
||||
/// Source-native revision or dump date.
|
||||
pub snapshot: String,
|
||||
/// Source replacement scope.
|
||||
pub scope: String,
|
||||
/// Maximum downloaded object bytes.
|
||||
pub maximum_bytes: u64,
|
||||
}
|
||||
|
||||
/// Result points to immutable cached bytes and their manifest.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct CachedSource {
|
||||
/// Local input path.
|
||||
pub input: PathBuf,
|
||||
/// Local manifest path.
|
||||
pub manifest: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
struct Resume {
|
||||
url: String,
|
||||
etag: String,
|
||||
total: u64,
|
||||
}
|
||||
|
||||
/// Validates source URLs against reviewed data endpoints, including redirects.
|
||||
/// No arbitrary URL or website assertion is ever fetched through this path.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects unknown endpoints/parameters, credentials, fragments and custom ports.
|
||||
#[allow(clippy::case_sensitive_file_extension_comparisons)] // HTTPS endpoint paths are case-sensitive.
|
||||
pub fn validate_source_url(source: Source, input: &str) -> anyhow::Result<()> {
|
||||
let url = url::Url::parse(input)?;
|
||||
ensure!(
|
||||
url.scheme() == "https"
|
||||
&& url.port().is_none()
|
||||
&& url.username().is_empty()
|
||||
&& url.password().is_none()
|
||||
&& url.fragment().is_none(),
|
||||
"source requires HTTPS without credentials/fragment/custom port"
|
||||
);
|
||||
let host = url.host_str().context("missing source host")?;
|
||||
let path = url.path();
|
||||
if source == Source::Wikidata && host == "www.wikidata.org" && path == "/w/api.php" {
|
||||
return validate_entity_query(&url);
|
||||
}
|
||||
ensure!(url.query().is_none(), "unexpected source query parameters");
|
||||
let allowed = match source {
|
||||
Source::Wikidata => {
|
||||
(host == "dumps.wikimedia.org"
|
||||
&& path.starts_with("/wikidatawiki/entities/")
|
||||
&& (path.ends_with(".json.gz") || path.ends_with(".json.bz2")))
|
||||
|| (host == "www.wikidata.org"
|
||||
&& path.starts_with("/wiki/Special:EntityData/")
|
||||
&& path.ends_with(".json"))
|
||||
}
|
||||
Source::Majestic => host == "downloads.majestic.com" && path == "/majestic_million.csv",
|
||||
Source::Crux => host == "developer.chrome.com" && path == "/docs/crux/bigquery/",
|
||||
Source::Curlie => {
|
||||
(host == "curlie.org" && path == "/directory-dl")
|
||||
|| (host == "share.innkube.fim.uni-passau.de"
|
||||
&& path == "/curlie-rdf/curlie-rdf-all.tar.gz")
|
||||
}
|
||||
Source::Psl => host == "publicsuffix.org" && path == "/list/public_suffix_list.dat",
|
||||
};
|
||||
ensure!(allowed, "unreviewed source endpoint: {host}{path}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn client() -> anyhow::Result<Client> {
|
||||
Ok(Client::builder()
|
||||
.user_agent("Argand-Site-Registry/0.1 (+https://argand.org)")
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.no_proxy()
|
||||
.connect_timeout(Duration::from_secs(15))
|
||||
.read_timeout(Duration::from_secs(60))
|
||||
.build()?)
|
||||
}
|
||||
|
||||
/// Downloads and seals source bytes. Re-running the same request reuses them.
|
||||
/// Change `snapshot` to request a refresh. Incomplete downloads resume only with
|
||||
/// a strong `ETag`, matching Content-Range, and the same final URL.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns URL policy, transport, size, locking, integrity, or filesystem errors.
|
||||
pub async fn download(cache: &Path, request: &Download) -> anyhow::Result<CachedSource> {
|
||||
ensure!(
|
||||
request.source != Source::Crux,
|
||||
"CrUX acquisition uses crux-download and authenticated BigQuery pagination"
|
||||
);
|
||||
validate_source_url(request.source, &request.url)?;
|
||||
ensure!(request.maximum_bytes > 0, "maximum bytes must be positive");
|
||||
let key = crate::digest(&serde_json::to_vec(request)?);
|
||||
let dir = cache.join(request.source.key()).join(key);
|
||||
fs::create_dir_all(&dir)?;
|
||||
let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(dir.join("download.lock"))?;
|
||||
lock.try_lock().context("source download already running")?;
|
||||
let complete = dir.join("source.json");
|
||||
if complete.exists() {
|
||||
let manifest: SourceManifest = crate::read_json(&complete)?;
|
||||
manifest.validate()?;
|
||||
ensure!(
|
||||
manifest.source == request.source
|
||||
&& manifest.format == request.format
|
||||
&& manifest.source_url == request.url
|
||||
&& manifest.snapshot == request.snapshot
|
||||
&& manifest.scope == request.scope
|
||||
&& manifest.bytes <= request.maximum_bytes,
|
||||
"cached source declaration differs from request"
|
||||
);
|
||||
let input = dir.join(&manifest.sha256);
|
||||
ensure!(
|
||||
crate::file_digest(&input)? == manifest.sha256
|
||||
&& input.metadata()?.len() == manifest.bytes,
|
||||
"cached object corrupted"
|
||||
);
|
||||
return Ok(CachedSource {
|
||||
input,
|
||||
manifest: complete,
|
||||
});
|
||||
}
|
||||
if request.source == Source::Psl {
|
||||
reserve_psl_refresh(cache)?;
|
||||
}
|
||||
let part = dir.join("download.part");
|
||||
let state = dir.join("resume.json");
|
||||
let client = client()?;
|
||||
let mut last_error = None;
|
||||
let attempts = if request.source == Source::Psl { 1 } else { 3 };
|
||||
for attempt in 0..attempts {
|
||||
match transfer(&client, request, &part, &state).await {
|
||||
Ok(()) => {
|
||||
last_error = None;
|
||||
break;
|
||||
}
|
||||
Err(error) => {
|
||||
last_error = Some(error);
|
||||
if attempt + 1 < attempts {
|
||||
tokio::time::sleep(Duration::from_secs(1 << attempt)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(error) = last_error {
|
||||
return Err(error);
|
||||
}
|
||||
let sha256 = crate::file_digest(&part)?;
|
||||
let bytes = part.metadata()?.len();
|
||||
let manifest = SourceManifest {
|
||||
schema: "argand.site-source/v1".into(),
|
||||
source: request.source,
|
||||
format: request.format,
|
||||
compression: request.compression,
|
||||
snapshot: request.snapshot.clone(),
|
||||
scope: request.scope.clone(),
|
||||
source_url: request.url.clone(),
|
||||
license: request.source.license().into(),
|
||||
license_url: request.source.license_url().into(),
|
||||
retrieved_at: Utc::now(),
|
||||
sha256: sha256.clone(),
|
||||
bytes,
|
||||
};
|
||||
manifest.validate()?;
|
||||
let input = dir.join(sha256);
|
||||
fs::rename(&part, &input)?;
|
||||
File::open(&dir)?.sync_all()?;
|
||||
argand_atomic::create_durable(&complete, &serde_json::to_vec_pretty(&manifest)?)?;
|
||||
Ok(CachedSource {
|
||||
input,
|
||||
manifest: complete,
|
||||
})
|
||||
}
|
||||
|
||||
async fn transfer(
|
||||
client: &Client,
|
||||
request: &Download,
|
||||
part: &Path,
|
||||
state: &Path,
|
||||
) -> anyhow::Result<()> {
|
||||
let resume: Option<Resume> = if state.exists() && part.exists() {
|
||||
Some(crate::read_json(state)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let offset = part.metadata().map_or(0, |m| m.len());
|
||||
let resume = resume.filter(|r| {
|
||||
!r.etag.is_empty() && !r.etag.starts_with("W/") && offset > 0 && offset < r.total
|
||||
});
|
||||
let mut url = request.url.clone();
|
||||
let mut response = None;
|
||||
for _ in 0..5 {
|
||||
validate_source_url(request.source, &url)?;
|
||||
let mut get = client.get(&url).header(header::ACCEPT_ENCODING, "identity");
|
||||
if let Some(r) = resume.as_ref().filter(|r| r.url == url) {
|
||||
get = get
|
||||
.header(header::RANGE, format!("bytes={offset}-"))
|
||||
.header(header::IF_RANGE, &r.etag);
|
||||
}
|
||||
let reply = get.send().await?;
|
||||
if reply.status().is_redirection() {
|
||||
let location = reply
|
||||
.headers()
|
||||
.get(header::LOCATION)
|
||||
.context("redirect missing Location")?
|
||||
.to_str()?;
|
||||
url = url::Url::parse(&url)?.join(location)?.to_string();
|
||||
} else {
|
||||
response = Some(reply);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut response = response.context("source redirect limit exceeded")?;
|
||||
let (start, total, etag) = response_extent(&response, resume.as_ref(), offset, &url)?;
|
||||
ensure!(
|
||||
total <= request.maximum_bytes,
|
||||
"source exceeds configured byte budget"
|
||||
);
|
||||
let mut output = OpenOptions::new() // atomic-writes: allow resumable unpublished partial file; seal by rename and receipt
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(start == 0)
|
||||
.append(start > 0)
|
||||
.open(part)?;
|
||||
argand_atomic::replace_durable(state, &serde_json::to_vec(&Resume { url, etag, total })?)?;
|
||||
let mut written = start;
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
written += u64::try_from(chunk.len())?;
|
||||
ensure!(
|
||||
(total == 0 || written <= total) && written <= request.maximum_bytes,
|
||||
"source exceeded declared byte length"
|
||||
);
|
||||
output.write_all(&chunk)?;
|
||||
}
|
||||
output.sync_all()?;
|
||||
ensure!(
|
||||
written > 0 && (total == 0 || written == total),
|
||||
"source transfer truncated or empty"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn response_extent(
|
||||
response: &reqwest::Response,
|
||||
resume: Option<&Resume>,
|
||||
offset: u64,
|
||||
url: &str,
|
||||
) -> anyhow::Result<(u64, u64, String)> {
|
||||
let status = response.status();
|
||||
ensure!(
|
||||
status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT,
|
||||
"source returned HTTP {status}"
|
||||
);
|
||||
ensure!(
|
||||
response
|
||||
.headers()
|
||||
.get(header::CONTENT_ENCODING)
|
||||
.is_none_or(|v| v == "identity"),
|
||||
"unexpected HTTP content encoding"
|
||||
);
|
||||
let etag = response
|
||||
.headers()
|
||||
.get(header::ETAG)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
let (start, total) = if status == StatusCode::PARTIAL_CONTENT {
|
||||
let range = response
|
||||
.headers()
|
||||
.get(header::CONTENT_RANGE)
|
||||
.context("missing Content-Range")?
|
||||
.to_str()?;
|
||||
let (start, end, total) = parse_range(range)?;
|
||||
let old = resume.context("unsolicited partial response")?;
|
||||
ensure!(
|
||||
start == offset
|
||||
&& end + 1 == total
|
||||
&& total == old.total
|
||||
&& old.url == url
|
||||
&& etag == old.etag,
|
||||
"range validator or offsets changed"
|
||||
);
|
||||
(start, total)
|
||||
} else {
|
||||
(0, response.content_length().unwrap_or(0))
|
||||
};
|
||||
Ok((start, total, etag))
|
||||
}
|
||||
|
||||
fn validate_entity_query(url: &url::Url) -> anyhow::Result<()> {
|
||||
let pairs: Vec<_> = url.query_pairs().collect();
|
||||
let mut params = std::collections::BTreeMap::new();
|
||||
for (key, value) in pairs {
|
||||
ensure!(
|
||||
params.insert(key, value).is_none(),
|
||||
"duplicate source query parameter"
|
||||
);
|
||||
}
|
||||
ensure!(
|
||||
params.len() == 4
|
||||
&& params.get("action").is_some_and(|s| s == "wbgetentities")
|
||||
&& params.get("format").is_some_and(|s| s == "json")
|
||||
&& params.get("maxlag").is_some_and(|s| s == "5"),
|
||||
"only the reviewed wbgetentities JSON query is supported"
|
||||
);
|
||||
let ids = params.get("ids").context("missing Wikidata entity IDs")?;
|
||||
ensure!(
|
||||
ids.len() <= 1024
|
||||
&& ids.split('|').count() <= 50
|
||||
&& ids.split('|').all(|id| {
|
||||
id.len() > 1
|
||||
&& id.starts_with('Q')
|
||||
&& id.as_bytes()[1] != b'0'
|
||||
&& id[1..].bytes().all(|b| b.is_ascii_digit())
|
||||
}),
|
||||
"invalid entity selection"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn parse_range(value: &str) -> anyhow::Result<(u64, u64, u64)> {
|
||||
let (bounds, total) = value
|
||||
.strip_prefix("bytes ")
|
||||
.context("invalid range unit")?
|
||||
.split_once('/')
|
||||
.context("invalid range total")?;
|
||||
let (start, end) = bounds.split_once('-').context("invalid range bounds")?;
|
||||
let result = (start.parse()?, end.parse()?, total.parse()?);
|
||||
ensure!(
|
||||
result.0 <= result.1 && result.1 < result.2,
|
||||
"invalid range interval"
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn reserve_psl_refresh(cache: &Path) -> anyhow::Result<()> {
|
||||
let daily_lock = OpenOptions::new() // atomic-writes: allow serialize the shared refresh timestamp
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(cache.join("psl/refresh.lock"))?;
|
||||
daily_lock
|
||||
.try_lock()
|
||||
.context("PSL refresh already running")?;
|
||||
let stamp = cache.join("psl/last-attempt.json");
|
||||
if stamp.exists() {
|
||||
let last: chrono::DateTime<Utc> = crate::read_json(&stamp)?;
|
||||
ensure!(
|
||||
Utc::now() - last >= chrono::Duration::days(1),
|
||||
"PSL network refresh limited to once per day; reuse the existing pinned snapshot"
|
||||
);
|
||||
}
|
||||
argand_atomic::replace_durable(&stamp, &serde_json::to_vec(&Utc::now())?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn validator_bound_ranges_and_source_allowlist() -> anyhow::Result<()> {
|
||||
let url = "https://downloads.majestic.com/majestic_million.csv";
|
||||
validate_source_url(Source::Majestic, url)?;
|
||||
for bad in [
|
||||
"http://downloads.majestic.com/majestic_million.csv",
|
||||
"https://downloads.majestic.com.evil.org/majestic_million.csv",
|
||||
"https://downloads.majestic.com/other.csv",
|
||||
"https://user@downloads.majestic.com/majestic_million.csv",
|
||||
] {
|
||||
assert!(validate_source_url(Source::Majestic, bad).is_err());
|
||||
}
|
||||
let response = reqwest::Response::from(
|
||||
http::Response::builder()
|
||||
.status(206)
|
||||
.header("content-range", "bytes 5-9/10")
|
||||
.header("etag", "\"version1\"")
|
||||
.body("12345")?,
|
||||
);
|
||||
let state = Resume {
|
||||
url: url.into(),
|
||||
etag: "\"version1\"".into(),
|
||||
total: 10,
|
||||
};
|
||||
assert_eq!(response_extent(&response, Some(&state), 5, url)?.0, 5);
|
||||
assert!(response_extent(&response, Some(&state), 4, url).is_err());
|
||||
assert!(response_extent(&response, None, 5, url).is_err());
|
||||
let changed = Resume {
|
||||
etag: "\"version2\"".into(),
|
||||
..state
|
||||
};
|
||||
assert!(response_extent(&response, Some(&changed), 5, url).is_err());
|
||||
for bad in ["items 1-2/3", "bytes 4-2/3", "bytes 1-3/3", "bytes 1-2/*"] {
|
||||
assert!(parse_range(bad).is_err());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
125
crates/argand-site-registry/src/evidence.rs
Normal file
125
crates/argand-site-registry/src/evidence.rs
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
// By Nic Weyand!
|
||||
//! Bounded, source-bearing entity and property metadata for registry consumers.
|
||||
|
||||
use anyhow::Context;
|
||||
use rusqlite::Connection;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// Stable entity identity with language-tagged labels, aliases and metadata.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct Entity {
|
||||
/// Stable source-derived Argand identity.
|
||||
pub id: String,
|
||||
/// Deterministic label selection; see the versioned build rule.
|
||||
pub canonical_name: String,
|
||||
/// Retained name/alias facts and their complete source declarations.
|
||||
pub names: Vec<Value>,
|
||||
/// All active name facts, before the output cap of 256.
|
||||
pub total_names: u64,
|
||||
/// Country/headquarters/language metadata, distinct from property scope.
|
||||
pub metadata: Vec<Value>,
|
||||
/// All active metadata facts, before the output cap of 256.
|
||||
pub total_metadata: u64,
|
||||
}
|
||||
|
||||
pub(crate) fn fact(db: &Connection, id: &str) -> anyhow::Result<Value> {
|
||||
let (manifest,native,selector,confidence,value,predicate):(String,String,String,u16,String,String)=db.query_row("SELECT s.manifest,r.native_id,f.selector,f.confidence,f.value,f.predicate FROM facts f JOIN sources s ON s.id=f.source_id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE f.id=?1",[id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?)))?;
|
||||
Ok(
|
||||
json!({"fact_id":id,"source":serde_json::from_str::<Value>(&manifest)?,"source_identifier":native,"selector":selector,"confidence":confidence,"predicate":predicate,"value":serde_json::from_str::<Value>(&value)?}),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn entity(db: &Connection, id: &str, name: &str) -> anyhow::Result<Entity> {
|
||||
let (total_names, names) = fields(db, id, true)?;
|
||||
let (total_metadata, metadata) = fields(db, id, false)?;
|
||||
Ok(Entity {
|
||||
id: id.into(),
|
||||
canonical_name: name.into(),
|
||||
names,
|
||||
total_names,
|
||||
metadata,
|
||||
total_metadata,
|
||||
})
|
||||
}
|
||||
|
||||
fn fields(db: &Connection, entity: &str, names: bool) -> anyhow::Result<(u64, Vec<Value>)> {
|
||||
let predicate = if names {
|
||||
"f.predicate='name'"
|
||||
} else {
|
||||
"f.predicate IN('P17','P159','P407','P1001','P297','P218','P219','P220')"
|
||||
};
|
||||
let from = format!(
|
||||
"FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.subject=?1 AND {predicate}"
|
||||
);
|
||||
let total = db.query_row(&format!("SELECT count(*) {from}"), [entity], |r| {
|
||||
crate::store::unsigned(r, 0)
|
||||
})?;
|
||||
let mut statement = db.prepare(&format!("SELECT f.id {from} ORDER BY f.id LIMIT 256"))?;
|
||||
let mut values = Vec::new();
|
||||
for id in statement.query_map([entity], |r| r.get::<_, String>(0))? {
|
||||
values.push(fact(db, &id?)?);
|
||||
}
|
||||
Ok((total, values))
|
||||
}
|
||||
|
||||
/// A scoped property assertion. Unknown locale/country stays null; entity-country
|
||||
/// or audience metadata is never substituted for website jurisdiction.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct PropertyScope {
|
||||
/// Unspecified for source assertions, or the explicitly reviewed role.
|
||||
pub role: String,
|
||||
/// Reviewed locale, if any.
|
||||
pub locale: Option<String>,
|
||||
/// Reviewed country, if any.
|
||||
pub country: Option<String>,
|
||||
/// Original language qualifiers as Wikidata entity IDs.
|
||||
pub language_entities: Vec<String>,
|
||||
/// Original jurisdiction qualifiers as Wikidata entity IDs.
|
||||
pub jurisdiction_entities: Vec<String>,
|
||||
/// Source fact IDs, or complete operator-decision provenance.
|
||||
pub provenance: Value,
|
||||
}
|
||||
|
||||
pub(crate) fn scopes(
|
||||
evidence: &Value,
|
||||
provenance: &[Value],
|
||||
review: Option<&Value>,
|
||||
) -> anyhow::Result<Vec<PropertyScope>> {
|
||||
let mut scopes = vec![PropertyScope {
|
||||
role: "unspecified".into(),
|
||||
locale: None,
|
||||
country: None,
|
||||
language_entities: qualifier_ids(evidence, "P407"),
|
||||
jurisdiction_entities: qualifier_ids(evidence, "P1001"),
|
||||
provenance: json!({"fact_ids":provenance.iter().map(|p|&p["fact_id"]).collect::<Vec<_>>()}),
|
||||
}];
|
||||
if let Some(review) = review {
|
||||
let text = |field| review[field].as_str().context("invalid review scope");
|
||||
let optional = |field| -> anyhow::Result<Option<String>> {
|
||||
Ok(Some(text(field)?)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_owned))
|
||||
};
|
||||
scopes.push(PropertyScope {
|
||||
role: text("role")?.into(),
|
||||
locale: optional("locale")?,
|
||||
country: optional("country")?,
|
||||
language_entities: Vec::new(),
|
||||
jurisdiction_entities: Vec::new(),
|
||||
provenance: review.clone(),
|
||||
});
|
||||
}
|
||||
Ok(scopes)
|
||||
}
|
||||
|
||||
fn qualifier_ids(evidence: &Value, property: &str) -> Vec<String> {
|
||||
evidence
|
||||
.pointer(&format!("/assertion/statement/qualifiers/{property}"))
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|v| v.pointer("/datavalue/value/id").and_then(Value::as_str))
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
220
crates/argand-site-registry/src/identity.rs
Normal file
220
crates/argand-site-registry/src/identity.rs
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
// By Nic Weyand!
|
||||
//! Explicit identity equivalences, bound to all names and website assertions.
|
||||
|
||||
use crate::{query::Registry, review::Review};
|
||||
use anyhow::ensure;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rusqlite::{Connection, params};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
/// Proposed identity relationship. Original entity IDs and assertions survive.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct Equivalence {
|
||||
/// Exact pair and evidence fingerprint to review.
|
||||
pub fingerprint: String,
|
||||
/// Deterministically ordered source entity IDs.
|
||||
pub entities: [String; 2],
|
||||
/// Digests of every current name and website assertion on each entity.
|
||||
pub signatures: [String; 2],
|
||||
/// Bounded entity metadata with its original source provenance.
|
||||
pub evidence: [crate::evidence::Entity; 2],
|
||||
}
|
||||
|
||||
/// Previews an explicit equivalence without changing the registry.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects equal or unknown entities and malformed evidence.
|
||||
pub fn propose(registry: &Registry, left: &str, right: &str) -> anyhow::Result<Equivalence> {
|
||||
ensure!(
|
||||
left != right,
|
||||
"identity equivalence needs distinct entity IDs"
|
||||
);
|
||||
let mut entities = [left.to_owned(), right.to_owned()];
|
||||
entities.sort();
|
||||
let signatures = [
|
||||
signature(®istry.db, &entities[0])?,
|
||||
signature(®istry.db, &entities[1])?,
|
||||
];
|
||||
let mut names = Vec::new();
|
||||
for entity in &entities {
|
||||
let name: String = registry.db.query_row(
|
||||
"SELECT canonical_name FROM entities WHERE id=?1",
|
||||
[entity],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
names.push(crate::evidence::entity(®istry.db, entity, &name)?);
|
||||
}
|
||||
Ok(Equivalence {
|
||||
fingerprint: crate::digest(&serde_json::to_vec(&(
|
||||
&entities,
|
||||
&signatures,
|
||||
crate::store::RULE_VERSION,
|
||||
))?),
|
||||
entities,
|
||||
signatures,
|
||||
evidence: [names.remove(0), names.remove(0)],
|
||||
})
|
||||
}
|
||||
|
||||
/// Appends a reviewed equivalence or revocation to the same signed decision log.
|
||||
/// Both complete source identities must exist in the writer store. Rebuild to use.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects stale fingerprints, scoped identity decisions or missing evidence.
|
||||
pub fn record(
|
||||
db: &Connection,
|
||||
registry: &Registry,
|
||||
pair: &Equivalence,
|
||||
review: &Review,
|
||||
) -> anyhow::Result<u64> {
|
||||
crate::review::validate(review)?;
|
||||
let expected = propose(registry, &pair.entities[0], &pair.entities[1])?;
|
||||
ensure!(
|
||||
review.fingerprint == expected.fingerprint && pair.fingerprint == expected.fingerprint,
|
||||
"identity review fingerprint differs from current evidence"
|
||||
);
|
||||
ensure!(
|
||||
review.role == "unspecified" && review.locale.is_empty() && review.country.is_empty(),
|
||||
"identity reviews cannot assert a destination role"
|
||||
);
|
||||
for entity in &expected.entities {
|
||||
let mut statement = registry.db.prepare("SELECT DISTINCT f.source_id FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.subject=?1")?;
|
||||
for id in statement.query_map([entity], |r| r.get::<_, String>(0))? {
|
||||
crate::store::source(db, &id?)?;
|
||||
}
|
||||
}
|
||||
let transaction = db.unchecked_transaction()?;
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO equivalences VALUES(?1,?2,?3,?4,?5)",
|
||||
params![
|
||||
expected.fingerprint,
|
||||
expected.entities[0],
|
||||
expected.entities[1],
|
||||
expected.signatures[0],
|
||||
expected.signatures[1]
|
||||
],
|
||||
)?;
|
||||
let sequence = crate::review::append(db, review)?;
|
||||
transaction.commit()?;
|
||||
Ok(sequence)
|
||||
}
|
||||
|
||||
fn signature(db: &Connection, entity: &str) -> anyhow::Result<String> {
|
||||
let name: String = db.query_row(
|
||||
"SELECT names_fingerprint FROM entities WHERE id=?1",
|
||||
[entity],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(name.as_bytes());
|
||||
let mut statement =
|
||||
db.prepare("SELECT fingerprint FROM edges WHERE entity=?1 ORDER BY fingerprint")?;
|
||||
for edge in statement.query_map([entity], |r| r.get::<_, String>(0))? {
|
||||
hash.update(edge?.as_bytes());
|
||||
}
|
||||
Ok(format!("{:x}", hash.finalize()))
|
||||
}
|
||||
|
||||
pub(crate) fn expand(
|
||||
registry: &Registry,
|
||||
initial: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> anyhow::Result<Option<(BTreeSet<String>, Vec<Value>)>> {
|
||||
let mut entities = BTreeSet::from([initial.to_owned()]);
|
||||
let mut pending = vec![initial.to_owned()];
|
||||
let mut signatures = BTreeMap::new();
|
||||
let mut evidence = BTreeMap::new();
|
||||
while let Some(entity) = pending.pop() {
|
||||
let mut statement=registry.db.prepare("SELECT e.fingerprint,e.left_entity,e.right_entity,e.left_signature,e.right_signature,r.sequence,r.reviewer,r.reason,r.evidence,r.reviewed_at,r.expires_at FROM equivalences e JOIN reviews r ON r.fingerprint=e.fingerprint WHERE (e.left_entity=?1 OR e.right_entity=?1) AND r.sequence=(SELECT max(sequence) FROM reviews WHERE fingerprint=e.fingerprint) AND r.decision='approve' ORDER BY e.fingerprint")?;
|
||||
let mut rows = statement.query([&entity])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let started: String = row.get(9)?;
|
||||
let expires: String = row.get(10)?;
|
||||
if now < DateTime::parse_from_rfc3339(&started)?
|
||||
|| now >= DateTime::parse_from_rfc3339(&expires)?
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let pair = [row.get::<_, String>(1)?, row.get::<_, String>(2)?];
|
||||
let mut valid = true;
|
||||
for (i, id) in pair.iter().enumerate() {
|
||||
if !signatures.contains_key(id) {
|
||||
let current = match signature(®istry.db, id) {
|
||||
Ok(signature) => Some(signature),
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.downcast_ref::<rusqlite::Error>(),
|
||||
Some(rusqlite::Error::QueryReturnedNoRows)
|
||||
) =>
|
||||
{
|
||||
None
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
signatures.insert(id.clone(), current);
|
||||
}
|
||||
if signatures.get(id).and_then(Option::as_ref)
|
||||
!= Some(&row.get::<_, String>(i + 3)?)
|
||||
{
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
continue;
|
||||
}
|
||||
let fingerprint: String = row.get(0)?;
|
||||
evidence.insert(fingerprint.clone(),json!({"fingerprint":fingerprint,"entities":pair,"source":"argand_operator_review","source_identifier":row.get::<_,i64>(5)?,"reviewer":row.get::<_,String>(6)?,"reason":row.get::<_,String>(7)?,"evidence":row.get::<_,String>(8)?,"retrieved_at":started,"expires_at":expires,"license":"CC0-1.0","confidence":9000}));
|
||||
for id in pair {
|
||||
if entities.insert(id.clone()) {
|
||||
pending.push(id);
|
||||
}
|
||||
}
|
||||
if entities.len() > 64 || evidence.len() > 256 {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some((entities, evidence.into_values().collect())))
|
||||
}
|
||||
|
||||
pub(crate) fn candidates(
|
||||
registry: &Registry,
|
||||
query: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> anyhow::Result<Vec<crate::query::Candidate>> {
|
||||
let key = crate::normalize::name_key(query)?;
|
||||
let mut statement = registry
|
||||
.db
|
||||
.prepare("SELECT DISTINCT entity FROM names WHERE key=?1 ORDER BY entity LIMIT 65")?;
|
||||
let matched = statement
|
||||
.query_map([key], |r| r.get::<_, String>(0))?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
if matched.is_empty() || matched.len() > 64 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let Some((entities, evidence)) = expand(registry, &matched[0], now)? else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
if matched.iter().any(|id| !entities.contains(id)) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
for entity in entities {
|
||||
let mut statement = registry.db.prepare(
|
||||
"SELECT fingerprint FROM edges WHERE entity=?1 ORDER BY fingerprint LIMIT 101",
|
||||
)?;
|
||||
for fingerprint in statement.query_map([entity], |r| r.get::<_, String>(0))? {
|
||||
let mut candidate = registry.candidate(&fingerprint?)?;
|
||||
candidate.identity_provenance.clone_from(&evidence);
|
||||
output.push(candidate);
|
||||
if output.len() > 100 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
76
crates/argand-site-registry/src/json.rs
Normal file
76
crates/argand-site-registry/src/json.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
// By Nic Weyand!
|
||||
//! Reject duplicate object keys instead of silently losing conflicting values.
|
||||
|
||||
use serde::de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor};
|
||||
use serde_json::{Map, Number, Value};
|
||||
use std::fmt;
|
||||
|
||||
struct Unique(Value);
|
||||
|
||||
impl<'de> Deserialize<'de> for Unique {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
deserializer.deserialize_any(UniqueVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
struct UniqueVisitor;
|
||||
impl<'de> Visitor<'de> for UniqueVisitor {
|
||||
type Value = Unique;
|
||||
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("JSON with unique object keys")
|
||||
}
|
||||
fn visit_bool<E: de::Error>(self, v: bool) -> Result<Unique, E> {
|
||||
Ok(Unique(Value::Bool(v)))
|
||||
}
|
||||
fn visit_i64<E: de::Error>(self, v: i64) -> Result<Unique, E> {
|
||||
Ok(Unique(Value::Number(v.into())))
|
||||
}
|
||||
fn visit_u64<E: de::Error>(self, v: u64) -> Result<Unique, E> {
|
||||
Ok(Unique(Value::Number(v.into())))
|
||||
}
|
||||
fn visit_f64<E: de::Error>(self, v: f64) -> Result<Unique, E> {
|
||||
Number::from_f64(v)
|
||||
.map(|n| Unique(Value::Number(n)))
|
||||
.ok_or_else(|| E::custom("nonfinite JSON number"))
|
||||
}
|
||||
fn visit_str<E: de::Error>(self, v: &str) -> Result<Unique, E> {
|
||||
Ok(Unique(Value::String(v.into())))
|
||||
}
|
||||
fn visit_unit<E: de::Error>(self) -> Result<Unique, E> {
|
||||
Ok(Unique(Value::Null))
|
||||
}
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Unique, A::Error> {
|
||||
let mut values = Vec::new();
|
||||
while let Some(Unique(v)) = seq.next_element()? {
|
||||
values.push(v);
|
||||
}
|
||||
Ok(Unique(Value::Array(values)))
|
||||
}
|
||||
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Unique, A::Error> {
|
||||
let mut values = Map::new();
|
||||
while let Some((key, Unique(value))) = map.next_entry::<String, Unique>()? {
|
||||
if values.insert(key, value).is_some() {
|
||||
return Err(de::Error::custom("duplicate JSON object key"));
|
||||
}
|
||||
}
|
||||
Ok(Unique(Value::Object(values)))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse(bytes: &[u8]) -> anyhow::Result<Value> {
|
||||
Ok(serde_json::from_slice::<Unique>(bytes)?.0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn duplicate_nested_keys_are_rejected() -> anyhow::Result<()> {
|
||||
assert!(super::parse(br#"{"a":{"url":"good","url":"bad"}}"#).is_err());
|
||||
let bytes = br#"{"a":[true,null,42,-8,0.2,"text"]}"#;
|
||||
assert_eq!(
|
||||
super::parse(bytes)?,
|
||||
serde_json::from_slice::<serde_json::Value>(bytes)?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
58
crates/argand-site-registry/src/lib.rs
Normal file
58
crates/argand-site-registry/src/lib.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
// By Nic Weyand!
|
||||
//! Source-separated website assertions and reviewed, immutable registry releases.
|
||||
|
||||
pub mod adapters;
|
||||
pub mod build;
|
||||
pub mod crux;
|
||||
pub mod download;
|
||||
pub mod evidence;
|
||||
pub mod identity;
|
||||
mod json;
|
||||
pub mod model;
|
||||
pub mod normalize;
|
||||
pub mod observation;
|
||||
pub mod query;
|
||||
pub mod release;
|
||||
pub mod review;
|
||||
pub mod store;
|
||||
pub mod update;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{fs::File, io::Read, path::Path};
|
||||
|
||||
/// Hashes bytes with SHA-256, using lowercase hexadecimal.
|
||||
#[must_use]
|
||||
pub fn digest(bytes: &[u8]) -> String {
|
||||
format!("{:x}", Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
/// Hashes a file with bounded memory.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns filesystem errors.
|
||||
pub fn file_digest(path: &Path) -> anyhow::Result<String> {
|
||||
let mut file = File::open(path)?;
|
||||
let mut hash = Sha256::new();
|
||||
let mut buffer = [0; 8192];
|
||||
loop {
|
||||
let count = file.read(&mut buffer)?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
hash.update(&buffer[..count]);
|
||||
}
|
||||
Ok(format!("{:x}", hash.finalize()))
|
||||
}
|
||||
|
||||
/// Reads bounded JSON metadata, never an unbounded dataset.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns malformed, oversized, or unreadable input errors.
|
||||
pub fn read_json<T: serde::de::DeserializeOwned>(path: &Path) -> anyhow::Result<T> {
|
||||
let mut bytes = Vec::new();
|
||||
File::open(path)?
|
||||
.take(1024 * 1024 + 1)
|
||||
.read_to_end(&mut bytes)?;
|
||||
anyhow::ensure!(bytes.len() <= 1024 * 1024, "metadata exceeds 1 MiB");
|
||||
Ok(serde_json::from_value(json::parse(&bytes)?)?)
|
||||
}
|
||||
8
crates/argand-site-registry/src/main.rs
Normal file
8
crates/argand-site-registry/src/main.rs
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
// By Nic Weyand!
|
||||
//! Native Site Registry CLI.
|
||||
mod cli;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
cli::run().await
|
||||
}
|
||||
214
crates/argand-site-registry/src/model.rs
Normal file
214
crates/argand-site-registry/src/model.rs
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
// By Nic Weyand!
|
||||
//! Source identities and source-native assertion envelopes.
|
||||
|
||||
use anyhow::ensure;
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
/// Only sources with reviewed commercial reuse terms are implemented.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, clap::ValueEnum, Eq, PartialEq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Source {
|
||||
/// Wikidata structured data.
|
||||
Wikidata,
|
||||
/// Majestic Million domain ranks.
|
||||
Majestic,
|
||||
/// Chrome User Experience Report origin ranks.
|
||||
Crux,
|
||||
/// Curlie directory entries and categories.
|
||||
Curlie,
|
||||
/// Public Suffix List, including PRIVATE rules.
|
||||
Psl,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
/// Stable source namespace.
|
||||
#[must_use]
|
||||
pub const fn key(self) -> &'static str {
|
||||
match self {
|
||||
Self::Wikidata => "wikidata",
|
||||
Self::Majestic => "majestic",
|
||||
Self::Crux => "crux",
|
||||
Self::Curlie => "curlie",
|
||||
Self::Psl => "psl",
|
||||
}
|
||||
}
|
||||
/// Exact SPDX data license.
|
||||
#[must_use]
|
||||
pub const fn license(self) -> &'static str {
|
||||
match self {
|
||||
Self::Wikidata => "CC0-1.0",
|
||||
Self::Majestic | Self::Curlie => "CC-BY-3.0",
|
||||
Self::Crux => "CC-BY-4.0",
|
||||
Self::Psl => "MPL-2.0",
|
||||
}
|
||||
}
|
||||
/// Authoritative license evidence page.
|
||||
#[must_use]
|
||||
pub const fn license_url(self) -> &'static str {
|
||||
match self {
|
||||
Self::Wikidata => "https://www.wikidata.org/wiki/Wikidata:Licensing",
|
||||
Self::Majestic => "https://majestic.com/reports/majestic-million",
|
||||
Self::Crux => "https://developer.chrome.com/docs/crux/methodology",
|
||||
Self::Curlie => "https://curlie.org/docs/en/license.html",
|
||||
Self::Psl => "https://publicsuffix.org/list/public_suffix_list.dat",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Source format is explicit; unsupported revisions fail closed.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, clap::ValueEnum, Eq, PartialEq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Format {
|
||||
/// Official JSON array, one entity per line.
|
||||
WikidataDump,
|
||||
/// Official Special:EntityData/API entities object.
|
||||
WikidataEntities,
|
||||
/// Header-bearing Majestic CSV.
|
||||
MajesticCsv,
|
||||
/// Registry's documented `BigQuery` projection: `origin,rank,yyyymm,country_code`.
|
||||
CruxCsv,
|
||||
/// Current Curlie tar.gz containing literal TSV files.
|
||||
CurlieTarGz,
|
||||
/// UTF-8 PSL text.
|
||||
PslText,
|
||||
}
|
||||
|
||||
/// Compression of the downloaded object (Curlie tar.gz uses `None` here).
|
||||
#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, clap::ValueEnum)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Compression {
|
||||
/// Uncompressed, or intrinsically compressed archive format.
|
||||
#[default]
|
||||
None,
|
||||
/// Concatenated gzip members.
|
||||
Gzip,
|
||||
/// Concatenated bzip2 streams.
|
||||
Bzip2,
|
||||
}
|
||||
|
||||
/// An immutable, externally auditable input declaration; local paths are separate.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SourceManifest {
|
||||
/// Must be `argand.site-source/v1`.
|
||||
pub schema: String,
|
||||
/// Approved provider.
|
||||
pub source: Source,
|
||||
/// Exact source format.
|
||||
pub format: Format,
|
||||
/// Outer object compression.
|
||||
#[serde(default)]
|
||||
pub compression: Compression,
|
||||
/// Source-native snapshot/revision identifier.
|
||||
pub snapshot: String,
|
||||
/// Replacement scope, e.g. `full` or `selection:facebook`.
|
||||
pub scope: String,
|
||||
/// Original distribution URL, not an arbitrary mirror.
|
||||
pub source_url: String,
|
||||
/// Exact data license identifier.
|
||||
pub license: String,
|
||||
/// Authoritative license evidence URL.
|
||||
pub license_url: String,
|
||||
/// Actual retrieval time, distinct from the observation period.
|
||||
pub retrieved_at: DateTime<Utc>,
|
||||
/// Hash over the original compressed bytes.
|
||||
pub sha256: String,
|
||||
/// Original compressed object length.
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
impl SourceManifest {
|
||||
/// Checks source, license, identity, and format consistency.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a descriptive error for unsupported source declarations.
|
||||
pub fn validate(&self) -> anyhow::Result<()> {
|
||||
ensure!(
|
||||
self.schema == "argand.site-source/v1",
|
||||
"unsupported source schema"
|
||||
);
|
||||
ensure!(
|
||||
self.license == self.source.license() && self.license_url == self.source.license_url(),
|
||||
"source license evidence mismatch"
|
||||
);
|
||||
ensure!(
|
||||
self.bytes > 0 && valid_digest(&self.sha256),
|
||||
"invalid source length or digest"
|
||||
);
|
||||
ensure!(
|
||||
!self.snapshot.is_empty()
|
||||
&& self.snapshot.len() <= 512
|
||||
&& !self.scope.is_empty()
|
||||
&& self.scope.len() <= 128,
|
||||
"invalid snapshot or replacement scope"
|
||||
);
|
||||
let valid = matches!(
|
||||
(self.source, self.format),
|
||||
(
|
||||
Source::Wikidata,
|
||||
Format::WikidataDump | Format::WikidataEntities
|
||||
) | (Source::Majestic, Format::MajesticCsv)
|
||||
| (Source::Crux, Format::CruxCsv)
|
||||
| (Source::Curlie, Format::CurlieTarGz)
|
||||
| (Source::Psl, Format::PslText)
|
||||
);
|
||||
ensure!(valid, "source/format mismatch");
|
||||
crate::download::validate_source_url(self.source, &self.source_url)?;
|
||||
Ok(())
|
||||
}
|
||||
/// Content identity of the declaration, including original retrieval time.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a serialization error.
|
||||
pub fn id(&self) -> anyhow::Result<String> {
|
||||
Ok(crate::digest(&serde_json::to_vec(self)?))
|
||||
}
|
||||
}
|
||||
|
||||
/// A source assertion. `value` retains source-native qualifiers and references.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Fact {
|
||||
/// Source-namespaced subject; never inferred from a similar name.
|
||||
pub subject: String,
|
||||
/// Typed predicate understood by the projection, or a retained metadata key.
|
||||
pub predicate: String,
|
||||
/// Structured source value.
|
||||
pub value: Value,
|
||||
/// Source record JSON pointer or TSV column selector.
|
||||
pub selector: String,
|
||||
/// Evidence confidence on a 0..10000 policy scale, not calibrated probability.
|
||||
pub confidence: u16,
|
||||
}
|
||||
|
||||
/// Common adapter output. Records are retained even if every URL is rejected.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Record {
|
||||
/// Source-native entity ID or member/row locator.
|
||||
pub native_id: String,
|
||||
/// Complete relevant source record; no flattened evidence loss.
|
||||
pub raw: Value,
|
||||
/// Assertions derived from this record.
|
||||
pub facts: Vec<Fact>,
|
||||
}
|
||||
|
||||
/// Stable identifier for a source entity independent of names and revisions.
|
||||
#[must_use]
|
||||
pub fn entity_id(source: Source, native: &str) -> String {
|
||||
format!(
|
||||
"argand:entity:{}:{}",
|
||||
source.key(),
|
||||
crate::digest(native.as_bytes())
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether a string is a full lowercase SHA-256.
|
||||
#[must_use]
|
||||
pub fn valid_digest(value: &str) -> bool {
|
||||
value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
}
|
||||
163
crates/argand-site-registry/src/normalize.rs
Normal file
163
crates/argand-site-registry/src/normalize.rs
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
// By Nic Weyand!
|
||||
//! Strict registry URL identity, independent of search document equivalences.
|
||||
|
||||
use anyhow::{Context, ensure};
|
||||
use publicsuffix::{List, Psl};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
use url::{Host, Url};
|
||||
|
||||
/// PSL-derived information; `psl_source` binds every derived field to its input.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||
pub struct Domain {
|
||||
/// ASCII IDNA hostname.
|
||||
pub hostname: String,
|
||||
/// Registrable domain using ICANN and PRIVATE sections.
|
||||
pub registrable_domain: String,
|
||||
/// Effective public suffix.
|
||||
pub public_suffix: String,
|
||||
/// Whether the matched rule is in the PRIVATE section.
|
||||
pub private_suffix: bool,
|
||||
/// Immutable source manifest identity.
|
||||
pub psl_source: String,
|
||||
/// Content digest, stable when retrieval time changes but list bytes do not.
|
||||
pub psl_sha256: String,
|
||||
}
|
||||
|
||||
/// Strict URL identity and its derived domain fields.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||
pub struct WebProperty {
|
||||
/// Stable URL-derived Argand ID, unaffected by PSL refresh.
|
||||
pub id: String,
|
||||
/// Normalized absolute URL; meaningful paths and queries are preserved.
|
||||
pub url: String,
|
||||
/// Domain evidence.
|
||||
pub domain: Domain,
|
||||
}
|
||||
|
||||
/// A pinned dynamic PSL. It never refreshes during an import or query.
|
||||
pub struct Normalizer {
|
||||
list: List,
|
||||
source: String,
|
||||
content_sha256: String,
|
||||
}
|
||||
|
||||
impl Normalizer {
|
||||
/// Constructs the parser from an independently pinned source.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects malformed, empty, or unsectioned lists.
|
||||
pub fn new(bytes: &[u8], source: String) -> anyhow::Result<Self> {
|
||||
let text = std::str::from_utf8(bytes)?;
|
||||
ensure!(
|
||||
text.contains("===BEGIN ICANN DOMAINS===")
|
||||
&& text.contains("===END PRIVATE DOMAINS==="),
|
||||
"incomplete PSL sections"
|
||||
);
|
||||
let list = List::from_bytes(bytes).map_err(|e| anyhow::anyhow!("invalid PSL: {e}"))?;
|
||||
ensure!(!list.is_empty(), "empty PSL");
|
||||
Ok(Self {
|
||||
list,
|
||||
source,
|
||||
content_sha256: crate::digest(bytes),
|
||||
})
|
||||
}
|
||||
|
||||
/// Parses a hostname without inventing a website URL for popularity rows.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects invalid, special-use, IP, and suffix-only hosts.
|
||||
pub fn domain(&self, input: &str) -> anyhow::Result<Domain> {
|
||||
ensure!(
|
||||
!input.is_empty()
|
||||
&& input.len() <= 1024
|
||||
&& !input.ends_with("..")
|
||||
&& !input.contains(['/', ':', '@', '\\', '?', '#'])
|
||||
&& !input.chars().any(char::is_whitespace),
|
||||
"invalid hostname"
|
||||
);
|
||||
let host = match Host::parse(input.trim_end_matches('.'))? {
|
||||
Host::Domain(host) => host.to_ascii_lowercase(),
|
||||
_ => anyhow::bail!("IP literals are not registry destinations"),
|
||||
};
|
||||
ensure!(
|
||||
host.len() <= 253
|
||||
&& host.split('.').all(|label| !label.is_empty()
|
||||
&& label.len() <= 63
|
||||
&& !label.starts_with('-')
|
||||
&& !label.ends_with('-')
|
||||
&& label
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || b == b'-')),
|
||||
"invalid DNS name"
|
||||
);
|
||||
ensure!(
|
||||
!["localhost", "local", "internal", "test", "invalid", "onion"]
|
||||
.iter()
|
||||
.any(|suffix| host == *suffix || host.ends_with(&format!(".{suffix}"))),
|
||||
"special-use hostname"
|
||||
);
|
||||
let domain = self
|
||||
.list
|
||||
.domain(host.as_bytes())
|
||||
.context("hostname is a public suffix")?;
|
||||
ensure!(domain.suffix().is_known(), "unknown public suffix");
|
||||
Ok(Domain {
|
||||
registrable_domain: std::str::from_utf8(domain.as_bytes())?.to_owned(),
|
||||
public_suffix: std::str::from_utf8(domain.suffix().as_bytes())?.to_owned(),
|
||||
private_suffix: domain.suffix().typ() == Some(publicsuffix::Type::Private),
|
||||
hostname: host,
|
||||
psl_source: self.source.clone(),
|
||||
psl_sha256: self.content_sha256.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Normalizes syntax without assuming HTTP/HTTPS, www, or path equivalence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects malformed, credential-bearing, special-use and non-HTTP(S) URLs.
|
||||
pub fn url(&self, input: &str) -> anyhow::Result<WebProperty> {
|
||||
ensure!(
|
||||
input
|
||||
.split_once("://")
|
||||
.is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("http")
|
||||
|| scheme.eq_ignore_ascii_case("https")),
|
||||
"URL needs an explicit HTTP(S) authority"
|
||||
);
|
||||
ensure!(
|
||||
input.len() <= 8192
|
||||
&& !input.chars().any(|c| c.is_control() || c.is_whitespace())
|
||||
&& !input.contains('\\'),
|
||||
"invalid URL characters or length"
|
||||
);
|
||||
let mut url = Url::parse(input)?;
|
||||
ensure!(
|
||||
matches!(url.scheme(), "http" | "https")
|
||||
&& url.username().is_empty()
|
||||
&& url.password().is_none(),
|
||||
"unsupported scheme or credentials"
|
||||
);
|
||||
let domain = self.domain(url.host_str().context("missing hostname")?)?;
|
||||
url.set_host(Some(&domain.hostname))?;
|
||||
url.set_fragment(None);
|
||||
let normalized = url.to_string();
|
||||
Ok(WebProperty {
|
||||
id: format!("argand:web:{}", crate::digest(normalized.as_bytes())),
|
||||
url: normalized,
|
||||
domain,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Same NFC/lowercase/space rule as the existing Rust navigation consumer.
|
||||
/// Original names remain in the assertion store. Confusables are never folded.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects controls, invisible directional text, and oversized names.
|
||||
pub fn name_key(input: &str) -> anyhow::Result<String> {
|
||||
ensure!(input.len() <= 4096 && !input.chars().any(|c| c.is_control() || matches!(c, '\u{00ad}' | '\u{061c}' | '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2060}'..='\u{206f}' | '\u{feff}')), "unsafe name characters or length");
|
||||
let lower: String = input.nfc().flat_map(char::to_lowercase).collect();
|
||||
let key = lower.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
ensure!(!key.is_empty(), "empty name");
|
||||
Ok(key)
|
||||
}
|
||||
60
crates/argand-site-registry/src/observation.rs
Normal file
60
crates/argand-site-registry/src/observation.rs
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
// By Nic Weyand!
|
||||
//! Extension contract for later crawler evidence. No network or ownership inference.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Observed relationship, distinct from an entity-ownership claim.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum ObservationKind {
|
||||
/// An actual HTTP redirect with its status code.
|
||||
Redirect {
|
||||
/// HTTP redirect status.
|
||||
status: u16,
|
||||
},
|
||||
/// A page's declared canonical link.
|
||||
Canonical,
|
||||
/// A page's declared alternate locale.
|
||||
Hreflang {
|
||||
/// Unmodified declared locale.
|
||||
locale: String,
|
||||
},
|
||||
/// A JSON-LD sameAs assertion, not proof of ownership.
|
||||
JsonLdSameAs,
|
||||
/// A URL actually present in a fetched sitemap.
|
||||
Sitemap,
|
||||
/// A site-provided country-selector link.
|
||||
CountrySelector {
|
||||
/// Country as declared by the site.
|
||||
country: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// Immutable evidence coordinates for a future crawler-source adapter.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Observation {
|
||||
/// Relationship type.
|
||||
pub relation: ObservationKind,
|
||||
/// Actual page that asserted the relationship.
|
||||
pub from_url: String,
|
||||
/// Exact observed target, resolved against the recorded document base.
|
||||
pub to_url: String,
|
||||
/// Source provider/capture collection.
|
||||
pub source: String,
|
||||
/// Immutable source-native capture identifier.
|
||||
pub source_identifier: String,
|
||||
/// License or rights declaration verified for this evidence.
|
||||
pub license: String,
|
||||
/// License evidence URL.
|
||||
pub license_url: String,
|
||||
/// Retrieval instant.
|
||||
pub retrieved_at: DateTime<Utc>,
|
||||
/// Captured source-content hash.
|
||||
pub content_sha256: String,
|
||||
/// JSON pointer, header name, or DOM selector for the assertion.
|
||||
pub selector: String,
|
||||
/// Confidence on the same 0..10000 policy scale as source facts.
|
||||
pub confidence: u16,
|
||||
}
|
||||
332
crates/argand-site-registry/src/query.rs
Normal file
332
crates/argand-site-registry/src/query.rs
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
// By Nic Weyand!
|
||||
//! Bounded native lookup; ambiguity is counted before limits or policy filtering.
|
||||
|
||||
use crate::{build::Receipt, normalize::name_key};
|
||||
use anyhow::{Context, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::Path;
|
||||
|
||||
/// Open verified generation. Hashes are checked once, outside the query path.
|
||||
pub struct Registry {
|
||||
pub(crate) db: Connection,
|
||||
/// External receipt pin supplied by the caller.
|
||||
pub identity: String,
|
||||
/// Verified manifest.
|
||||
pub receipt: Receipt,
|
||||
}
|
||||
|
||||
/// One entity/property assertion with provenance and review state.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct Candidate {
|
||||
/// Explicit active identity reviews used to connect source entities.
|
||||
pub identity_provenance: Vec<Value>,
|
||||
/// Canonical name, aliases and entity metadata with their source facts.
|
||||
pub entity: crate::evidence::Entity,
|
||||
/// Stable entity identity.
|
||||
pub entity_id: String,
|
||||
/// Deterministic canonical display name.
|
||||
pub canonical_name: String,
|
||||
/// Exact proposed destination.
|
||||
pub url: String,
|
||||
/// URL, hostname, registrable-domain and pinned PSL provenance.
|
||||
pub web_property: Value,
|
||||
/// Conflicting source/reviewer scope assertions, without destructive merging.
|
||||
pub property_scopes: Vec<crate::evidence::PropertyScope>,
|
||||
/// Assertion relation, never a rank-derived ownership inference.
|
||||
pub relation: String,
|
||||
/// Conservative minimum source-assertion confidence, separate from review.
|
||||
pub confidence: u16,
|
||||
/// Exact evidence fingerprint to review.
|
||||
pub fingerprint: String,
|
||||
/// Statement qualifiers and identity evidence.
|
||||
pub evidence: Value,
|
||||
/// Every source fact supporting this assertion.
|
||||
pub provenance: Vec<Value>,
|
||||
/// Latest review, including expiry and its own provenance declaration.
|
||||
pub review: Option<Value>,
|
||||
/// Whether this assertion can be considered for approval.
|
||||
pub eligible: bool,
|
||||
}
|
||||
|
||||
/// Audit result. Counts include all matches before the output limit.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Lookup {
|
||||
/// Normalized query.
|
||||
pub query: String,
|
||||
/// Distinct matching entities, including rejected/expired alternatives.
|
||||
pub total_entities: u64,
|
||||
/// Total assertion groups across matching entities.
|
||||
pub total_edges: u64,
|
||||
/// True when the output limit omits assertions.
|
||||
pub truncated: bool,
|
||||
/// Bounded candidate list.
|
||||
pub candidates: Vec<Candidate>,
|
||||
/// License attribution required when displaying imported names.
|
||||
pub attribution: Value,
|
||||
}
|
||||
|
||||
/// Complete exact-name alternatives, independent of display limits or reviews.
|
||||
/// The identity binds the whole immutable registry and all matching name/edge
|
||||
/// fingerprints. It is evidence for query review, not a destination approval.
|
||||
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
|
||||
pub struct SelectionContext {
|
||||
/// SHA-256 of the complete normalized alternative context.
|
||||
pub identity: String,
|
||||
/// Versioned native name key.
|
||||
pub normalized_query: String,
|
||||
/// Matching entities, including those without an eligible website.
|
||||
pub total_entities: u64,
|
||||
/// All matching assertion edges, before any output limit.
|
||||
pub total_edges: u64,
|
||||
/// Whether the requested fingerprint belongs to a matching entity.
|
||||
pub selected_assertion_present: bool,
|
||||
}
|
||||
|
||||
impl Registry {
|
||||
/// Streams the complete alternative context for an explicitly selected edge.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects malformed keys/fingerprints and corrupt registry data.
|
||||
pub fn selection_context(
|
||||
&self,
|
||||
query: &str,
|
||||
fingerprint: &str,
|
||||
) -> anyhow::Result<SelectionContext> {
|
||||
use sha2::{Digest, Sha256};
|
||||
ensure!(
|
||||
crate::model::valid_digest(fingerprint),
|
||||
"invalid selection fingerprint"
|
||||
);
|
||||
let key = name_key(query)?;
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(serde_json::to_vec(&(
|
||||
"argand.site-selection/v1",
|
||||
&self.identity,
|
||||
&key,
|
||||
))?);
|
||||
let mut statement = self.db.prepare(
|
||||
"SELECT n.id,n.names_fingerprint,e.fingerprint FROM entities n LEFT JOIN edges e ON e.entity=n.id WHERE n.id IN(SELECT entity FROM names WHERE key=?1) ORDER BY n.id,e.fingerprint"
|
||||
)?;
|
||||
let mut rows = statement.query([&key])?;
|
||||
let mut previous = None;
|
||||
let mut total_entities = 0;
|
||||
let mut total_edges = 0;
|
||||
let mut present = false;
|
||||
while let Some(row) = rows.next()? {
|
||||
let entity: String = row.get(0)?;
|
||||
let names: String = row.get(1)?;
|
||||
let edge: Option<String> = row.get(2)?;
|
||||
hash.update(serde_json::to_vec(&(&entity, names, &edge))?);
|
||||
if previous.as_ref() != Some(&entity) {
|
||||
total_entities += 1;
|
||||
previous = Some(entity);
|
||||
}
|
||||
if let Some(edge) = edge {
|
||||
total_edges += 1;
|
||||
present |= edge == fingerprint;
|
||||
}
|
||||
}
|
||||
Ok(SelectionContext {
|
||||
identity: format!("{:x}", hash.finalize()),
|
||||
normalized_query: key,
|
||||
total_entities,
|
||||
total_edges,
|
||||
selected_assertion_present: present,
|
||||
})
|
||||
}
|
||||
|
||||
/// Opens only a complete, externally pinned generation.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects altered receipts/databases and unsupported contracts.
|
||||
pub fn open(path: &Path, expected_pin: &str) -> anyhow::Result<Self> {
|
||||
ensure!(
|
||||
crate::model::valid_digest(expected_pin),
|
||||
"provide a full trusted receipt SHA-256"
|
||||
);
|
||||
ensure!(
|
||||
crate::file_digest(&path.join("COMPLETE.json"))? == expected_pin,
|
||||
"receipt pin mismatch"
|
||||
);
|
||||
let receipt: Receipt = crate::read_json(&path.join("COMPLETE.json"))?;
|
||||
ensure!(
|
||||
crate::file_digest(&path.join("LICENSE_SOURCES.md"))? == receipt.licenses_sha256
|
||||
&& crate::file_digest(&path.join("ATTRIBUTION.json"))?
|
||||
== receipt.attribution_sha256,
|
||||
"registry license or attribution digest mismatch"
|
||||
);
|
||||
ensure!(
|
||||
receipt.schema == "argand.site-registry/v1"
|
||||
&& receipt.rules == crate::store::RULE_VERSION,
|
||||
"unsupported registry contract"
|
||||
);
|
||||
let database = path.join("registry.sqlite");
|
||||
ensure!(
|
||||
std::fs::symlink_metadata(&database)?.is_file(),
|
||||
"database must be a regular file"
|
||||
);
|
||||
ensure!(
|
||||
crate::file_digest(&database)? == receipt.database_sha256,
|
||||
"registry database digest mismatch"
|
||||
);
|
||||
let db = Connection::open_with_flags(database, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
||||
crate::store::configure(&db)?;
|
||||
Ok(Self {
|
||||
db,
|
||||
identity: expected_pin.into(),
|
||||
receipt,
|
||||
})
|
||||
}
|
||||
|
||||
/// Indexed exact-name/alias lookup, with counts independent of result limits.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns invalid query bounds, malformed data, or SQLite errors.
|
||||
pub fn lookup(&self, query: &str, limit: u32) -> anyhow::Result<Lookup> {
|
||||
ensure!((1..=100).contains(&limit), "lookup limit must be 1..100");
|
||||
let key = name_key(query)?;
|
||||
let total_entities = self.db.query_row(
|
||||
"SELECT count(DISTINCT entity) FROM names WHERE key=?1",
|
||||
[&key],
|
||||
|r| crate::store::unsigned(r, 0),
|
||||
)?;
|
||||
let total_edges = self.db.query_row(
|
||||
"SELECT count(*) FROM edges WHERE entity IN(SELECT entity FROM names WHERE key=?1)",
|
||||
[&key],
|
||||
|r| crate::store::unsigned(r, 0),
|
||||
)?;
|
||||
let mut stmt=self.db.prepare("SELECT fingerprint FROM edges WHERE entity IN(SELECT entity FROM names WHERE key=?1) ORDER BY entity,property,fingerprint LIMIT ?2")?;
|
||||
let ids = stmt
|
||||
.query_map(params![key, limit], |r| r.get::<_, String>(0))?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let candidates = ids
|
||||
.iter()
|
||||
.map(|id| self.candidate(id))
|
||||
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||
Ok(Lookup {
|
||||
query: key,
|
||||
total_entities,
|
||||
total_edges,
|
||||
truncated: total_edges > u64::from(limit),
|
||||
candidates,
|
||||
attribution: crate::release::attribution(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns one assertion and its complete supporting source declarations.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails for absent fingerprints or corrupt projection data.
|
||||
pub fn candidate(&self, fingerprint: &str) -> anyhow::Result<Candidate> {
|
||||
let (entity,name,url,property,relation,evidence,eligible,facts):(String,String,String,String,String,String,bool,String)=self.db.query_row("SELECT e.entity,n.canonical_name,p.url,p.derived_json,e.relation,e.evidence,e.eligible,e.facts FROM edges e JOIN entities n ON n.id=e.entity JOIN properties p ON p.id=e.property WHERE e.fingerprint=?1",[fingerprint],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?)))?;
|
||||
let ids: Vec<String> = serde_json::from_str(&facts)?;
|
||||
let mut provenance = Vec::new();
|
||||
for id in ids {
|
||||
provenance.push(crate::evidence::fact(&self.db, &id)?);
|
||||
}
|
||||
let review=self.db.query_row("SELECT sequence,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country FROM reviews WHERE fingerprint=?1 ORDER BY sequence DESC LIMIT 1",[fingerprint],|r|Ok(json!({"sequence":crate::store::unsigned(r,0)?,"decision":r.get::<_,String>(1)?,"reviewer":r.get::<_,String>(2)?,"reason":r.get::<_,String>(3)?,"evidence":r.get::<_,String>(4)?,"retrieved_at":r.get::<_,String>(5)?,"expires_at":r.get::<_,String>(6)?,"role":r.get::<_,String>(7)?,"locale":r.get::<_,String>(8)?,"country":r.get::<_,String>(9)?,"source":"argand_operator_review","source_identifier":format!("{}:{}",fingerprint,crate::store::unsigned(r,0)?),"license":"CC0-1.0","license_url":"https://creativecommons.org/publicdomain/zero/1.0/","confidence":9000}))).optional()?;
|
||||
Ok(Candidate {
|
||||
identity_provenance: Vec::new(),
|
||||
entity: crate::evidence::entity(&self.db, &entity, &name)?,
|
||||
property_scopes: crate::evidence::scopes(
|
||||
&serde_json::from_str(&evidence)?,
|
||||
&provenance,
|
||||
review.as_ref(),
|
||||
)?,
|
||||
entity_id: entity,
|
||||
canonical_name: name,
|
||||
url,
|
||||
web_property: serde_json::from_str(&property)?,
|
||||
relation,
|
||||
confidence: provenance
|
||||
.iter()
|
||||
.filter_map(|p| p["confidence"].as_u64())
|
||||
.min()
|
||||
.map(u16::try_from)
|
||||
.transpose()?
|
||||
.unwrap_or(0),
|
||||
fingerprint: fingerprint.into(),
|
||||
evidence: serde_json::from_str(&evidence)?,
|
||||
provenance,
|
||||
review,
|
||||
eligible,
|
||||
})
|
||||
}
|
||||
|
||||
/// Chooses an approved regional property or an explicitly reviewed primary.
|
||||
/// Never returns a destination for ambiguous entities or tied properties.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns malformed requests/data or database failures.
|
||||
pub fn resolve(
|
||||
&self,
|
||||
query: &str,
|
||||
locale: Option<&str>,
|
||||
country: Option<&str>,
|
||||
now: DateTime<Utc>,
|
||||
) -> anyhow::Result<Option<Candidate>> {
|
||||
let candidates = crate::identity::candidates(self, query, now)?;
|
||||
let mut best = None;
|
||||
let mut score = 0;
|
||||
let mut ambiguous = false;
|
||||
for candidate in candidates {
|
||||
if !candidate.eligible {
|
||||
continue;
|
||||
}
|
||||
let Some(review) = candidate.review.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if review["decision"] != "approve" {
|
||||
continue;
|
||||
}
|
||||
let expires = DateTime::parse_from_rfc3339(
|
||||
review["expires_at"]
|
||||
.as_str()
|
||||
.context("invalid review expiry")?,
|
||||
)?;
|
||||
let starts = DateTime::parse_from_rfc3339(
|
||||
review["retrieved_at"]
|
||||
.as_str()
|
||||
.context("invalid review timestamp")?,
|
||||
)?;
|
||||
if now < starts || now >= expires {
|
||||
continue;
|
||||
}
|
||||
let region = review["country"].as_str().unwrap_or_default();
|
||||
let language = review["locale"].as_str().unwrap_or_default();
|
||||
let matches_country =
|
||||
!region.is_empty() && country.is_some_and(|c| c.eq_ignore_ascii_case(region));
|
||||
let matches_locale =
|
||||
!language.is_empty() && locale.is_some_and(|l| l.eq_ignore_ascii_case(language));
|
||||
let current = if review["role"] == "regional" {
|
||||
// All asserted dimensions must match; a language alone cannot
|
||||
// override an explicit country mismatch.
|
||||
if (!region.is_empty() && !matches_country)
|
||||
|| (!language.is_empty() && !matches_locale)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
2 + u8::from(matches_country) + u8::from(matches_locale)
|
||||
} else if review["role"] == "primary" {
|
||||
1
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
if current > score {
|
||||
best = Some(candidate);
|
||||
score = current;
|
||||
ambiguous = false;
|
||||
} else if current == score
|
||||
&& best
|
||||
.as_ref()
|
||||
.is_some_and(|b: &Candidate| b.url != candidate.url)
|
||||
{
|
||||
ambiguous = true;
|
||||
}
|
||||
}
|
||||
Ok(if ambiguous { None } else { best })
|
||||
}
|
||||
}
|
||||
237
crates/argand-site-registry/src/release.rs
Normal file
237
crates/argand-site-registry/src/release.rs
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
// By Nic Weyand!
|
||||
//! Provenance-bearing export and externally authenticated release activation.
|
||||
|
||||
use crate::query::Registry;
|
||||
use anyhow::{Context, ensure};
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::{BufWriter, Write},
|
||||
path::Path,
|
||||
process::{Command, Stdio},
|
||||
};
|
||||
|
||||
/// Source terms shipped and authenticated with every generation.
|
||||
pub const LICENSES: &str = include_str!("../LICENSE_SOURCES.md");
|
||||
|
||||
/// Source attribution envelope for CLI and JSON consumers.
|
||||
#[must_use]
|
||||
pub fn attribution() -> Value {
|
||||
json!({"wikidata":{"license":"CC0-1.0","url":"https://www.wikidata.org/"},
|
||||
"majestic":{"license":"CC-BY-3.0","credit":"Majestic Million, Majestic","url":"https://majestic.com/reports/majestic-million","license_url":"https://creativecommons.org/licenses/by/3.0/"},
|
||||
"crux":{"license":"CC-BY-4.0","credit":"Chrome UX Report, Google","url":"https://developer.chrome.com/docs/crux/","license_url":"https://creativecommons.org/licenses/by/4.0/"},
|
||||
"curlie":{"license":"CC-BY-3.0","credit":"With content from Curlie.org - the largest human-edited directory of the web. Contribute by submitting a website or becoming an editor.","url":"https://curlie.org/","license_url":"https://creativecommons.org/licenses/by/3.0/","public_display":"Use the prescribed HTML attribution on every page using Curlie content: https://curlie.org/docs/en/license.html"},
|
||||
"psl":{"license":"MPL-2.0","url":"https://publicsuffix.org/list/","license_url":"https://mozilla.org/MPL/2.0/"},
|
||||
"changes":"Argand normalizes and combines assertions; provider endorsement is not implied."})
|
||||
}
|
||||
|
||||
/// Streams all retained facts with their source declarations and attribution.
|
||||
/// Description values and raw records are omitted unless explicitly requested.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects existing destinations or malformed registry data.
|
||||
pub fn export(
|
||||
registry: &Registry,
|
||||
output: &Path,
|
||||
include_descriptions: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
argand_atomic::create_durable_with(output, |file| {
|
||||
export_inner(registry, file, include_descriptions).map_err(std::io::Error::other)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn export_inner(
|
||||
registry: &Registry,
|
||||
file: &mut File,
|
||||
include_descriptions: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut writer = BufWriter::new(file);
|
||||
writeln!(
|
||||
writer,
|
||||
"{}",
|
||||
json!({"schema":"argand.site-export/v1","registry":registry.identity,"attribution":attribution(),"descriptions_included":include_descriptions})
|
||||
)?;
|
||||
let mut stmt=registry.db.prepare("SELECT f.id,f.subject,f.predicate,f.value,f.selector,f.confidence,s.manifest,r.native_id,f.source_id FROM facts f JOIN sources s ON s.id=f.source_id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE s.complete=1 ORDER BY f.id")?;
|
||||
let mut rows = stmt.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let predicate: String = row.get(2)?;
|
||||
if !include_descriptions && predicate == "description" {
|
||||
continue;
|
||||
}
|
||||
let mut value: Value = serde_json::from_str(&row.get::<_, String>(3)?)?;
|
||||
if !include_descriptions
|
||||
&& predicate == "category"
|
||||
&& let Some(object) = value.as_object_mut()
|
||||
{
|
||||
object.remove("description");
|
||||
}
|
||||
writeln!(
|
||||
writer,
|
||||
"{}",
|
||||
json!({"type":"assertion","id":row.get::<_,String>(0)?,"subject":row.get::<_,String>(1)?,"predicate":predicate,"value":value,"selector":row.get::<_,String>(4)?,"confidence":row.get::<_,u16>(5)?,"source":serde_json::from_str::<Value>(&row.get::<_,String>(6)?)?,"source_identifier":row.get::<_,String>(7)?,"source_snapshot_id":row.get::<_,String>(8)?,"description_redacted":!include_descriptions && predicate=="category"})
|
||||
)?;
|
||||
}
|
||||
writer.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Signs a complete release using the operator's SSH signing key.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns missing key, existing signature, and signing process failures.
|
||||
pub fn sign(generation: &Path, key: &Path, pin: &str) -> anyhow::Result<()> {
|
||||
Registry::open(generation, pin)?;
|
||||
ensure!(
|
||||
!generation.join("COMPLETE.json.sig").exists(),
|
||||
"signature already exists"
|
||||
);
|
||||
let status = Command::new("ssh-keygen")
|
||||
.args(["-Y", "sign", "-n", "argand-site-registry", "-f"])
|
||||
.arg(key)
|
||||
.arg(generation.join("COMPLETE.json"))
|
||||
.status()?;
|
||||
ensure!(status.success(), "SSH signing failed");
|
||||
File::open(generation.join("COMPLETE.json.sig"))?.sync_all()?;
|
||||
File::open(generation)?.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verifies a signature against an external allowed-signers trust file.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects untrusted signatures, altered receipts, and corrupt databases.
|
||||
pub fn verify_signed(
|
||||
generation: &Path,
|
||||
signers: &Path,
|
||||
identity: &str,
|
||||
) -> anyhow::Result<Registry> {
|
||||
let pin = crate::file_digest(&generation.join("COMPLETE.json"))?;
|
||||
let status = Command::new("ssh-keygen")
|
||||
.args(["-Y", "verify", "-n", "argand-site-registry", "-f"])
|
||||
.arg(signers)
|
||||
.arg("-I")
|
||||
.arg(identity)
|
||||
.arg("-s")
|
||||
.arg(generation.join("COMPLETE.json.sig"))
|
||||
.stdin(Stdio::from(File::open(generation.join("COMPLETE.json"))?))
|
||||
.stdout(Stdio::null())
|
||||
.status()?;
|
||||
ensure!(status.success(), "untrusted registry signature");
|
||||
Registry::open(generation, &pin)
|
||||
}
|
||||
|
||||
/// Activates a verified generation using one durable pointer. Refuses rollback
|
||||
/// past distributed revocations; rebuild old inputs with the current review log.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns signature, rollback, lock, or filesystem failures.
|
||||
pub fn activate(
|
||||
generation: &Path,
|
||||
current: &Path,
|
||||
signers: &Path,
|
||||
identity: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let registry = verify_signed(generation, signers, identity)?;
|
||||
let parent = current
|
||||
.parent()
|
||||
.context("current pointer needs a parent directory")?;
|
||||
fs::create_dir_all(parent)?;
|
||||
let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(parent.join("activation.lock"))?;
|
||||
lock.try_lock().context("another activation is running")?;
|
||||
let revocation: u64 = registry.db.query_row(
|
||||
"SELECT coalesce(max(sequence),0) FROM reviews WHERE decision='revoke'",
|
||||
[],
|
||||
|r| crate::store::unsigned(r, 0),
|
||||
)?;
|
||||
if current.exists() {
|
||||
let previous: Value = crate::read_json(current)?;
|
||||
ensure!(
|
||||
revocation
|
||||
>= previous["revocation_sequence"]
|
||||
.as_u64()
|
||||
.context("invalid current pointer")?,
|
||||
"rollback would discard revocations; rebuild using the current review log"
|
||||
);
|
||||
let old = Registry::open(
|
||||
Path::new(
|
||||
previous["generation"]
|
||||
.as_str()
|
||||
.context("invalid previous generation")?,
|
||||
),
|
||||
previous["receipt_sha256"]
|
||||
.as_str()
|
||||
.context("invalid previous receipt")?,
|
||||
)?;
|
||||
preserve_revocations(&old, ®istry)?;
|
||||
}
|
||||
argand_atomic::replace_durable(
|
||||
current,
|
||||
&serde_json::to_vec_pretty(
|
||||
&json!({"schema":"argand.site-current/v1","generation":generation.canonicalize()?,"receipt_sha256":registry.identity,"signer":identity,"revocation_sequence":revocation}),
|
||||
)?,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn preserve_revocations(old: &Registry, new: &Registry) -> anyhow::Result<()> {
|
||||
// A sequence number alone is insufficient: a forked log could contain an
|
||||
// unrelated revocation with a larger sequence. Require every exact old entry.
|
||||
let mut statement = old
|
||||
.db
|
||||
.prepare("SELECT * FROM reviews WHERE decision='revoke' ORDER BY sequence")?;
|
||||
let mut rows = statement.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let sequence: i64 = row.get(0)?;
|
||||
let expected = (1..11)
|
||||
.map(|i| row.get::<_, String>(i))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let found = new
|
||||
.db
|
||||
.query_row("SELECT * FROM reviews WHERE sequence=?1", [sequence], |r| {
|
||||
(1..11)
|
||||
.map(|i| r.get::<_, String>(i))
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
})
|
||||
.context("rollback would discard a revocation")?;
|
||||
ensure!(
|
||||
found == expected,
|
||||
"rollback would replace revocation history"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Streams added/removed assertion fingerprints between two pinned generations.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns query or output errors.
|
||||
pub fn diff(old: &Registry, new: &Registry, output: &mut dyn Write) -> anyhow::Result<()> {
|
||||
for (kind, from, to) in [("removed", old, new), ("added", new, old)] {
|
||||
let mut stmt = from
|
||||
.db
|
||||
.prepare("SELECT fingerprint,entity,property FROM edges ORDER BY fingerprint")?;
|
||||
let mut rows = stmt.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let fingerprint: String = row.get(0)?;
|
||||
let exists: bool = to.db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM edges WHERE fingerprint=?1)",
|
||||
[&fingerprint],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if !exists {
|
||||
writeln!(
|
||||
output,
|
||||
"{}",
|
||||
json!({"change":kind,"fingerprint":fingerprint,"entity":row.get::<_,String>(1)?,"property":row.get::<_,String>(2)?})
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
116
crates/argand-site-registry/src/review.rs
Normal file
116
crates/argand-site-registry/src/review.rs
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
// By Nic Weyand!
|
||||
//! Append-only operator decisions bound to exact assertions and all entity names.
|
||||
|
||||
use crate::query::Registry;
|
||||
use anyhow::ensure;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rusqlite::{Connection, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Explicit review input, independent of imported source confidence.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Review {
|
||||
/// Exact fingerprint printed by pinned lookup.
|
||||
pub fingerprint: String,
|
||||
/// `approve` or `revoke`.
|
||||
pub decision: String,
|
||||
/// Human/operator identity, carried into signed release.
|
||||
pub reviewer: String,
|
||||
/// Explanation, including how ownership and the role were checked.
|
||||
pub reason: String,
|
||||
/// Immutable external evidence reference or digest.
|
||||
pub evidence: String,
|
||||
/// Actual decision timestamp.
|
||||
pub reviewed_at: DateTime<Utc>,
|
||||
/// Hard expiry; refresh never extends this automatically.
|
||||
pub expires_at: DateTime<Utc>,
|
||||
/// `primary`, `regional`, or `unspecified`.
|
||||
pub role: String,
|
||||
/// Explicitly reviewed locale, or empty if unspecified.
|
||||
pub locale: String,
|
||||
/// Explicitly reviewed two-letter country code, or empty.
|
||||
pub country: String,
|
||||
}
|
||||
|
||||
/// Appends a decision after verifying the exact generation and fingerprint.
|
||||
/// Rebuild and promote to distribute it; existing immutable artifacts never mutate.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects missing evidence, expired/overlong approval, and ineligible assertions.
|
||||
pub fn record(db: &Connection, registry: &Registry, review: &Review) -> anyhow::Result<u64> {
|
||||
validate(review)?;
|
||||
let candidate = registry.candidate(&review.fingerprint)?;
|
||||
if review.decision == "approve" {
|
||||
ensure!(
|
||||
candidate.eligible,
|
||||
"deprecated/invalid assertion cannot be approved"
|
||||
);
|
||||
}
|
||||
// A reviewed assertion must originate in this store, not an unrelated artifact.
|
||||
for provenance in &candidate.provenance {
|
||||
let id = provenance["fact_id"].as_str().unwrap_or_default();
|
||||
let exists: bool = db.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM facts WHERE id=?1)",
|
||||
[id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(exists, "review source is absent from this store");
|
||||
}
|
||||
append(db, review)
|
||||
}
|
||||
|
||||
pub(crate) fn validate(review: &Review) -> anyhow::Result<()> {
|
||||
ensure!(
|
||||
matches!(review.decision.as_str(), "approve" | "revoke"),
|
||||
"invalid decision"
|
||||
);
|
||||
ensure!(
|
||||
!review.reviewer.trim().is_empty()
|
||||
&& review.reviewer.len() <= 256
|
||||
&& !review.reason.trim().is_empty()
|
||||
&& review.reason.len() <= 8192
|
||||
&& !review.evidence.trim().is_empty()
|
||||
&& review.evidence.len() <= 8192,
|
||||
"reviewer, reason and evidence are required and bounded"
|
||||
);
|
||||
ensure!(
|
||||
matches!(review.role.as_str(), "primary" | "regional" | "unspecified"),
|
||||
"invalid property role"
|
||||
);
|
||||
ensure!(
|
||||
review.locale.len() <= 64
|
||||
&& review
|
||||
.locale
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || b == b'-'),
|
||||
"invalid locale"
|
||||
);
|
||||
ensure!(
|
||||
review.country.is_empty()
|
||||
|| (review.country.len() == 2
|
||||
&& review.country.bytes().all(|b| b.is_ascii_uppercase())),
|
||||
"country must be uppercase two-letter code"
|
||||
);
|
||||
ensure!(
|
||||
review.role != "regional" || !review.locale.is_empty() || !review.country.is_empty(),
|
||||
"regional role needs locale or country evidence"
|
||||
);
|
||||
ensure!(
|
||||
review.role != "primary" || (review.locale.is_empty() && review.country.is_empty()),
|
||||
"scoped destinations use the regional role; primary is the global fallback"
|
||||
);
|
||||
if review.decision == "approve" {
|
||||
ensure!(
|
||||
review.expires_at > review.reviewed_at
|
||||
&& review.expires_at - review.reviewed_at <= chrono::Duration::days(90),
|
||||
"approval must expire within 90 days"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn append(db: &Connection, review: &Review) -> anyhow::Result<u64> {
|
||||
db.execute("INSERT INTO reviews(fingerprint,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",params![review.fingerprint,review.decision,review.reviewer,review.reason,review.evidence,review.reviewed_at.to_rfc3339(),review.expires_at.to_rfc3339(),review.role,review.locale,review.country])?;
|
||||
Ok(u64::try_from(db.last_insert_rowid())?)
|
||||
}
|
||||
209
crates/argand-site-registry/src/store.rs
Normal file
209
crates/argand-site-registry/src/store.rs
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
// By Nic Weyand!
|
||||
//! Transactional source import and durable replay checkpoints.
|
||||
|
||||
use crate::{
|
||||
adapters::{self, RecordSink},
|
||||
model::{Compression, Record, SourceManifest},
|
||||
};
|
||||
use anyhow::{Context, ensure};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{BufReader, Read},
|
||||
path::Path,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// Adapter/normalization contract recorded in all generation identities.
|
||||
pub const RULE_VERSION: &str = "argand.site-rules/v1";
|
||||
|
||||
/// Opens or migrates the local assertion store with bounded page cache.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects newer schemas and SQLite/filesystem failures.
|
||||
pub fn open(path: &Path) -> anyhow::Result<Connection> {
|
||||
if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let db = Connection::open(path)?;
|
||||
configure(&db)?;
|
||||
let version: i64 = db.query_row("PRAGMA user_version", [], |r| r.get(0))?;
|
||||
match version {
|
||||
0 => {
|
||||
db.execute_batch("BEGIN IMMEDIATE")?;
|
||||
db.execute_batch(include_str!("../migrations/001.sql"))?;
|
||||
db.execute_batch("COMMIT")?;
|
||||
}
|
||||
1 => {}
|
||||
_ => anyhow::bail!("unsupported registry schema {version}"),
|
||||
}
|
||||
let rules: String = db.query_row(
|
||||
"SELECT rules FROM registry_metadata WHERE singleton=1",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
rules == RULE_VERSION,
|
||||
"source parser rules changed; migrate explicitly or reimport pinned sources into a new store"
|
||||
);
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
pub(crate) fn configure(db: &Connection) -> anyhow::Result<()> {
|
||||
db.busy_timeout(Duration::from_secs(10))?;
|
||||
db.execute_batch("PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA cache_size=-8192; PRAGMA temp_store=FILE; PRAGMA synchronous=FULL;")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verifies a source object before and after parsing, publishing completion last.
|
||||
/// Incomplete transactions are rolled back; committed batches replay idempotently.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns source integrity, parser, filesystem, or database errors.
|
||||
pub fn import(
|
||||
db: &mut Connection,
|
||||
manifest: &SourceManifest,
|
||||
path: &Path,
|
||||
) -> anyhow::Result<String> {
|
||||
manifest.validate()?;
|
||||
verify_input(manifest, path)?;
|
||||
let id = manifest.id()?;
|
||||
let declaration = serde_json::to_string(manifest)?;
|
||||
db.execute("INSERT OR IGNORE INTO sources(id,source,scope,retrieved_at,manifest) VALUES(?1,?2,?3,?4,?5)", params![id, manifest.source.key(),manifest.scope,manifest.retrieved_at.to_rfc3339(),declaration])?;
|
||||
let (checkpoint, complete): (u64, bool) = db.query_row(
|
||||
"SELECT checkpoint,complete FROM sources WHERE id=?1",
|
||||
[&id],
|
||||
|r| Ok((unsigned(r, 0)?, r.get(1)?)),
|
||||
)?;
|
||||
if complete {
|
||||
return Ok(id);
|
||||
}
|
||||
let file = File::open(path)?;
|
||||
let reader: Box<dyn Read> = match manifest.compression {
|
||||
Compression::None => Box::new(file),
|
||||
Compression::Gzip => Box::new(flate2::read::MultiGzDecoder::new(file)),
|
||||
Compression::Bzip2 => Box::new(bzip2::read::MultiBzDecoder::new(file)),
|
||||
};
|
||||
db.execute_batch("BEGIN IMMEDIATE")?;
|
||||
let result = {
|
||||
let mut sink = SqlSink {
|
||||
db,
|
||||
source: &id,
|
||||
ordinal: 0,
|
||||
checkpoint,
|
||||
};
|
||||
let parsed =
|
||||
adapters::adapter(manifest.format).ingest(&mut BufReader::new(reader), &mut sink);
|
||||
parsed.and_then(|()| {
|
||||
ensure!(
|
||||
sink.ordinal > 0 && sink.ordinal >= checkpoint,
|
||||
"source empty or shorter than checkpoint"
|
||||
);
|
||||
verify_input(manifest, path)?;
|
||||
sink.db.execute(
|
||||
"UPDATE sources SET checkpoint=?2,complete=1 WHERE id=?1",
|
||||
params![id, i64::try_from(sink.ordinal)?],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
};
|
||||
match result {
|
||||
Ok(()) => db.execute_batch("COMMIT")?,
|
||||
Err(error) => {
|
||||
db.execute_batch("ROLLBACK")?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
fn verify_input(manifest: &SourceManifest, path: &Path) -> anyhow::Result<()> {
|
||||
let metadata = std::fs::symlink_metadata(path)?;
|
||||
ensure!(
|
||||
metadata.is_file() && metadata.len() == manifest.bytes,
|
||||
"source length/type mismatch"
|
||||
);
|
||||
ensure!(
|
||||
crate::file_digest(path)? == manifest.sha256,
|
||||
"source digest mismatch"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SqlSink<'a> {
|
||||
db: &'a Connection,
|
||||
source: &'a str,
|
||||
ordinal: u64,
|
||||
checkpoint: u64,
|
||||
}
|
||||
|
||||
impl RecordSink for SqlSink<'_> {
|
||||
fn emit(&mut self, record: Record) -> anyhow::Result<()> {
|
||||
self.ordinal += 1;
|
||||
if self.ordinal <= self.checkpoint {
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
record.native_id.len() <= 8192 && record.facts.len() <= 100_000,
|
||||
"record identity/fact bound exceeded"
|
||||
);
|
||||
self.db
|
||||
.prepare_cached("INSERT INTO records VALUES(?1,?2,?3,?4)")?
|
||||
.execute(params![
|
||||
self.source,
|
||||
i64::try_from(self.ordinal)?,
|
||||
record.native_id,
|
||||
serde_json::to_string(&record.raw)?
|
||||
])?;
|
||||
for fact in record.facts {
|
||||
ensure!(fact.confidence <= 10000, "invalid confidence");
|
||||
let id = crate::digest(&serde_json::to_vec(&(self.source, self.ordinal, &fact))?);
|
||||
self.db
|
||||
.prepare_cached("INSERT OR IGNORE INTO facts VALUES(?1,?2,?3,?4,?5,?6,?7,?8)")?
|
||||
.execute(params![
|
||||
id,
|
||||
self.source,
|
||||
i64::try_from(self.ordinal)?,
|
||||
fact.subject,
|
||||
fact.predicate,
|
||||
serde_json::to_string(&fact.value)?,
|
||||
fact.selector,
|
||||
fact.confidence
|
||||
])?;
|
||||
}
|
||||
if self.ordinal.is_multiple_of(256) {
|
||||
self.db.execute(
|
||||
"UPDATE sources SET checkpoint=?2 WHERE id=?1",
|
||||
params![self.source, i64::try_from(self.ordinal)?],
|
||||
)?;
|
||||
self.db.execute_batch("COMMIT; BEGIN IMMEDIATE")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the manifest of a complete source snapshot.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails on missing, incomplete, or malformed declarations.
|
||||
pub fn source(db: &Connection, id: &str) -> anyhow::Result<SourceManifest> {
|
||||
let raw: String = db
|
||||
.query_row(
|
||||
"SELECT manifest FROM sources WHERE id=?1 AND complete=1",
|
||||
[id],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?
|
||||
.context("source is absent or incomplete")?;
|
||||
Ok(serde_json::from_str(&raw)?)
|
||||
}
|
||||
|
||||
pub(crate) fn unsigned(row: &rusqlite::Row<'_>, index: usize) -> rusqlite::Result<u64> {
|
||||
u64::try_from(row.get::<_, i64>(index)?).map_err(|e| {
|
||||
rusqlite::Error::FromSqlConversionFailure(
|
||||
index,
|
||||
rusqlite::types::Type::Integer,
|
||||
Box::new(e),
|
||||
)
|
||||
})
|
||||
}
|
||||
96
crates/argand-site-registry/src/update.rs
Normal file
96
crates/argand-site-registry/src/update.rs
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
// By Nic Weyand!
|
||||
//! Scheduler entry point; build candidates automatically and report failures.
|
||||
|
||||
use crate::{
|
||||
download::{CachedSource, Download},
|
||||
model::SourceManifest,
|
||||
};
|
||||
use anyhow::Context;
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fs::{self, OpenOptions},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
/// Update configuration, usable from systemd or cron without a resident daemon.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Config {
|
||||
/// Source cache root.
|
||||
pub cache: PathBuf,
|
||||
/// Mutable import/review database.
|
||||
pub database: PathBuf,
|
||||
/// Immutable generation parent.
|
||||
pub generations: PathBuf,
|
||||
/// Explicit bounded network downloads. `{date}` and `{month}` expand in snapshot.
|
||||
#[serde(default)]
|
||||
pub downloads: Vec<Download>,
|
||||
/// Pinned local sources, including existing acquisitions.
|
||||
#[serde(default)]
|
||||
pub inputs: Vec<CachedSource>,
|
||||
/// Explicit billed `CrUX` jobs, empty by default.
|
||||
#[serde(default)]
|
||||
pub crux: Vec<crate::crux::CruxDownload>,
|
||||
}
|
||||
|
||||
/// Runs all declared imports, refusing candidate publication on any failure.
|
||||
/// Complete prior generations remain available. Does not sign or activate.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns configuration, source, lock, import, or build errors.
|
||||
pub async fn run(config: &Config) -> anyhow::Result<PathBuf> {
|
||||
fs::create_dir_all(&config.generations)?;
|
||||
let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(config.generations.join("update.lock"))?;
|
||||
lock.try_lock().context("registry update already running")?;
|
||||
let mut inputs = config.inputs.clone();
|
||||
let now = Utc::now();
|
||||
for request in &config.downloads {
|
||||
let mut request = request.clone();
|
||||
request.snapshot = request
|
||||
.snapshot
|
||||
.replace("{date}", &now.format("%Y-%m-%d").to_string())
|
||||
.replace("{month}", &now.format("%Y-%m").to_string());
|
||||
inputs.push(crate::download::download(&config.cache, &request).await?);
|
||||
}
|
||||
for request in &config.crux {
|
||||
let mut request = request.clone();
|
||||
if request.month == "{previous_month}" {
|
||||
request.month = now
|
||||
.checked_sub_months(chrono::Months::new(1))
|
||||
.context("previous calendar month unavailable")?
|
||||
.format("%Y%m")
|
||||
.to_string();
|
||||
}
|
||||
inputs.push(crate::crux::download(&config.cache, &request).await?);
|
||||
}
|
||||
anyhow::ensure!(!inputs.is_empty(), "update config contains no sources");
|
||||
let mut db = crate::store::open(&config.database)?;
|
||||
for input in inputs {
|
||||
let manifest: SourceManifest = crate::read_json(&input.manifest)?;
|
||||
crate::store::import(&mut db, &manifest, &input.input)?;
|
||||
}
|
||||
// Build in a unique unpublished directory, then name the complete generation
|
||||
// by its receipt. Identical inputs/reviews reuse the same immutable artifact.
|
||||
let pending = config
|
||||
.generations
|
||||
.join(format!("pending-{}", now.format("%Y%m%dT%H%M%S%.9fZ")));
|
||||
crate::build::build(&db, &pending)?;
|
||||
let pin = crate::file_digest(&pending.join("COMPLETE.json"))?;
|
||||
let output = config.generations.join(format!("candidate-{pin}"));
|
||||
if output.exists() {
|
||||
crate::query::Registry::open(&output, &pin)?;
|
||||
// This directory was created by this invocation and contains only its
|
||||
// successfully validated duplicate build, never an operator generation.
|
||||
fs::remove_dir_all(&pending)?;
|
||||
} else {
|
||||
fs::rename(&pending, &output)?;
|
||||
std::fs::File::open(&config.generations)?.sync_all()?;
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
349
crates/argand-site-registry/tests/cli.rs
Normal file
349
crates/argand-site-registry/tests/cli.rs
Normal file
|
|
@ -0,0 +1,349 @@
|
|||
// By Nic Weyand!
|
||||
//! Fresh-process proof for all source imports and the signed release lifecycle.
|
||||
#[allow(dead_code)] // Shared fixture helpers also support the library contract suite.
|
||||
mod common;
|
||||
|
||||
use anyhow::{Context, ensure};
|
||||
use argand_site_registry::model::{Format, Source};
|
||||
use serde_json::{Value, json};
|
||||
use std::{fs, path::Path, process::Command};
|
||||
|
||||
fn run(args: &[&str]) -> anyhow::Result<Value> {
|
||||
let output = Command::new(env!("CARGO_BIN_EXE_argand-site-registry"))
|
||||
.args(args)
|
||||
.output()?;
|
||||
ensure!(
|
||||
output.status.success(),
|
||||
"CLI failed: {:?}\n{}",
|
||||
args,
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
Ok(serde_json::from_slice(&output.stdout)?)
|
||||
}
|
||||
|
||||
fn text(path: &Path) -> anyhow::Result<&str> {
|
||||
path.to_str().context("non-UTF8 fixture path")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Result<()> {
|
||||
let temporary = tempfile::tempdir()?;
|
||||
let configured = std::env::var_os("ARGAND_REGISTRY_E2E_OUTPUT").map(std::path::PathBuf::from);
|
||||
let root = configured.as_deref().unwrap_or(temporary.path());
|
||||
if configured.is_some() {
|
||||
fs::create_dir(root)?;
|
||||
}
|
||||
let database = root.join("store.sqlite");
|
||||
import_sources(root, &database)?;
|
||||
let candidate = root.join("candidate");
|
||||
let built = run(&[
|
||||
"build",
|
||||
"--database",
|
||||
text(&database)?,
|
||||
"--output",
|
||||
text(&candidate)?,
|
||||
])?;
|
||||
let pin = built["pin"].as_str().context("missing pin")?;
|
||||
let lookup = run(&[
|
||||
"lookup",
|
||||
"--generation",
|
||||
text(&candidate)?,
|
||||
"--pin",
|
||||
pin,
|
||||
"--query",
|
||||
"facebook",
|
||||
])?;
|
||||
assert_eq!(lookup["candidates"][0]["canonical_name"], "Facebook");
|
||||
assert_eq!(
|
||||
lookup["candidates"][0]["web_property"]["domain"]["registrable_domain"],
|
||||
"facebook.com"
|
||||
);
|
||||
assert!(
|
||||
run(&[
|
||||
"resolve",
|
||||
"--generation",
|
||||
text(&candidate)?,
|
||||
"--pin",
|
||||
pin,
|
||||
"--query",
|
||||
"facebook"
|
||||
])?["destination"]
|
||||
.is_null()
|
||||
);
|
||||
let now = chrono::Utc::now() - chrono::Duration::seconds(1);
|
||||
let decision = json!({"fingerprint":lookup["candidates"][0]["fingerprint"],"decision":"approve","reviewer":"synthetic fixture reviewer","reason":"E2E test only, not actual site verification","evidence":"synthetic:fixture","reviewed_at":now,"expires_at":now+chrono::Duration::days(1),"role":"primary","locale":"","country":""});
|
||||
let decision_path = root.join("review.json");
|
||||
fs::write(&decision_path, serde_json::to_vec(&decision)?)?;
|
||||
run(&[
|
||||
"review",
|
||||
"--database",
|
||||
text(&database)?,
|
||||
"--generation",
|
||||
text(&candidate)?,
|
||||
"--pin",
|
||||
pin,
|
||||
"--decision",
|
||||
text(&decision_path)?,
|
||||
])?;
|
||||
let approved = root.join("approved");
|
||||
review_identity(root, &database, &candidate, pin, &decision)?;
|
||||
let built = run(&[
|
||||
"build",
|
||||
"--database",
|
||||
text(&database)?,
|
||||
"--output",
|
||||
text(&approved)?,
|
||||
])?;
|
||||
let approved_pin = built["pin"].as_str().context("approved pin")?;
|
||||
assert_eq!(
|
||||
run(&[
|
||||
"resolve",
|
||||
"--generation",
|
||||
text(&approved)?,
|
||||
"--pin",
|
||||
approved_pin,
|
||||
"--query",
|
||||
"FB"
|
||||
])?["destination"]["url"],
|
||||
"https://facebook.com/"
|
||||
);
|
||||
let (revoked, revoked_pin) = release_lifecycle(
|
||||
root,
|
||||
&database,
|
||||
&approved,
|
||||
approved_pin,
|
||||
decision,
|
||||
&decision_path,
|
||||
)?;
|
||||
export_fixture(root, &revoked, &revoked_pin)?;
|
||||
println!("Native fixture lifecycle passed: {}", root.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn import_sources(root: &Path, database: &Path) -> anyhow::Result<()> {
|
||||
let sources = [
|
||||
(
|
||||
Source::Psl,
|
||||
Format::PslText,
|
||||
common::PSL.as_bytes().to_vec(),
|
||||
),
|
||||
(
|
||||
Source::Wikidata,
|
||||
Format::WikidataEntities,
|
||||
serde_json::to_vec(&common::wikidata())?,
|
||||
),
|
||||
(
|
||||
Source::Majestic,
|
||||
Format::MajesticCsv,
|
||||
common::MAJESTIC.as_bytes().to_vec(),
|
||||
),
|
||||
(
|
||||
Source::Crux,
|
||||
Format::CruxCsv,
|
||||
common::CRUX.as_bytes().to_vec(),
|
||||
),
|
||||
(Source::Curlie, Format::CurlieTarGz, common::curlie()?),
|
||||
];
|
||||
for (source, format, bytes) in sources {
|
||||
let input = root.join(format!("{}.input", source.key()));
|
||||
let manifest = root.join(format!("{}.json", source.key()));
|
||||
fs::write(&input, &bytes)?;
|
||||
fs::write(
|
||||
&manifest,
|
||||
serde_json::to_vec_pretty(&common::manifest(source, format, &bytes)?)?,
|
||||
)?;
|
||||
let args = [
|
||||
"import",
|
||||
"--database",
|
||||
text(database)?,
|
||||
"--input",
|
||||
text(&input)?,
|
||||
"--manifest",
|
||||
text(&manifest)?,
|
||||
];
|
||||
assert_eq!(run(&args)?, run(&args)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn release_lifecycle(
|
||||
root: &Path,
|
||||
database: &Path,
|
||||
approved: &Path,
|
||||
approved_pin: &str,
|
||||
mut decision: Value,
|
||||
decision_path: &Path,
|
||||
) -> anyhow::Result<(std::path::PathBuf, String)> {
|
||||
let key = root.join("signer");
|
||||
let status = Command::new("ssh-keygen")
|
||||
.args(["-q", "-t", "ed25519", "-N", "", "-f"])
|
||||
.arg(&key)
|
||||
.status()?;
|
||||
ensure!(status.success(), "generate test key");
|
||||
let allowed = root.join("allowed_signers");
|
||||
fs::write(
|
||||
&allowed,
|
||||
format!("fixture {}", fs::read_to_string(key.with_extension("pub"))?),
|
||||
)?;
|
||||
sign_and_activate(root, approved, approved_pin)?;
|
||||
decision["decision"] = json!("revoke");
|
||||
fs::write(decision_path, serde_json::to_vec(&decision)?)?;
|
||||
run(&[
|
||||
"review",
|
||||
"--database",
|
||||
text(database)?,
|
||||
"--generation",
|
||||
text(approved)?,
|
||||
"--pin",
|
||||
approved_pin,
|
||||
"--decision",
|
||||
text(decision_path)?,
|
||||
])?;
|
||||
let revoked = root.join("revoked");
|
||||
let built = run(&[
|
||||
"build",
|
||||
"--database",
|
||||
text(database)?,
|
||||
"--output",
|
||||
text(&revoked)?,
|
||||
])?;
|
||||
let revoked_pin = built["pin"].as_str().context("revoked pin")?;
|
||||
assert!(
|
||||
run(&[
|
||||
"resolve",
|
||||
"--generation",
|
||||
text(&revoked)?,
|
||||
"--pin",
|
||||
revoked_pin,
|
||||
"--query",
|
||||
"FB"
|
||||
])?["destination"]
|
||||
.is_null()
|
||||
);
|
||||
sign_and_activate(root, &revoked, revoked_pin)?;
|
||||
assert!(
|
||||
run(&[
|
||||
"activate",
|
||||
"--generation",
|
||||
text(approved)?,
|
||||
"--current",
|
||||
text(&root.join("current.json"))?,
|
||||
"--allowed-signers",
|
||||
text(&root.join("allowed_signers"))?,
|
||||
"--identity",
|
||||
"fixture"
|
||||
])
|
||||
.is_err()
|
||||
);
|
||||
Ok((revoked, revoked_pin.into()))
|
||||
}
|
||||
|
||||
fn sign_and_activate(root: &Path, approved: &Path, approved_pin: &str) -> anyhow::Result<()> {
|
||||
let key = root.join("signer");
|
||||
let allowed = root.join("allowed_signers");
|
||||
run(&[
|
||||
"sign",
|
||||
"--generation",
|
||||
text(approved)?,
|
||||
"--pin",
|
||||
approved_pin,
|
||||
"--key",
|
||||
text(&key)?,
|
||||
])?;
|
||||
let current = root.join("current.json");
|
||||
assert!(
|
||||
run(&[
|
||||
"activate",
|
||||
"--generation",
|
||||
text(approved)?,
|
||||
"--current",
|
||||
text(¤t)?,
|
||||
"--allowed-signers",
|
||||
text(&allowed)?,
|
||||
"--identity",
|
||||
"untrusted"
|
||||
])
|
||||
.is_err()
|
||||
);
|
||||
run(&[
|
||||
"activate",
|
||||
"--generation",
|
||||
text(approved)?,
|
||||
"--current",
|
||||
text(¤t)?,
|
||||
"--allowed-signers",
|
||||
text(&allowed)?,
|
||||
"--identity",
|
||||
"fixture",
|
||||
])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn review_identity(
|
||||
root: &Path,
|
||||
database: &Path,
|
||||
generation: &Path,
|
||||
pin: &str,
|
||||
template: &Value,
|
||||
) -> anyhow::Result<()> {
|
||||
let wiki = run(&[
|
||||
"lookup",
|
||||
"--generation",
|
||||
text(generation)?,
|
||||
"--pin",
|
||||
pin,
|
||||
"--query",
|
||||
"FB",
|
||||
])?;
|
||||
let curlie = run(&[
|
||||
"lookup",
|
||||
"--generation",
|
||||
text(generation)?,
|
||||
"--pin",
|
||||
pin,
|
||||
"--query",
|
||||
"Facebook directory listing",
|
||||
])?;
|
||||
let left = wiki["candidates"][0]["entity_id"]
|
||||
.as_str()
|
||||
.context("Wiki identity")?;
|
||||
let right = curlie["candidates"][0]["entity_id"]
|
||||
.as_str()
|
||||
.context("Curlie identity")?;
|
||||
let args = [
|
||||
"equivalence",
|
||||
"--generation",
|
||||
text(generation)?,
|
||||
"--pin",
|
||||
pin,
|
||||
"--left",
|
||||
left,
|
||||
"--right",
|
||||
right,
|
||||
];
|
||||
let preview = run(&args)?;
|
||||
let mut decision = template.clone();
|
||||
decision["fingerprint"] = preview["fingerprint"].clone();
|
||||
decision["role"] = json!("unspecified");
|
||||
let path = root.join("identity-review.json");
|
||||
fs::write(&path, serde_json::to_vec(&decision)?)?;
|
||||
let mut args = args.to_vec();
|
||||
args.extend(["--database", text(database)?, "--decision", text(&path)?]);
|
||||
run(&args)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn export_fixture(root: &Path, revoked: &Path, revoked_pin: &str) -> anyhow::Result<()> {
|
||||
let export = root.join("registry.jsonl");
|
||||
run(&[
|
||||
"export",
|
||||
"--generation",
|
||||
text(revoked)?,
|
||||
"--pin",
|
||||
revoked_pin,
|
||||
"--output",
|
||||
text(&export)?,
|
||||
])?;
|
||||
assert!(fs::read_to_string(export)?.contains("CC-BY-4.0"));
|
||||
Ok(())
|
||||
}
|
||||
179
crates/argand-site-registry/tests/common/mod.rs
Normal file
179
crates/argand-site-registry/tests/common/mod.rs
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
// By Nic Weyand!
|
||||
//! Synthetic source-shaped fixtures; these are not real ownership evidence.
|
||||
|
||||
use anyhow::Context;
|
||||
use argand_site_registry::{
|
||||
model::{Compression, Format, Source, SourceManifest},
|
||||
query::Registry,
|
||||
store,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::{Value, json};
|
||||
use std::{fs, io::Write, path::Path};
|
||||
|
||||
pub const PSL: &str = "// This fixture is authored for Argand tests.\n// ===BEGIN ICANN DOMAINS===\ncom\norg\nuk\nco.uk\nde\nbe\nfr\njp\n*.kawasaki.jp\n!city.kawasaki.jp\n// ===END ICANN DOMAINS===\n// ===BEGIN PRIVATE DOMAINS===\nblogspot.com\n// ===END PRIVATE DOMAINS===\n";
|
||||
pub const MAJESTIC: &str = "GlobalRank,TldRank,Domain,TLD,RefSubNets,RefIPs,IDN_Domain,IDN_TLD,PrevGlobalRank,PrevTldRank,PrevRefSubNets,PrevRefIPs\n2,2,facebook.com,com,100,200,facebook.com,com,2,2,99,199\n3,3,atlas.example.co.uk,uk,90,180,atlas.example.co.uk,uk,3,3,90,180\n";
|
||||
pub const CRUX: &str = "origin,rank,yyyymm,country_code\nhttps://facebook.com,1000,202608,US\nhttps://atlas.example.co.uk,100000,202608,GB\n";
|
||||
|
||||
pub fn timestamp() -> anyhow::Result<DateTime<Utc>> {
|
||||
Ok(DateTime::parse_from_rfc3339("2026-09-12T12:00:00Z")?.with_timezone(&Utc))
|
||||
}
|
||||
|
||||
pub fn entity(id: &str, name: &str, aliases: &[&str], urls: &[&str]) -> Value {
|
||||
let websites:Vec<Value>=urls.iter().enumerate().map(|(i,url)|json!({"id":format!("{id}${i}"),"rank":"normal","mainsnak":{"property":"P856","snaktype":"value","datavalue":{"type":"string","value":url}},"references":[{"hash":"synthetic-reference","snaks":{}}]})).collect();
|
||||
json!({"id":id,"type":"item","lastrevid":1,"labels":{"en":{"language":"en","value":name}},"aliases":{"en":aliases.iter().map(|text|json!({"language":"en","value":text})).collect::<Vec<_>>()},"claims":{"P856":websites}})
|
||||
}
|
||||
|
||||
pub fn wikidata() -> Value {
|
||||
let mut atlas = entity(
|
||||
"Q900001",
|
||||
"Atlas Fixture",
|
||||
&["Atlas", "Cafe\u{301} Atlas"],
|
||||
&[
|
||||
"https://atlas.example.com/",
|
||||
"https://atlas.example.co.uk/",
|
||||
"https://atlas.example.de/",
|
||||
],
|
||||
);
|
||||
atlas["claims"]["P856"][1]["qualifiers"] = json!({"P1001":[{"snaktype":"value","property":"P1001","datavalue":{"value":{"id":"Q145"}}}],"P407":[{"snaktype":"value","property":"P407","datavalue":{"value":{"id":"Q1860"}}}]});
|
||||
json!({"entities":{"Q355":entity("Q355","Facebook",&["FB"],&["https://facebook.com/"]),"Q900001":atlas}})
|
||||
}
|
||||
|
||||
pub fn curlie() -> anyhow::Result<Vec<u8>> {
|
||||
let gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
let mut archive = tar::Builder::new(gzip);
|
||||
for (name, bytes) in [
|
||||
(
|
||||
"curlie-rdf/rdf-Top-c.tsv",
|
||||
"https://facebook.com/\tFacebook directory listing\tSynthetic editorial description\t42\n",
|
||||
),
|
||||
(
|
||||
"curlie-rdf/rdf-Top-s.tsv",
|
||||
"42\tComputers/Internet\t1\tSynthetic category description\t\t\n",
|
||||
),
|
||||
] {
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(u64::try_from(bytes.len())?);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
archive.append_data(&mut header, name, bytes.as_bytes())?;
|
||||
}
|
||||
Ok(archive.into_inner()?.finish()?)
|
||||
}
|
||||
|
||||
pub fn manifest(source: Source, format: Format, bytes: &[u8]) -> anyhow::Result<SourceManifest> {
|
||||
let source_url = match source {
|
||||
Source::Psl => "https://publicsuffix.org/list/public_suffix_list.dat",
|
||||
Source::Wikidata => "https://www.wikidata.org/wiki/Special:EntityData/Q355.json",
|
||||
Source::Majestic => "https://downloads.majestic.com/majestic_million.csv",
|
||||
Source::Crux => "https://developer.chrome.com/docs/crux/bigquery/",
|
||||
Source::Curlie => "https://curlie.org/directory-dl",
|
||||
};
|
||||
Ok(SourceManifest {
|
||||
schema: "argand.site-source/v1".into(),
|
||||
source,
|
||||
format,
|
||||
compression: Compression::None,
|
||||
snapshot: "synthetic-fixture-v1".into(),
|
||||
scope: "fixture".into(),
|
||||
source_url: source_url.into(),
|
||||
license: source.license().into(),
|
||||
license_url: source.license_url().into(),
|
||||
retrieved_at: timestamp()?,
|
||||
sha256: argand_site_registry::digest(bytes),
|
||||
bytes: u64::try_from(bytes.len())?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn import(
|
||||
db: &mut rusqlite::Connection,
|
||||
root: &Path,
|
||||
source: Source,
|
||||
format: Format,
|
||||
bytes: &[u8],
|
||||
) -> anyhow::Result<SourceManifest> {
|
||||
let manifest = manifest(source, format, bytes)?;
|
||||
let input = root.join(format!("{}.input", manifest.sha256));
|
||||
fs::write(&input, bytes)?;
|
||||
store::import(db, &manifest, &input)?;
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
pub fn fixture(root: &Path) -> anyhow::Result<rusqlite::Connection> {
|
||||
let mut db = store::open(&root.join("store.sqlite"))?;
|
||||
import(&mut db, root, Source::Psl, Format::PslText, PSL.as_bytes())?;
|
||||
import(
|
||||
&mut db,
|
||||
root,
|
||||
Source::Wikidata,
|
||||
Format::WikidataEntities,
|
||||
&serde_json::to_vec(&wikidata())?,
|
||||
)?;
|
||||
import(
|
||||
&mut db,
|
||||
root,
|
||||
Source::Majestic,
|
||||
Format::MajesticCsv,
|
||||
MAJESTIC.as_bytes(),
|
||||
)?;
|
||||
import(
|
||||
&mut db,
|
||||
root,
|
||||
Source::Crux,
|
||||
Format::CruxCsv,
|
||||
CRUX.as_bytes(),
|
||||
)?;
|
||||
import(
|
||||
&mut db,
|
||||
root,
|
||||
Source::Curlie,
|
||||
Format::CurlieTarGz,
|
||||
&curlie()?,
|
||||
)?;
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
pub fn build(db: &rusqlite::Connection, root: &Path, name: &str) -> anyhow::Result<Registry> {
|
||||
let generation = root.join(name);
|
||||
argand_site_registry::build::build(db, &generation)?;
|
||||
Registry::open(
|
||||
&generation,
|
||||
&argand_site_registry::file_digest(&generation.join("COMPLETE.json"))?,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn approve(
|
||||
db: &rusqlite::Connection,
|
||||
registry: &Registry,
|
||||
query: &str,
|
||||
url: &str,
|
||||
role: &str,
|
||||
country: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let candidate = registry
|
||||
.lookup(query, 100)?
|
||||
.candidates
|
||||
.into_iter()
|
||||
.find(|c| c.url == url)
|
||||
.context("fixture destination missing")?;
|
||||
let review = argand_site_registry::review::Review {
|
||||
fingerprint: candidate.fingerprint,
|
||||
decision: "approve".into(),
|
||||
reviewer: "synthetic fixture reviewer".into(),
|
||||
reason: "Test only; not a real-world ownership assertion".into(),
|
||||
evidence: "synthetic:fixture-observation".into(),
|
||||
reviewed_at: timestamp()?,
|
||||
expires_at: timestamp()? + chrono::Duration::days(7),
|
||||
role: role.into(),
|
||||
locale: String::new(),
|
||||
country: country.into(),
|
||||
};
|
||||
argand_site_registry::review::record(db, registry, &review)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn gzip(bytes: &[u8]) -> anyhow::Result<Vec<u8>> {
|
||||
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
encoder.write_all(bytes)?;
|
||||
Ok(encoder.finish()?)
|
||||
}
|
||||
272
crates/argand-site-registry/tests/failures.rs
Normal file
272
crates/argand-site-registry/tests/failures.rs
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
// By Nic Weyand!
|
||||
//! Source poisoning, repeatability and metadata regression cases.
|
||||
#[allow(dead_code)] // Same source-shaped helpers as the native lifecycle suite.
|
||||
mod common;
|
||||
use argand_site_registry::{
|
||||
download::CachedSource,
|
||||
model::{Compression, Format, Source},
|
||||
query::Registry,
|
||||
store,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::{fs, io::Write};
|
||||
|
||||
#[test]
|
||||
fn removed_entity_websites_retire_the_previous_selection() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = common::fixture(root.path())?;
|
||||
let before = common::build(&db, root.path(), "before-retirement")?;
|
||||
common::approve(&db, &before, "FB", "https://facebook.com/", "primary", "")?;
|
||||
let bytes = br#"{"entities":{"Q355":{"id":"Q355","missing":""}}}"#;
|
||||
let mut source = common::manifest(Source::Wikidata, Format::WikidataEntities, bytes)?;
|
||||
source.retrieved_at += chrono::Duration::days(1);
|
||||
let input = root.path().join("retirement.json");
|
||||
fs::write(&input, bytes)?;
|
||||
store::import(&mut db, &source, &input)?;
|
||||
let retired = common::build(&db, root.path(), "retired")?;
|
||||
assert_eq!(retired.lookup("FB", 1)?.total_entities, 0);
|
||||
assert!(
|
||||
retired
|
||||
.resolve("FB", None, None, common::timestamp()?)?
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(before.lookup("FB", 1)?.total_entities, 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_order_does_not_change_generation_and_psl_refresh_preserves_review() -> anyhow::Result<()>
|
||||
{
|
||||
let root = tempfile::tempdir()?;
|
||||
let a = common::fixture(root.path())?;
|
||||
let mut b = store::open(&root.path().join("other.sqlite"))?;
|
||||
for (source, format, bytes) in [
|
||||
(Source::Curlie, Format::CurlieTarGz, common::curlie()?),
|
||||
(
|
||||
Source::Crux,
|
||||
Format::CruxCsv,
|
||||
common::CRUX.as_bytes().to_vec(),
|
||||
),
|
||||
(
|
||||
Source::Majestic,
|
||||
Format::MajesticCsv,
|
||||
common::MAJESTIC.as_bytes().to_vec(),
|
||||
),
|
||||
(
|
||||
Source::Wikidata,
|
||||
Format::WikidataEntities,
|
||||
serde_json::to_vec(&common::wikidata())?,
|
||||
),
|
||||
(
|
||||
Source::Psl,
|
||||
Format::PslText,
|
||||
common::PSL.as_bytes().to_vec(),
|
||||
),
|
||||
] {
|
||||
common::import(&mut b, root.path(), source, format, &bytes)?;
|
||||
}
|
||||
let first = common::build(&a, root.path(), "a")?;
|
||||
let second = common::build(&b, root.path(), "b")?;
|
||||
assert_eq!(first.identity, second.identity);
|
||||
common::approve(&b, &second, "FB", "https://facebook.com/", "primary", "")?;
|
||||
let input = root.path().join("psl");
|
||||
fs::write(&input, common::PSL)?;
|
||||
let mut manifest = common::manifest(Source::Psl, Format::PslText, common::PSL.as_bytes())?;
|
||||
manifest.retrieved_at += chrono::Duration::days(1);
|
||||
store::import(&mut b, &manifest, &input)?;
|
||||
let refreshed = common::build(&b, root.path(), "refresh")?;
|
||||
assert!(
|
||||
refreshed
|
||||
.resolve("FB", None, None, common::timestamp()?)?
|
||||
.is_some()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_archives_and_incomplete_refresh_never_replace_sources() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = common::fixture(root.path())?;
|
||||
let mut archive = tar::Builder::new(flate2::write::GzEncoder::new(
|
||||
Vec::new(),
|
||||
flate2::Compression::default(),
|
||||
));
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_entry_type(tar::EntryType::Symlink);
|
||||
header.set_size(0);
|
||||
header.set_mode(0o777);
|
||||
archive.append_link(&mut header, "curlie-rdf/evil-c.tsv", "/etc/passwd")?;
|
||||
let link = archive.into_inner()?.finish()?;
|
||||
let mut truncated = common::curlie()?;
|
||||
truncated.truncate(truncated.len() - 5);
|
||||
for bytes in [&link, &truncated] {
|
||||
assert!(
|
||||
common::import(
|
||||
&mut db,
|
||||
root.path(),
|
||||
Source::Curlie,
|
||||
Format::CurlieTarGz,
|
||||
bytes
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let registry = common::build(&db, root.path(), "complete")?;
|
||||
assert_eq!(registry.receipt.sources.len(), 5);
|
||||
assert_eq!(
|
||||
registry
|
||||
.lookup("Facebook directory listing", 1)?
|
||||
.total_entities,
|
||||
1
|
||||
);
|
||||
fs::write(root.path().join("complete/ATTRIBUTION.json"), b"{}")?;
|
||||
assert!(Registry::open(&root.path().join("complete"), ®istry.identity).is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn temporal_deprecated_and_nonvalue_assertions_keep_evidence_without_admission()
|
||||
-> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = store::open(&root.path().join("data.sqlite"))?;
|
||||
common::import(
|
||||
&mut db,
|
||||
root.path(),
|
||||
Source::Psl,
|
||||
Format::PslText,
|
||||
common::PSL.as_bytes(),
|
||||
)?;
|
||||
let mut entity = common::entity(
|
||||
"Q100",
|
||||
"Historical",
|
||||
&[],
|
||||
&[
|
||||
"https://old.example.com",
|
||||
"https://ancient.example.com",
|
||||
"https://unknown.example.com",
|
||||
],
|
||||
);
|
||||
entity["claims"]["P856"][0]["qualifiers"] =
|
||||
json!({"P582":[{"datavalue":{"value":{"time":"+2001-01-01T00:00:00Z"}}}]});
|
||||
entity["claims"]["P856"][1]["rank"] = json!("deprecated");
|
||||
entity["claims"]["P856"][2]["mainsnak"] = json!({"property":"P856","snaktype":"novalue"});
|
||||
let bytes = serde_json::to_vec(&json!({"entities":{"Q100":entity}}))?;
|
||||
common::import(
|
||||
&mut db,
|
||||
root.path(),
|
||||
Source::Wikidata,
|
||||
Format::WikidataEntities,
|
||||
&bytes,
|
||||
)?;
|
||||
let registry = common::build(&db, root.path(), "generation")?;
|
||||
assert_eq!(registry.receipt.rejected, 1);
|
||||
assert!(
|
||||
registry
|
||||
.lookup("Historical", 100)?
|
||||
.candidates
|
||||
.iter()
|
||||
.all(|c| !c.eligible)
|
||||
);
|
||||
assert!(
|
||||
common::approve(
|
||||
&db,
|
||||
®istry,
|
||||
"Historical",
|
||||
"https://old.example.com/",
|
||||
"primary",
|
||||
""
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bzip2_multistream_and_property_scope_metadata() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = store::open(&root.path().join("data.sqlite"))?;
|
||||
let data = serde_json::to_vec(&common::wikidata())?;
|
||||
let mut bytes = Vec::new();
|
||||
for half in data.chunks(data.len().div_ceil(2)) {
|
||||
let mut bz = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::fast());
|
||||
bz.write_all(half)?;
|
||||
bytes.extend(bz.finish()?);
|
||||
}
|
||||
let input = root.path().join("entities.bz2");
|
||||
fs::write(&input, &bytes)?;
|
||||
let mut manifest = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?;
|
||||
manifest.compression = Compression::Bzip2;
|
||||
store::import(&mut db, &manifest, &input)?;
|
||||
common::import(
|
||||
&mut db,
|
||||
root.path(),
|
||||
Source::Psl,
|
||||
Format::PslText,
|
||||
common::PSL.as_bytes(),
|
||||
)?;
|
||||
let registry = common::build(&db, root.path(), "generation")?;
|
||||
let candidates = registry.lookup("Atlas", 10)?.candidates;
|
||||
let regional = candidates
|
||||
.iter()
|
||||
.find(|c| c.url.contains("co.uk"))
|
||||
.ok_or_else(|| anyhow::anyhow!("regional missing"))?;
|
||||
assert_eq!(regional.property_scopes[0].jurisdiction_entities, ["Q145"]);
|
||||
assert_eq!(regional.property_scopes[0].language_entities, ["Q1860"]);
|
||||
assert_eq!(regional.property_scopes[0].country, None);
|
||||
assert!(
|
||||
regional
|
||||
.entity
|
||||
.names
|
||||
.iter()
|
||||
.any(|n| n["value"]["text"] == "Atlas" && n["source"]["license"] == "CC0-1.0")
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_update_reuses_generation_and_failure_preserves_it() -> anyhow::Result<()> {
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut inputs = Vec::new();
|
||||
for (source, format, bytes) in [
|
||||
(
|
||||
Source::Psl,
|
||||
Format::PslText,
|
||||
common::PSL.as_bytes().to_vec(),
|
||||
),
|
||||
(
|
||||
Source::Wikidata,
|
||||
Format::WikidataEntities,
|
||||
serde_json::to_vec(&common::wikidata())?,
|
||||
),
|
||||
] {
|
||||
let input = root.path().join(source.key());
|
||||
let manifest = input.with_extension("json");
|
||||
fs::write(&input, &bytes)?;
|
||||
fs::write(
|
||||
&manifest,
|
||||
serde_json::to_vec(&common::manifest(source, format, &bytes)?)?,
|
||||
)?;
|
||||
inputs.push(CachedSource { input, manifest });
|
||||
}
|
||||
let config = argand_site_registry::update::Config {
|
||||
cache: root.path().join("cache"),
|
||||
database: root.path().join("data.sqlite"),
|
||||
generations: root.path().join("generations"),
|
||||
downloads: vec![],
|
||||
inputs,
|
||||
crux: vec![],
|
||||
};
|
||||
let first = argand_site_registry::update::run(&config).await?;
|
||||
let second = argand_site_registry::update::run(&config).await?;
|
||||
assert_eq!(first, second);
|
||||
let pin = argand_site_registry::file_digest(&first.join("COMPLETE.json"))?;
|
||||
fs::write(&config.inputs[1].input, "truncated")?;
|
||||
assert!(argand_site_registry::update::run(&config).await.is_err());
|
||||
assert_eq!(
|
||||
Registry::open(&first, &pin)?
|
||||
.lookup("FB", 1)?
|
||||
.total_entities,
|
||||
1
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
173
crates/argand-site-registry/tests/identity.rs
Normal file
173
crates/argand-site-registry/tests/identity.rs
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
// By Nic Weyand!
|
||||
//! Cross-source collisions need an exact, revocable identity decision.
|
||||
#[allow(dead_code)]
|
||||
mod common;
|
||||
use argand_site_registry::{
|
||||
identity,
|
||||
model::{Format, Source},
|
||||
review::{self, Review},
|
||||
store,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
fn decision(fingerprint: String) -> anyhow::Result<Review> {
|
||||
Ok(Review {
|
||||
fingerprint,
|
||||
decision: "approve".into(),
|
||||
reviewer: "synthetic reviewer".into(),
|
||||
reason: "Synthetic identity test, not actual ownership".into(),
|
||||
evidence: "synthetic:identity".into(),
|
||||
reviewed_at: common::timestamp()?,
|
||||
expires_at: common::timestamp()? + chrono::Duration::days(2),
|
||||
role: "unspecified".into(),
|
||||
locale: String::new(),
|
||||
country: String::new(),
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_equivalence_resolves_cross_source_aliases_and_expires_or_revokes() -> anyhow::Result<()>
|
||||
{
|
||||
let root = tempfile::tempdir()?;
|
||||
let db = common::fixture(root.path())?;
|
||||
let baseline = common::build(&db, root.path(), "baseline")?;
|
||||
common::approve(&db, &baseline, "FB", "https://facebook.com/", "primary", "")?;
|
||||
let wiki = &baseline.lookup("FB", 1)?.candidates[0].entity_id;
|
||||
let curlie = &baseline.lookup("Facebook directory listing", 1)?.candidates[0].entity_id;
|
||||
let pair = identity::propose(&baseline, wiki, curlie)?;
|
||||
let mut review = decision(pair.fingerprint.clone())?;
|
||||
identity::record(&db, &baseline, &pair, &review)?;
|
||||
let linked = common::build(&db, root.path(), "linked")?;
|
||||
let resolved = linked
|
||||
.resolve(
|
||||
"Facebook directory listing",
|
||||
None,
|
||||
None,
|
||||
common::timestamp()?,
|
||||
)?
|
||||
.ok_or_else(|| anyhow::anyhow!("approved alias not linked"))?;
|
||||
assert_eq!(resolved.entity_id, *wiki);
|
||||
assert_eq!(resolved.identity_provenance.len(), 1);
|
||||
assert!(
|
||||
linked
|
||||
.resolve("Facebook directory listing", None, None, review.expires_at)?
|
||||
.is_none()
|
||||
);
|
||||
review.decision = "revoke".into();
|
||||
identity::record(&db, &linked, &pair, &review)?;
|
||||
let revoked = common::build(&db, root.path(), "revoked")?;
|
||||
assert!(
|
||||
revoked
|
||||
.resolve(
|
||||
"Facebook directory listing",
|
||||
None,
|
||||
None,
|
||||
common::timestamp()?
|
||||
)?
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
revoked
|
||||
.resolve("FB", None, None, common::timestamp()?)?
|
||||
.is_some()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_collision_remains_ambiguous_until_review_and_changes_invalidate_link() -> anyhow::Result<()>
|
||||
{
|
||||
let root = tempfile::tempdir()?;
|
||||
let mut db = common::fixture(root.path())?;
|
||||
let mut entity = common::entity(
|
||||
"Q355",
|
||||
"Facebook",
|
||||
&["FB", "Facebook directory listing"],
|
||||
&["https://facebook.com/"],
|
||||
);
|
||||
let bytes = serde_json::to_vec(&json!({"entities":{"Q355":entity}}))?;
|
||||
let mut source = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?;
|
||||
source.retrieved_at += chrono::Duration::hours(1);
|
||||
let input = root.path().join("changed");
|
||||
std::fs::write(&input, &bytes)?;
|
||||
store::import(&mut db, &source, &input)?;
|
||||
let baseline = common::build(&db, root.path(), "baseline")?;
|
||||
common::approve(&db, &baseline, "FB", "https://facebook.com/", "primary", "")?;
|
||||
let approved = common::build(&db, root.path(), "approved")?;
|
||||
assert_eq!(
|
||||
approved
|
||||
.lookup("Facebook directory listing", 1)?
|
||||
.total_entities,
|
||||
2
|
||||
);
|
||||
assert!(
|
||||
approved
|
||||
.resolve(
|
||||
"Facebook directory listing",
|
||||
None,
|
||||
None,
|
||||
common::timestamp()?
|
||||
)?
|
||||
.is_none()
|
||||
);
|
||||
let candidates = baseline
|
||||
.lookup("Facebook directory listing", 10)?
|
||||
.candidates;
|
||||
let pair = identity::propose(
|
||||
&baseline,
|
||||
&candidates[0].entity_id,
|
||||
&candidates[1].entity_id,
|
||||
)?;
|
||||
identity::record(&db, &baseline, &pair, &decision(pair.fingerprint.clone())?)?;
|
||||
let linked = common::build(&db, root.path(), "linked")?;
|
||||
assert!(
|
||||
linked
|
||||
.resolve(
|
||||
"Facebook directory listing",
|
||||
None,
|
||||
None,
|
||||
common::timestamp()?
|
||||
)?
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(
|
||||
linked
|
||||
.lookup("Facebook directory listing", 1)?
|
||||
.total_entities,
|
||||
2
|
||||
);
|
||||
// Even renewing a destination review does not silently renew identity evidence.
|
||||
entity["aliases"]["en"]
|
||||
.as_array_mut()
|
||||
.ok_or_else(|| anyhow::anyhow!("aliases"))?
|
||||
.push(json!({"language":"en","value":"new unreviewed alias"}));
|
||||
let bytes = serde_json::to_vec(&json!({"entities":{"Q355":entity}}))?;
|
||||
source.sha256 = argand_site_registry::digest(&bytes);
|
||||
source.bytes = u64::try_from(bytes.len())?;
|
||||
source.retrieved_at += chrono::Duration::hours(1);
|
||||
std::fs::write(&input, &bytes)?;
|
||||
store::import(&mut db, &source, &input)?;
|
||||
let changed = common::build(&db, root.path(), "changed-generation")?;
|
||||
common::approve(&db, &changed, "FB", "https://facebook.com/", "primary", "")?;
|
||||
let renewed = common::build(&db, root.path(), "renewed-destination")?;
|
||||
assert!(
|
||||
renewed
|
||||
.resolve(
|
||||
"Facebook directory listing",
|
||||
None,
|
||||
None,
|
||||
common::timestamp()?
|
||||
)?
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
renewed
|
||||
.resolve("FB", None, None, common::timestamp()?)?
|
||||
.is_some()
|
||||
);
|
||||
let mut tampered = decision(pair.fingerprint.clone())?;
|
||||
tampered.role = "primary".into();
|
||||
assert!(identity::record(&db, &baseline, &pair, &tampered).is_err());
|
||||
assert!(review::record(&db, &baseline, &tampered).is_err());
|
||||
Ok(())
|
||||
}
|
||||
322
crates/argand-site-registry/tests/registry.rs
Normal file
322
crates/argand-site-registry/tests/registry.rs
Normal file
|
|
@ -0,0 +1,322 @@
|
|||
// By Nic Weyand!
|
||||
//! Registry contracts exercised through real SQLite stores and immutable releases.
|
||||
mod common;
|
||||
use anyhow::{Context, ensure};
|
||||
use argand_site_registry::{
|
||||
model::{Compression, Format, Source},
|
||||
normalize::{Normalizer, name_key},
|
||||
query::Registry,
|
||||
review::{self, Review},
|
||||
store,
|
||||
};
|
||||
use common::{approve, build, fixture, import, timestamp};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn normalization_psl_and_identity() -> anyhow::Result<()> {
|
||||
let n = Normalizer::new(common::PSL.as_bytes(), "fixture-psl".into())?;
|
||||
let p = n.url("https://WWW.Example.co.uk:443/a?b=2&a=1#fragment")?;
|
||||
assert_eq!(p.url, "https://www.example.co.uk/a?b=2&a=1");
|
||||
assert_eq!(p.domain.registrable_domain, "example.co.uk");
|
||||
assert_eq!(p.domain.public_suffix, "co.uk");
|
||||
assert_eq!(
|
||||
n.domain("foo.blogspot.com")?.registrable_domain,
|
||||
"foo.blogspot.com"
|
||||
);
|
||||
assert!(n.domain("foo.blogspot.com")?.private_suffix);
|
||||
assert_eq!(
|
||||
n.domain("a.city.kawasaki.jp")?.registrable_domain,
|
||||
"city.kawasaki.jp"
|
||||
);
|
||||
assert_eq!(n.domain("a.b.kawasaki.jp")?.public_suffix, "b.kawasaki.jp");
|
||||
assert_eq!(n.domain("BÜCHER.de.")?.hostname, "xn--bcher-kva.de");
|
||||
for url in [
|
||||
"javascript:alert(1)",
|
||||
"https://user:pass@example.com/",
|
||||
"https://127.1/",
|
||||
"https://[::1]/",
|
||||
"https://foo.local/",
|
||||
"https://co.uk/",
|
||||
"https://example.com\\@evil.com/",
|
||||
"https://example.com/\n",
|
||||
"https:/example.com/",
|
||||
"https://example.com../",
|
||||
] {
|
||||
assert!(n.url(url).is_err(), "accepted {url}");
|
||||
}
|
||||
assert_ne!(
|
||||
n.url("http://example.com/")?.id,
|
||||
n.url("https://example.com/")?.id
|
||||
);
|
||||
assert_ne!(
|
||||
n.url("https://example.com/")?.id,
|
||||
n.url("https://www.example.com/")?.id
|
||||
);
|
||||
assert_eq!(name_key(" Cafe\u{301} ATLAS ")?, "café atlas");
|
||||
assert!(name_key("Face\u{202e}book").is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliases_deduplication_provenance_and_separate_popularity() -> anyhow::Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let mut db = fixture(dir.path())?;
|
||||
let before: i64 = db.query_row("SELECT count(*) FROM facts", [], |r| r.get(0))?;
|
||||
import(
|
||||
&mut db,
|
||||
dir.path(),
|
||||
Source::Wikidata,
|
||||
Format::WikidataEntities,
|
||||
&serde_json::to_vec(&common::wikidata())?,
|
||||
)?;
|
||||
let after: i64 = db.query_row("SELECT count(*) FROM facts", [], |r| r.get(0))?;
|
||||
assert_eq!(before, after);
|
||||
let mut selectors = db.prepare("SELECT r.raw_json,f.selector FROM facts f JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal JOIN sources s ON s.id=f.source_id WHERE s.source='wikidata' AND f.predicate='name'")?;
|
||||
for item in selectors.query_map([], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||
})? {
|
||||
let (raw, selector) = item?;
|
||||
assert!(
|
||||
serde_json::from_str::<serde_json::Value>(&raw)?
|
||||
.pointer(&selector)
|
||||
.is_some(),
|
||||
"invalid source selector {selector}"
|
||||
);
|
||||
}
|
||||
let r = build(&db, dir.path(), "generation")?;
|
||||
assert_eq!(r.lookup("FB", 10)?.candidates[0].canonical_name, "Facebook");
|
||||
assert_eq!(r.lookup("Café Atlas", 10)?.total_edges, 3);
|
||||
let facebook = r.lookup("Facebook", 10)?;
|
||||
assert_eq!(facebook.total_entities, 1);
|
||||
let c = &facebook.candidates[0];
|
||||
assert_eq!(c.provenance[0]["source"]["license"], "CC0-1.0");
|
||||
assert!(c.evidence["assertion"]["statement"]["references"].is_array());
|
||||
assert_eq!(r.receipt.properties, 4); // Curlie Facebook shares the same property.
|
||||
assert_eq!(r.receipt.entities, 3); // Curlie is not automatically the Wikidata entity.
|
||||
assert!(r.resolve("Facebook", None, None, timestamp()?)?.is_none());
|
||||
let export = dir.path().join("export.jsonl");
|
||||
argand_site_registry::release::export(&r, &export, false)?;
|
||||
let text = std::fs::read_to_string(&export)?;
|
||||
assert!(!text.contains("Synthetic editorial description"));
|
||||
assert!(!text.contains("Synthetic category description"));
|
||||
assert!(text.contains("With content from Curlie.org"));
|
||||
assert!(text.contains("referring_subnets") && text.contains("coarse_rank"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regional_review_expiry_and_revocation() -> anyhow::Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let db = fixture(dir.path())?;
|
||||
let candidate = build(&db, dir.path(), "candidate")?;
|
||||
approve(
|
||||
&db,
|
||||
&candidate,
|
||||
"Atlas",
|
||||
"https://atlas.example.com/",
|
||||
"primary",
|
||||
"",
|
||||
)?;
|
||||
approve(
|
||||
&db,
|
||||
&candidate,
|
||||
"Atlas",
|
||||
"https://atlas.example.co.uk/",
|
||||
"regional",
|
||||
"GB",
|
||||
)?;
|
||||
let approved = build(&db, dir.path(), "approved")?;
|
||||
assert_eq!(
|
||||
approved
|
||||
.resolve("Atlas", None, Some("GB"), timestamp()?)?
|
||||
.context("regional missing")?
|
||||
.url,
|
||||
"https://atlas.example.co.uk/"
|
||||
);
|
||||
assert_eq!(
|
||||
approved
|
||||
.resolve("Atlas", None, Some("DE"), timestamp()?)?
|
||||
.context("primary missing")?
|
||||
.url,
|
||||
"https://atlas.example.com/"
|
||||
);
|
||||
assert!(
|
||||
approved
|
||||
.resolve(
|
||||
"Atlas",
|
||||
None,
|
||||
None,
|
||||
timestamp()? + chrono::Duration::days(8)
|
||||
)?
|
||||
.is_none()
|
||||
);
|
||||
let fingerprint = approved
|
||||
.lookup("Atlas", 10)?
|
||||
.candidates
|
||||
.into_iter()
|
||||
.find(|c| c.url == "https://atlas.example.co.uk/")
|
||||
.context("missing GB")?
|
||||
.fingerprint;
|
||||
review::record(
|
||||
&db,
|
||||
&approved,
|
||||
&Review {
|
||||
fingerprint,
|
||||
decision: "revoke".into(),
|
||||
reviewer: "test".into(),
|
||||
reason: "test revocation".into(),
|
||||
evidence: "synthetic:revocation".into(),
|
||||
reviewed_at: timestamp()?,
|
||||
expires_at: timestamp()?,
|
||||
role: "unspecified".into(),
|
||||
locale: String::new(),
|
||||
country: String::new(),
|
||||
},
|
||||
)?;
|
||||
let revoked = build(&db, dir.path(), "revoked")?;
|
||||
assert_eq!(
|
||||
revoked
|
||||
.resolve("Atlas", None, Some("GB"), timestamp()?)?
|
||||
.context("fallback missing")?
|
||||
.url,
|
||||
"https://atlas.example.com/"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguity_survives_limits_and_same_named_domains_do_not_merge() -> anyhow::Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let mut db = fixture(dir.path())?;
|
||||
let raw = json!({"entities":{"Q900002":common::entity("Q900002","Atlas Fixture",&["Atlas"],&["https://atlas.example.fr/"])}});
|
||||
let bytes = serde_json::to_vec(&raw)?;
|
||||
let mut m = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?;
|
||||
m.scope = "additional".into();
|
||||
let path = dir.path().join("conflict.json");
|
||||
std::fs::write(&path, bytes)?;
|
||||
store::import(&mut db, &m, &path)?;
|
||||
let r = build(&db, dir.path(), "conflicts")?;
|
||||
let result = r.lookup("Atlas", 1)?;
|
||||
assert_eq!(result.total_entities, 2);
|
||||
assert_eq!(result.total_edges, 4);
|
||||
assert!(result.truncated);
|
||||
assert!(r.resolve("Atlas", None, None, timestamp()?)?.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_names_and_urls_invalidate_approval_but_history_survives() -> anyhow::Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let mut db = fixture(dir.path())?;
|
||||
let initial = build(&db, dir.path(), "initial")?;
|
||||
approve(
|
||||
&db,
|
||||
&initial,
|
||||
"Facebook",
|
||||
"https://facebook.com/",
|
||||
"primary",
|
||||
"",
|
||||
)?;
|
||||
let mut raw = common::wikidata();
|
||||
raw["entities"]["Q355"]["aliases"]["en"]
|
||||
.as_array_mut()
|
||||
.context("aliases")?
|
||||
.push(json!({"language":"en","value":"New alias"}));
|
||||
let bytes = serde_json::to_vec(&raw)?;
|
||||
let mut m = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?;
|
||||
m.snapshot = "v2".into();
|
||||
m.retrieved_at += chrono::Duration::days(1);
|
||||
let path = dir.path().join("v2.json");
|
||||
std::fs::write(&path, bytes)?;
|
||||
store::import(&mut db, &m, &path)?;
|
||||
let r = build(&db, dir.path(), "changed")?;
|
||||
assert!(
|
||||
r.resolve(
|
||||
"Facebook",
|
||||
None,
|
||||
None,
|
||||
timestamp()? + chrono::Duration::days(1)
|
||||
)?
|
||||
.is_none()
|
||||
);
|
||||
let sources: i64 = db.query_row(
|
||||
"SELECT count(*) FROM sources WHERE source='wikidata' AND complete=1",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
assert_eq!(sources, 2);
|
||||
assert_eq!(
|
||||
initial.lookup("Facebook", 10)?.candidates[0].entity_id,
|
||||
r.lookup("Facebook", 10)?.candidates[0].entity_id
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_dumps_resume_and_corruption_fail_closed() -> anyhow::Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let mut db = store::open(&dir.path().join("store"))?;
|
||||
let mut dump = String::from("[\n");
|
||||
for i in 0..300 {
|
||||
if i > 0 {
|
||||
dump.push_str(",\n");
|
||||
}
|
||||
dump.push_str(&serde_json::to_string(&common::entity(
|
||||
&format!("Q{}", 900_000 + i),
|
||||
&format!("Fixture {i}"),
|
||||
&[],
|
||||
&["https://example.com/"],
|
||||
))?);
|
||||
}
|
||||
dump.push_str("\n]\n");
|
||||
let bytes = common::gzip(dump.as_bytes())?;
|
||||
let mut m = common::manifest(Source::Wikidata, Format::WikidataDump, &bytes)?;
|
||||
m.compression = Compression::Gzip;
|
||||
let path = dir.path().join("dump.gz");
|
||||
std::fs::write(&path, &bytes)?;
|
||||
// Interrupt the sink after its first committed batch; rerun the exact input.
|
||||
db.execute_batch("CREATE TRIGGER simulated_crash BEFORE INSERT ON records WHEN NEW.ordinal=270 BEGIN SELECT RAISE(ABORT,'simulated interruption'); END;")?;
|
||||
assert!(store::import(&mut db, &m, &path).is_err());
|
||||
let checkpoint: i64 = db.query_row("SELECT checkpoint FROM sources", [], |r| r.get(0))?;
|
||||
assert_eq!(checkpoint, 256);
|
||||
db.execute_batch("DROP TRIGGER simulated_crash")?;
|
||||
store::import(&mut db, &m, &path)?;
|
||||
let records: i64 = db.query_row("SELECT count(*) FROM records", [], |r| r.get(0))?;
|
||||
assert_eq!(records, 300);
|
||||
let mut damaged = bytes.clone();
|
||||
damaged.pop();
|
||||
std::fs::write(&path, &damaged)?;
|
||||
assert!(store::import(&mut db, &m, &path).is_err());
|
||||
let mut truncated = common::manifest(Source::Wikidata, Format::WikidataDump, &damaged)?;
|
||||
truncated.compression = Compression::Gzip;
|
||||
assert!(store::import(&mut db, &truncated, &path).is_err());
|
||||
let complete: i64 = db.query_row(
|
||||
"SELECT complete FROM sources WHERE id=?1",
|
||||
[truncated.id()?],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
assert_eq!(complete, 0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_builds_have_identical_bytes_and_corruption_is_rejected() -> anyhow::Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let db = fixture(dir.path())?;
|
||||
let first = build(&db, dir.path(), "first")?;
|
||||
let second = build(&db, dir.path(), "second")?;
|
||||
assert_eq!(first.identity, second.identity);
|
||||
assert_eq!(
|
||||
first.receipt.database_sha256,
|
||||
second.receipt.database_sha256
|
||||
);
|
||||
assert!(argand_site_registry::build::build(&db, &dir.path().join("first")).is_err());
|
||||
let path = dir.path().join("second/registry.sqlite");
|
||||
std::fs::write(path, b"corrupted")?;
|
||||
assert!(Registry::open(&dir.path().join("second"), &second.identity).is_err());
|
||||
ensure!(
|
||||
first.lookup("Facebook", 1)?.total_entities == 1,
|
||||
"old generation damaged"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue