argand-site-registry/crates/argand-site-registry/tests/registry.rs

// By Nic Weyand!
//! Registry contracts exercised through real SQLite stores and immutable releases.
mod common;
use anyhow::{Context, ensure};
use argand_site_registry::{
    model::{Compression, Format, Source},
    normalize::{Normalizer, name_key},
    query::{Registry, ResolutionStatus},
    review::{self, Review},
    store,
};
use common::{approve, build, fixture, import, timestamp};
use serde_json::json;

#[test]
fn normalization_psl_and_identity() -> anyhow::Result<()> {
    let n = Normalizer::new(common::PSL.as_bytes(), "fixture-psl".into())?;
    let p = n.url("https://WWW.Example.co.uk:443/a?b=2&a=1#fragment")?;
    assert_eq!(p.url, "https://www.example.co.uk/a?b=2&a=1");
    assert_eq!(p.domain.registrable_domain, "example.co.uk");
    assert_eq!(p.domain.public_suffix, "co.uk");
    assert_eq!(
        n.domain("foo.blogspot.com")?.registrable_domain,
        "foo.blogspot.com"
    );
    assert!(n.domain("foo.blogspot.com")?.private_suffix);
    assert_eq!(
        n.domain("a.city.kawasaki.jp")?.registrable_domain,
        "city.kawasaki.jp"
    );
    assert_eq!(n.domain("a.b.kawasaki.jp")?.public_suffix, "b.kawasaki.jp");
    assert_eq!(n.domain("BÜCHER.de.")?.hostname, "xn--bcher-kva.de");
    for url in [
        "javascript:alert(1)",
        "https://user:pass@example.com/",
        "https://127.1/",
        "https://[::1]/",
        "https://foo.local/",
        "https://co.uk/",
        "https://example.com\\@evil.com/",
        "https://example.com/\n",
        "https:/example.com/",
        "https://example.com../",
    ] {
        assert!(n.url(url).is_err(), "accepted {url}");
    }
    assert_ne!(
        n.url("http://example.com/")?.id,
        n.url("https://example.com/")?.id
    );
    assert_ne!(
        n.url("https://example.com/")?.id,
        n.url("https://www.example.com/")?.id
    );
    assert_eq!(name_key(" Cafe\u{301} ATLAS ")?, "café atlas");
    assert!(name_key("Face\u{202e}book").is_err());
    Ok(())
}

#[test]
fn aliases_deduplication_provenance_and_separate_popularity() -> anyhow::Result<()> {
    let dir = tempfile::tempdir()?;
    let mut db = fixture(dir.path())?;
    let before: i64 = db.query_row("SELECT count(*) FROM facts", [], |r| r.get(0))?;
    import(
        &mut db,
        dir.path(),
        Source::Wikidata,
        Format::WikidataEntities,
        &serde_json::to_vec(&common::wikidata())?,
    )?;
    let after: i64 = db.query_row("SELECT count(*) FROM facts", [], |r| r.get(0))?;
    assert_eq!(before, after);
    let mut selectors = db.prepare("SELECT r.raw_json,f.selector FROM facts f JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal JOIN sources s ON s.id=f.source_id WHERE s.source='wikidata' AND f.predicate='name'")?;
    for item in selectors.query_map([], |row| {
        Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
    })? {
        let (raw, selector) = item?;
        assert!(
            serde_json::from_str::<serde_json::Value>(&raw)?
                .pointer(&selector)
                .is_some(),
            "invalid source selector {selector}"
        );
    }
    let r = build(&db, dir.path(), "generation")?;
    assert_eq!(r.lookup("FB", 10)?.candidates[0].canonical_name, "Facebook");
    assert_eq!(r.lookup("Café Atlas", 10)?.total_edges, 3);
    let facebook = r.lookup("Facebook", 10)?;
    assert_eq!(facebook.total_entities, 1);
    let c = &facebook.candidates[0];
    assert_eq!(c.provenance[0]["source"]["license"], "CC0-1.0");
    assert!(c.evidence["assertion"]["statement"]["references"].is_array());
    assert_eq!(r.receipt.properties, 4); // Curlie Facebook shares the same property.
    assert_eq!(r.receipt.entities, 3); // Curlie is not automatically the Wikidata entity.
    assert!(r.resolve("Facebook", None, None, timestamp()?)?.is_none());
    let unresolved = r.resolve_explained("Facebook", None, None, timestamp()?)?;
    assert_eq!(unresolved.status, ResolutionStatus::NoActiveReview);
    assert_eq!(unresolved.counts.missing_review, 1);
    let reverse = r.lookup_web("facebook.com", 10)?;
    assert_eq!(reverse.total_properties, 1);
    assert_eq!(reverse.total_edges, 2);
    assert!(
        reverse
            .matches
            .iter()
            .any(|item| item.candidate.canonical_name == "Facebook")
    );
    let popularity = r.popularity("https://facebook.com/", 10)?;
    assert_eq!(popularity.total, 2);
    assert_eq!(
        popularity
            .observations
            .iter()
            .map(|item| item.source.as_str())
            .collect::<std::collections::BTreeSet<_>>(),
        std::collections::BTreeSet::from(["crux", "majestic"])
    );
    let entity = r
        .entity_by_id(&c.entity_id, 10)?
        .context("entity lookup missing")?;
    assert_eq!(entity.entity.canonical_name, "Facebook");
    let category = r.category("42", 10)?;
    assert_eq!(category.total_members, 1);
    assert!(!serde_json::to_string(&category)?.contains("Synthetic category description"));
    let export = dir.path().join("export.jsonl");
    argand_site_registry::release::export(&r, &export, false)?;
    let text = std::fs::read_to_string(&export)?;
    assert!(!text.contains("Synthetic editorial description"));
    assert!(!text.contains("Synthetic category description"));
    assert!(text.contains("With content from Curlie.org"));
    assert!(text.contains("referring_subnets") && text.contains("coarse_rank"));
    Ok(())
}

#[test]
fn regional_review_expiry_and_revocation() -> anyhow::Result<()> {
    let dir = tempfile::tempdir()?;
    let db = fixture(dir.path())?;
    let candidate = build(&db, dir.path(), "candidate")?;
    approve(
        &db,
        &candidate,
        "Atlas",
        "https://atlas.example.com/",
        "primary",
        "",
    )?;
    approve(
        &db,
        &candidate,
        "Atlas",
        "https://atlas.example.co.uk/",
        "regional",
        "GB",
    )?;
    let approved = build(&db, dir.path(), "approved")?;
    assert_eq!(
        approved
            .resolve("Atlas", None, Some("GB"), timestamp()?)?
            .context("regional missing")?
            .url,
        "https://atlas.example.co.uk/"
    );
    assert_eq!(
        approved
            .resolve("Atlas", None, Some("DE"), timestamp()?)?
            .context("primary missing")?
            .url,
        "https://atlas.example.com/"
    );
    assert!(
        approved
            .resolve(
                "Atlas",
                None,
                None,
                timestamp()? + chrono::Duration::days(8)
            )?
            .is_none()
    );
    assert_eq!(
        approved
            .resolve_explained(
                "Atlas",
                None,
                None,
                timestamp()? + chrono::Duration::days(8)
            )?
            .status,
        ResolutionStatus::NoActiveReview
    );
    let fingerprint = approved
        .lookup("Atlas", 10)?
        .candidates
        .into_iter()
        .find(|c| c.url == "https://atlas.example.co.uk/")
        .context("missing GB")?
        .fingerprint;
    review::record(
        &db,
        &approved,
        &Review {
            fingerprint,
            decision: "revoke".into(),
            reviewer: "test".into(),
            reason: "test revocation".into(),
            evidence: "synthetic:revocation".into(),
            reviewed_at: timestamp()?,
            expires_at: timestamp()?,
            role: "unspecified".into(),
            locale: String::new(),
            country: String::new(),
        },
    )?;
    let revoked = build(&db, dir.path(), "revoked")?;
    assert_eq!(
        revoked
            .resolve("Atlas", None, Some("GB"), timestamp()?)?
            .context("fallback missing")?
            .url,
        "https://atlas.example.com/"
    );
    Ok(())
}

#[test]
fn ambiguity_survives_limits_and_same_named_domains_do_not_merge() -> anyhow::Result<()> {
    let dir = tempfile::tempdir()?;
    let mut db = fixture(dir.path())?;
    let raw = json!({"entities":{"Q900002":common::entity("Q900002","Atlas Fixture",&["Atlas"],&["https://atlas.example.fr/"])}});
    let bytes = serde_json::to_vec(&raw)?;
    let mut m = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?;
    m.scope = "additional".into();
    let path = dir.path().join("conflict.json");
    std::fs::write(&path, bytes)?;
    store::import(&mut db, &m, &path)?;
    let r = build(&db, dir.path(), "conflicts")?;
    let result = r.lookup("Atlas", 1)?;
    assert_eq!(result.total_entities, 2);
    assert_eq!(result.total_edges, 4);
    assert!(result.truncated);
    assert!(r.resolve("Atlas", None, None, timestamp()?)?.is_none());
    assert_eq!(
        r.resolve_explained("Atlas", None, None, timestamp()?)?
            .status,
        ResolutionStatus::AmbiguousIdentity
    );
    assert_eq!(
        r.resolve_explained("absent", None, None, timestamp()?)?
            .status,
        ResolutionStatus::NoNameMatch
    );
    Ok(())
}

#[test]
fn changed_names_and_urls_invalidate_approval_but_history_survives() -> anyhow::Result<()> {
    let dir = tempfile::tempdir()?;
    let mut db = fixture(dir.path())?;
    let initial = build(&db, dir.path(), "initial")?;
    approve(
        &db,
        &initial,
        "Facebook",
        "https://facebook.com/",
        "primary",
        "",
    )?;
    let mut raw = common::wikidata();
    raw["entities"]["Q355"]["aliases"]["en"]
        .as_array_mut()
        .context("aliases")?
        .push(json!({"language":"en","value":"New alias"}));
    let bytes = serde_json::to_vec(&raw)?;
    let mut m = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?;
    m.snapshot = "v2".into();
    m.retrieved_at += chrono::Duration::days(1);
    let path = dir.path().join("v2.json");
    std::fs::write(&path, bytes)?;
    store::import(&mut db, &m, &path)?;
    let r = build(&db, dir.path(), "changed")?;
    assert!(
        r.resolve(
            "Facebook",
            None,
            None,
            timestamp()? + chrono::Duration::days(1)
        )?
        .is_none()
    );
    let sources: i64 = db.query_row(
        "SELECT count(*) FROM sources WHERE source='wikidata' AND complete=1",
        [],
        |r| r.get(0),
    )?;
    assert_eq!(sources, 2);
    assert_eq!(
        initial.lookup("Facebook", 10)?.candidates[0].entity_id,
        r.lookup("Facebook", 10)?.candidates[0].entity_id
    );
    Ok(())
}

#[test]
fn compressed_dumps_resume_and_corruption_fail_closed() -> anyhow::Result<()> {
    let dir = tempfile::tempdir()?;
    let mut db = store::open(&dir.path().join("store"))?;
    let mut dump = String::from("[\n");
    for i in 0..300 {
        if i > 0 {
            dump.push_str(",\n");
        }
        dump.push_str(&serde_json::to_string(&common::entity(
            &format!("Q{}", 900_000 + i),
            &format!("Fixture {i}"),
            &[],
            &["https://example.com/"],
        ))?);
    }
    dump.push_str("\n]\n");
    let bytes = common::gzip(dump.as_bytes())?;
    let mut m = common::manifest(Source::Wikidata, Format::WikidataDump, &bytes)?;
    m.compression = Compression::Gzip;
    let path = dir.path().join("dump.gz");
    std::fs::write(&path, &bytes)?;
    // A handled sink error removes committed rows whose complete stream digest
    // was not established. A process crash still leaves the durable checkpoint.
    db.execute_batch("CREATE TRIGGER simulated_crash BEFORE INSERT ON records WHEN NEW.ordinal=270 BEGIN SELECT RAISE(ABORT,'simulated interruption'); END;")?;
    assert!(store::import(&mut db, &m, &path).is_err());
    let partial: i64 = db.query_row("SELECT count(*) FROM sources", [], |r| r.get(0))?;
    assert_eq!(partial, 0);
    db.execute_batch("DROP TRIGGER simulated_crash")?;
    store::import(&mut db, &m, &path)?;
    let records: i64 = db.query_row("SELECT count(*) FROM records", [], |r| r.get(0))?;
    assert_eq!(records, 300);
    let mut damaged = bytes.clone();
    damaged.pop();
    std::fs::write(&path, &damaged)?;
    assert!(store::import(&mut db, &m, &path).is_err());
    let mut truncated = common::manifest(Source::Wikidata, Format::WikidataDump, &damaged)?;
    truncated.compression = Compression::Gzip;
    assert!(store::import(&mut db, &truncated, &path).is_err());
    let retained: i64 = db.query_row(
        "SELECT count(*) FROM sources WHERE id=?1",
        [truncated.id()?],
        |r| r.get(0),
    )?;
    assert_eq!(retained, 0);
    Ok(())
}

#[test]
fn repeated_builds_have_identical_bytes_and_corruption_is_rejected() -> anyhow::Result<()> {
    let dir = tempfile::tempdir()?;
    let db = fixture(dir.path())?;
    let first = build(&db, dir.path(), "first")?;
    let second = build(&db, dir.path(), "second")?;
    assert_eq!(first.identity, second.identity);
    assert_eq!(
        first.receipt.database_sha256,
        second.receipt.database_sha256
    );
    assert!(argand_site_registry::build::build(&db, &dir.path().join("first")).is_err());
    let path = dir.path().join("second/registry.sqlite");
    std::fs::write(path, b"corrupted")?;
    assert!(Registry::open(&dir.path().join("second"), &second.identity).is_err());
    ensure!(
        first.lookup("Facebook", 1)?.total_entities == 1,
        "old generation damaged"
    );
    Ok(())
}