argand-site-registry/crates/argand-site-registry/tests/v05.rs

// By Nic Weyand!
//! v0.5 source-lineage and ROR schema acceptance proof.
#[allow(dead_code)]
mod common;

use anyhow::{Context, ensure};
use argand_site_registry::{
    build, evaluation,
    model::{CoverageKind, Format, IntegrityProof, Source, SourceCoverage, SourceLineage},
    policy::ReviewPolicy,
    query::Registry,
    store,
};
use serde_json::{Value, json};
use std::{
    io::{Cursor, Write},
    process::{Command, Stdio},
    thread,
    time::{Duration, Instant},
};

fn record(id: &str, name: &str, status: &str, website: &str) -> Value {
    json!({
        "admin":{"created":{"date":"2020-01-01","schema_version":"1.0"},"last_modified":{"date":"2026-08-01","schema_version":"2.1"}},
        "domains":[url::Url::parse(website).ok().and_then(|url| url.host_str().map(str::to_owned)).unwrap_or_default()],
        "established":2020,
        "external_ids":[{"all":["Q42"],"preferred":"Q42","type":"wikidata"}],
        "id":format!("https://ror.org/{id}"),
        "links":[{"type":"website","value":website},{"type":"wikipedia","value":"https://en.wikipedia.org/wiki/Example"}],
        "locations":[{"geonames_id":2_643_743,"geonames_details":{"continent_code":"EU","continent_name":"Europe","country_code":"GB","country_name":"United Kingdom","country_subdivision_code":"ENG","country_subdivision_name":"England","lat":51.5,"lng":-0.1,"name":"London"}}],
        "names":[{"lang":"en","types":["ror_display","label"],"value":name},{"lang":null,"types":["acronym"],"value":"EU"}],
        "relationships":[],
        "status":status,
        "types":["education"]
    })
}

fn ror_zip(records: &[Value], extra: Option<(&str, &[u8])>) -> anyhow::Result<Vec<u8>> {
    let mut archive = zip::ZipWriter::new(Cursor::new(Vec::new()));
    let options = zip::write::SimpleFileOptions::default()
        .compression_method(zip::CompressionMethod::Deflated);
    archive.start_file("v2.12-2026-08-25-ror-data.json", options)?;
    archive.write_all(&serde_json::to_vec_pretty(records)?)?;
    archive.start_file("v2.12-2026-08-25-ror-data.csv", options)?;
    archive.write_all(b"id,name\n")?;
    if let Some((name, bytes)) = extra {
        archive.start_file(name, options)?;
        archive.write_all(bytes)?;
    }
    Ok(archive.finish()?.into_inner())
}

#[test]
#[allow(clippy::too_many_lines)] // One source scenario covers every reviewed ROR mapping and hold.
fn ror_schema_21_is_separate_ineligible_when_inactive_and_lineage_is_exported() -> anyhow::Result<()>
{
    let root = tempfile::tempdir()?;
    let mut db = store::open(&root.path().join("writer.sqlite"))?;
    common::import(
        &mut db,
        root.path(),
        Source::Psl,
        Format::PslText,
        common::PSL.as_bytes(),
    )?;
    let wikidata = json!({"entities":{"Q42":common::entity(
        "Q42",
        "Example University Wikidata",
        &[],
        &["https://wikidata-example.org/"]
    )}});
    common::import(
        &mut db,
        root.path(),
        Source::Wikidata,
        Format::WikidataEntities,
        &serde_json::to_vec(&wikidata)?,
    )?;
    let mut sparse = record(
        "fedcba210",
        "Conflicting Organization",
        "active",
        "https://example.org/",
    );
    sparse["domains"] = json!([]);
    sparse["established"] = Value::Null;
    sparse["external_ids"] = json!([]);
    sparse["locations"] = json!([]);
    sparse["names"] =
        json!([{"lang":"en","types":["ror_display"],"value":"Conflicting Organization"}]);
    sparse["links"] = json!([
        {"type":"website","value":"https://example.org/"},
        {"type":"website","value":"https://regional.example.org/"}
    ]);
    let bytes = ror_zip(
        &[
            record(
                "012345678",
                "Example University",
                "active",
                "https://example.org/",
            ),
            record(
                "abcdef012",
                "Former Institute",
                "inactive",
                "https://former.example.org/",
            ),
            sparse,
        ],
        None,
    )?;
    let mut manifest = common::manifest(Source::Ror, Format::RorZip, &bytes)?;
    manifest.schema = "argand.site-source/v3".into();
    manifest.integrity = vec![IntegrityProof {
        method: "content_digest".into(),
        algorithm: "sha256".into(),
        value: manifest.sha256.clone(),
        evidence_url: None,
    }];
    manifest.lineage = Some(SourceLineage::direct(Source::Ror));
    let input = root.path().join("ror.zip");
    std::fs::write(&input, &bytes)?;
    let first = store::import(&mut db, &manifest, &input)?;
    assert_eq!(first, store::import(&mut db, &manifest, &input)?);
    let registry = common::build(&db, root.path(), "generation")?;
    let active = registry.lookup("Example University", 20)?;
    assert_eq!(active.total_entities, 1);
    assert_eq!(active.candidates[0].relation, "asserted_official");
    assert!(active.candidates[0].eligible);
    assert_eq!(
        active.candidates[0].provenance[0]["source"]["lineage"]["upstream_datasets"][0],
        "geonames"
    );
    assert!(!registry.lookup("Former Institute", 20)?.candidates[0].eligible);
    assert_eq!(registry.lookup("EU", 20)?.total_entities, 2);
    assert_eq!(
        registry
            .lookup("Conflicting Organization", 20)?
            .candidates
            .len(),
        2
    );
    let ror_id = argand_site_registry::model::entity_id(Source::Ror, "https://ror.org/012345678");
    let wikidata_id = argand_site_registry::model::entity_id(Source::Wikidata, "Q42");
    let equivalence = argand_site_registry::identity::propose(&registry, &ror_id, &wikidata_id)?;
    assert_eq!(equivalence.entities.len(), 2);
    let audit = root.path().join("audit.jsonl");
    argand_site_registry::release::export_audit(&registry, &audit, false)?;
    let audit = std::fs::read_to_string(audit)?;
    assert!(audit.contains("declared_domain"));
    assert!(audit.contains("geonames"));
    assert!(audit.contains("wikidata"));
    let report = evaluation::run(
        &registry,
        Cursor::new(
            r#"{"id":"ror","query":"Example University","expected_status":"no_active_name_review"}"#,
        ),
        1,
        common::timestamp()?,
    )?;
    let lineage = report
        .source_lineage
        .iter()
        .find(|source| source.source == Source::Ror)
        .context("ROR evaluation lineage missing")?;
    assert_eq!(
        lineage
            .lineage
            .as_ref()
            .context("ROR lineage missing")?
            .upstream_datasets,
        vec!["geonames".to_owned()]
    );
    Ok(())
}

#[test]
fn ror_zip_and_schema_drift_fail_before_completion() -> anyhow::Result<()> {
    let root = tempfile::tempdir()?;
    let mut db = store::open(&root.path().join("writer.sqlite"))?;
    let mut changed = record(
        "012345678",
        "Changed Schema",
        "active",
        "https://example.org/",
    );
    changed
        .as_object_mut()
        .context("record object")?
        .insert("unreviewed_field".into(), json!(true));
    for bytes in [
        ror_zip(&[changed], None)?,
        ror_zip(
            &[record(
                "012345678",
                "Extra Member",
                "active",
                "https://example.org/",
            )],
            Some(("unexpected.txt", b"bad")),
        )?,
    ] {
        let manifest = common::manifest(Source::Ror, Format::RorZip, &bytes)?;
        let input = root.path().join(format!("{}.zip", manifest.sha256));
        std::fs::write(&input, bytes)?;
        assert!(store::import(&mut db, &manifest, &input).is_err());
        let incomplete: bool = db.query_row(
            "SELECT EXISTS(SELECT 1 FROM sources WHERE id=?1)",
            [manifest.id()?],
            |row| row.get(0),
        )?;
        ensure!(!incomplete, "failed ROR source remained in writer store");
    }
    Ok(())
}

#[test]
#[allow(clippy::too_many_lines)] // One lifecycle proves package, compact, verify, export, sign, and tamper behavior.
fn compact_generation_keeps_runtime_results_and_authenticates_cold_history() -> anyhow::Result<()> {
    let root = tempfile::tempdir()?;
    let mut db = common::fixture(root.path())?;
    let before = common::build(&db, root.path(), "before-revision")?;

    // A later legacy snapshot in the same source scope leaves the earlier source
    // available for audit while replacing it in the active projection.
    let mut replacement = json!({"entities":{"Q355":common::entity(
        "Q355",
        "Facebook",
        &["FB"],
        &["https://facebook.com/"]
    )}});
    replacement["entities"]["Q355"]["lastrevid"] = json!(2);
    let bytes = serde_json::to_vec(&replacement)?;
    let mut manifest = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?;
    manifest.retrieved_at += chrono::Duration::days(1);
    let input = root.path().join("wikidata-replacement.json");
    std::fs::write(&input, &bytes)?;
    store::import(&mut db, &manifest, &input)?;

    let full = common::build(&db, root.path(), "full")?;
    assert_eq!(
        before.lookup("Facebook", 20)?.candidates[0].fingerprint,
        full.lookup("Facebook", 20)?.candidates[0].fingerprint
    );
    let mut revision_diff = Vec::new();
    argand_site_registry::release::diff(&before, &full, &mut revision_diff)?;
    let revision_diff = String::from_utf8(revision_diff)?;
    assert!(revision_diff.contains("revision"));
    assert!(revision_diff.contains("\"subject\":\"fact\""));
    let audit_store = root.path().join("audit-store");
    let compact_path = root.path().join("compact");
    let compact_receipt = build::build_compact_with_policy_and_trust(
        &db,
        &compact_path,
        &ReviewPolicy::legacy_compatible(),
        None,
        &audit_store,
    )?;
    let compact = Registry::open(
        &compact_path,
        &argand_site_registry::file_digest(&compact_path.join("COMPLETE.json"))?,
    )?;

    assert_eq!(compact_receipt.runtime_layout, "compact-v1");
    assert_eq!(
        serde_json::to_value(full.lookup("Facebook", 20)?.candidates)?,
        serde_json::to_value(compact.lookup("Facebook", 20)?.candidates)?
    );
    let compact_queue =
        argand_site_registry::queue::review_queue(&compact, common::timestamp()?, 100, 1_000)?;
    assert!(!compact_queue.items.is_empty());
    assert!(
        std::fs::metadata(compact_path.join("registry.sqlite"))?.len()
            < std::fs::metadata(root.path().join("full/registry.sqlite"))?.len()
    );
    let verified = argand_site_registry::audit::verify(&compact, &audit_store)?;
    assert_eq!(
        usize::try_from(verified.bundles)?,
        compact_receipt.audit_bundles.len()
    );
    assert!(
        compact_receipt
            .audit_bundles
            .iter()
            .any(|item| !item.selected)
    );
    let checkpoint_path = root.path().join("retention.json");
    let checkpoint = argand_site_registry::audit::checkpoint(
        &compact,
        &audit_store,
        common::timestamp()?,
        &checkpoint_path,
    )?;
    assert!(!checkpoint.deletion_authorized);
    assert_eq!(checkpoint.retain, compact_receipt.audit_bundles);
    let key = root.path().join("retention-key");
    ensure!(
        Command::new("ssh-keygen")
            .args(["-q", "-t", "ed25519", "-N", "", "-f"])
            .arg(&key)
            .status()?
            .success(),
        "generate retention test key"
    );
    let allowed = root.path().join("retention-allowed-signers");
    std::fs::write(
        &allowed,
        format!(
            "retention {}",
            std::fs::read_to_string(key.with_extension("pub"))?
        ),
    )?;
    let signature = root.path().join("retention.json.sig");
    argand_site_registry::audit::sign_retention(&checkpoint_path, &signature, &key)?;
    assert_eq!(
        argand_site_registry::audit::verify_retention(
            &checkpoint_path,
            &signature,
            &allowed,
            "retention"
        )?,
        checkpoint
    );

    let export = root.path().join("cold-history.jsonl");
    argand_site_registry::audit::export(&compact, &audit_store, &export, false)?;
    let export = std::fs::read_to_string(export)?;
    assert!(export.contains("superseded"));
    assert!(export.contains("\"revision\":2"));
    assert!(
        argand_site_registry::release::export_audit(
            &compact,
            &root.path().join("must-not-exist.jsonl"),
            false
        )
        .is_err()
    );

    // A second build reuses the exact immutable bundle objects.
    let second_path = root.path().join("compact-second");
    let second = build::build_compact_with_policy_and_trust(
        &db,
        &second_path,
        &ReviewPolicy::legacy_compatible(),
        None,
        &audit_store,
    )?;
    assert_eq!(compact_receipt.audit_bundles, second.audit_bundles);

    let receipt_path = compact_path.join("COMPLETE.json");
    let original_receipt = std::fs::read(&receipt_path)?;
    let mut omitted: build::Receipt = serde_json::from_slice(&original_receipt)?;
    omitted
        .audit_bundles
        .retain(|reference| !reference.selected);
    std::fs::write(&receipt_path, serde_json::to_vec_pretty(&omitted)?)?;
    let omitted_pin = argand_site_registry::file_digest(&receipt_path)?;
    assert!(Registry::open(&compact_path, &omitted_pin).is_err());
    std::fs::write(&receipt_path, original_receipt)?;

    let reference = &compact_receipt.audit_bundles[0];
    let object = argand_site_registry::audit::path(&audit_store, reference);
    let original = std::fs::read(&object)?;
    std::fs::write(&object, &original[..original.len() - 1])?;
    assert!(argand_site_registry::audit::verify(&compact, &audit_store).is_err());
    std::fs::write(&object, &original)?;
    assert!(argand_site_registry::audit::verify(&compact, &audit_store).is_ok());
    std::fs::remove_file(&object)?;
    assert!(argand_site_registry::audit::verify(&compact, &audit_store).is_err());
    Ok(())
}

#[test]
fn wikidata_dump_streams_multiline_entities_with_an_authenticated_large_record_bound()
-> anyhow::Result<()> {
    let root = tempfile::tempdir()?;
    let mut db = store::open(&root.path().join("writer.sqlite"))?;
    let mut entity = common::entity("Q355", "Facebook", &["FB"], &["https://facebook.com/"]);
    entity["unconsumed_large_field"] = Value::String("x".repeat(17 * 1024 * 1024));
    let bytes = serde_json::to_vec_pretty(&vec![entity])?;
    let input = root.path().join("wikidata-large.json");
    std::fs::write(&input, &bytes)?;

    let mut legacy = common::manifest(Source::Wikidata, Format::WikidataDump, &bytes)?;
    assert!(store::import(&mut db, &legacy, &input).is_err());
    legacy.schema = "argand.site-source/v3".into();
    legacy.maximum_record_bytes = Some(32 * 1024 * 1024);
    legacy.integrity = vec![IntegrityProof {
        method: "content_digest".into(),
        algorithm: "sha256".into(),
        value: legacy.sha256.clone(),
        evidence_url: None,
    }];
    legacy.lineage = Some(SourceLineage::direct(Source::Wikidata));
    let source_id = store::import(&mut db, &legacy, &input)?;
    let records: i64 = db.query_row(
        "SELECT count(*) FROM records WHERE source_id=?1",
        [source_id],
        |row| row.get(0),
    )?;
    assert_eq!(records, 1);
    Ok(())
}

#[test]
fn wikidata_json_delta_turns_removed_websites_into_auditable_tombstones() -> anyhow::Result<()> {
    let root = tempfile::tempdir()?;
    let mut db = store::open(&root.path().join("writer.sqlite"))?;
    common::import(
        &mut db,
        root.path(),
        Source::Psl,
        Format::PslText,
        common::PSL.as_bytes(),
    )?;
    let base_bytes = serde_json::to_vec(&vec![common::entity(
        "Q355",
        "Facebook",
        &[],
        &["https://facebook.com/"],
    )])?;
    let mut base = common::manifest(Source::Wikidata, Format::WikidataDump, &base_bytes)?;
    base.schema = "argand.site-source/v3".into();
    base.coverage = Some(SourceCoverage {
        collection: "wikidata-entities".into(),
        kind: CoverageKind::Full,
        partition: None,
        base: None,
        sequence: None,
        supersedes: Vec::new(),
    });
    base.integrity = vec![IntegrityProof {
        method: "content_digest".into(),
        algorithm: "sha256".into(),
        value: base.sha256.clone(),
        evidence_url: None,
    }];
    base.lineage = Some(SourceLineage::direct(Source::Wikidata));
    let base_input = root.path().join("wikidata-base.json");
    std::fs::write(&base_input, &base_bytes)?;
    let base_id = store::import(&mut db, &base, &base_input)?;

    let removed =
        json!({"id":"Q355","type":"item","lastrevid":2,"labels":{},"aliases":{},"claims":{}});
    let delta_bytes = serde_json::to_vec(&vec![removed])?;
    let mut delta = common::manifest(Source::Wikidata, Format::WikidataDump, &delta_bytes)?;
    delta.schema = "argand.site-source/v3".into();
    delta.retrieved_at += chrono::Duration::days(1);
    delta.coverage = Some(SourceCoverage {
        collection: "wikidata-entities".into(),
        kind: CoverageKind::Delta,
        partition: None,
        base: Some(base_id.clone()),
        sequence: Some(1),
        supersedes: vec![base_id],
    });
    delta.integrity = vec![IntegrityProof {
        method: "content_digest".into(),
        algorithm: "sha256".into(),
        value: delta.sha256.clone(),
        evidence_url: None,
    }];
    delta.lineage = Some(SourceLineage::direct(Source::Wikidata));
    let delta_input = root.path().join("wikidata-delta.json");
    std::fs::write(&delta_input, &delta_bytes)?;
    store::import(&mut db, &delta, &delta_input)?;

    let generation = root.path().join("compact-delta");
    let audit_store = root.path().join("delta-audit");
    build::build_compact_with_policy_and_trust(
        &db,
        &generation,
        &ReviewPolicy::legacy_compatible(),
        None,
        &audit_store,
    )?;
    let registry = Registry::open(
        &generation,
        &argand_site_registry::file_digest(&generation.join("COMPLETE.json"))?,
    )?;
    assert_eq!(registry.lookup("Facebook", 20)?.total_entities, 0);
    let export = root.path().join("delta-audit.jsonl");
    argand_site_registry::audit::export(&registry, &audit_store, &export, false)?;
    let export = std::fs::read_to_string(export)?;
    assert!(export.contains("tombstone"));
    let replaced = &delta
        .coverage
        .as_ref()
        .context("delta coverage")?
        .supersedes[0];
    assert!(export.contains(replaced));
    Ok(())
}

#[test]
fn killed_import_resumes_from_a_committed_checkpoint() -> anyhow::Result<()> {
    const RECORDS: usize = 30_000;
    let root = tempfile::tempdir()?;
    let database = root.path().join("writer.sqlite");
    drop(store::open(&database)?);
    let mut dump = String::from("[");
    for index in 0..RECORDS {
        if index > 0 {
            dump.push(',');
        }
        dump.push_str(&serde_json::to_string(&common::entity(
            &format!("Q{}", 10_000_000 + index),
            &format!("Checkpoint {index}"),
            &[],
            &["https://example.org/"],
        ))?);
    }
    dump.push(']');
    let bytes = common::gzip(dump.as_bytes())?;
    let mut manifest = common::manifest(Source::Wikidata, Format::WikidataDump, &bytes)?;
    manifest.compression = argand_site_registry::model::Compression::Gzip;
    let source_id = manifest.id()?;
    let input = root.path().join("large.json.gz");
    let declaration = root.path().join("source.json");
    std::fs::write(&input, bytes)?;
    std::fs::write(&declaration, serde_json::to_vec_pretty(&manifest)?)?;

    for target in [256_i64, 2_048] {
        let mut child = Command::new(env!("CARGO_BIN_EXE_argand-site-registry"))
            .args(["import", "--database"])
            .arg(&database)
            .arg("--input")
            .arg(&input)
            .arg("--manifest")
            .arg(&declaration)
            .stdout(Stdio::null())
            .stderr(Stdio::null())
            .spawn()?;
        let deadline = Instant::now() + Duration::from_secs(30);
        let checkpoint = loop {
            if child.try_wait()?.is_some() {
                anyhow::bail!("fixture import completed before checkpoint interruption");
            }
            let connection = rusqlite::Connection::open(&database)?;
            let observed: i64 = connection.query_row(
                "SELECT COALESCE(max(checkpoint),0) FROM sources WHERE id=?1",
                [&source_id],
                |row| row.get(0),
            )?;
            if observed >= target {
                child.kill()?;
                child.wait()?;
                break observed;
            }
            ensure!(Instant::now() < deadline, "import checkpoint timed out");
            thread::sleep(Duration::from_millis(2));
        };
        assert!(checkpoint >= target);
    }
    let connection = rusqlite::Connection::open(&database)?;
    let partial: (i64, bool) = connection.query_row(
        "SELECT checkpoint,complete FROM sources WHERE id=?1",
        [&source_id],
        |row| Ok((row.get(0)?, row.get(1)?)),
    )?;
    assert!(partial.0 >= 256 && !partial.1);
    drop(connection);

    let status = Command::new(env!("CARGO_BIN_EXE_argand-site-registry"))
        .args(["import", "--database"])
        .arg(&database)
        .arg("--input")
        .arg(&input)
        .arg("--manifest")
        .arg(&declaration)
        .stdout(Stdio::null())
        .status()?;
    ensure!(status.success(), "checkpoint replay failed");
    let connection = store::open(&database)?;
    let completed: (i64, bool) = connection.query_row(
        "SELECT checkpoint,complete FROM sources WHERE id=?1",
        [&source_id],
        |row| Ok((row.get(0)?, row.get(1)?)),
    )?;
    assert_eq!(completed, (i64::try_from(RECORDS)?, true));
    Ok(())
}