argand-site-registry/crates/argand-site-registry/tests/webgraph.rs

// By Nic Weyand!
//! Common Crawl Web Graph evidence stays source separated and cannot authorize routes.

#![allow(dead_code)] // Shared integration helpers intentionally cover a wider fixture surface.

mod common;

use argand_site_registry::{
    download::validate_source_url,
    model::{Compression, Format, Source, SourceManifest},
    query::ResolutionStatus,
    store,
};
use std::{fmt::Write as _, time::Instant};

const WEBGRAPH: &str = "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\
1\t3.2914686E7\t1\t0.018076941061056315\tcom.googleapis\t4482\n\
2\t3.2131562E7\t3\t0.012273178013351222\tcom.facebook\t18795\n";

fn import_identity_candidates(
    db: &mut rusqlite::Connection,
    root: &std::path::Path,
) -> anyhow::Result<()> {
    common::import(
        db,
        root,
        Source::Psl,
        Format::PslText,
        common::PSL.as_bytes(),
    )?;
    common::import(
        db,
        root,
        Source::Wikidata,
        Format::WikidataEntities,
        &serde_json::to_vec(&common::wikidata())?,
    )?;
    Ok(())
}

fn graph_manifest(db: &rusqlite::Connection, bytes: &[u8]) -> anyhow::Result<SourceManifest> {
    let mut manifest = common::manifest(
        Source::CommonCrawlWebGraph,
        Format::CommonCrawlDomainRanksTsv,
        bytes,
    )?;
    manifest.scope = store::web_graph_selection(db)?.scope;
    Ok(manifest)
}

fn import_graph(
    db: &mut rusqlite::Connection,
    root: &std::path::Path,
    bytes: &[u8],
) -> anyhow::Result<()> {
    let manifest = graph_manifest(db, bytes)?;
    let input = root.join(format!("{}.graph", manifest.sha256));
    std::fs::write(&input, bytes)?;
    store::import(db, &manifest, &input)?;
    Ok(())
}

#[test]
fn domain_ranks_are_popularity_only_and_preserve_provider_fields() -> anyhow::Result<()> {
    let root = tempfile::tempdir()?;
    let mut db = store::open(&root.path().join("store.sqlite"))?;
    import_identity_candidates(&mut db, root.path())?;
    let identity_before: i64 = db.query_row(
        "SELECT count(*) FROM facts WHERE predicate NOT IN ('psl','popularity')",
        [],
        |row| row.get(0),
    )?;
    import_graph(&mut db, root.path(), WEBGRAPH.as_bytes())?;

    let identity_facts: i64 = db.query_row(
        "SELECT count(*) FROM facts WHERE predicate NOT IN ('psl','popularity')",
        [],
        |row| row.get(0),
    )?;
    assert_eq!(identity_facts, identity_before);
    for table in ["reviews", "votes"] {
        let count: i64 = db.query_row(&format!("SELECT count(*) FROM {table}"), [], |row| {
            row.get(0)
        })?;
        assert_eq!(count, 0, "Web Graph unexpectedly populated {table}");
    }

    let first = common::build(&db, root.path(), "first")?;
    let second = common::build(&db, root.path(), "second")?;
    assert_eq!(first.identity, second.identity);
    assert_eq!(first.lookup("Facebook", 10)?.total_entities, 1);
    assert_ne!(
        first
            .resolve_explained("Facebook", None, None, common::timestamp()?)?
            .status,
        ResolutionStatus::Resolved
    );

    assert_eq!(first.popularity("googleapis.com", 10)?.total, 0);

    let lookup = first.popularity("facebook.com", 10)?;
    assert_eq!(lookup.total, 1);
    let observation = &lookup.observations[0];
    assert_eq!(observation.source, "common_crawl_web_graph");
    assert_eq!(observation.target, "facebook.com");
    assert_eq!(observation.value["harmonic_rank"], 2);
    assert_eq!(observation.value["harmonic_value"], 3.213_156_2E7);
    assert_eq!(observation.value["pagerank_rank"], 3);
    assert_eq!(
        observation.value["pagerank_value"],
        0.012_273_178_013_351_222_f64
    );
    assert_eq!(observation.value["member_hosts"], 18_795);
    assert!(observation.value.get("source_hosts").is_none());
    assert_eq!(
        observation.provenance["source"]["license"],
        "LicenseRef-Common-Crawl-Terms-of-Use"
    );
    let native_id: String = db.query_row(
        "SELECT r.native_id FROM records r JOIN sources s ON s.id=r.source_id WHERE s.source='common_crawl_web_graph'",
        [],
        |row| row.get(0),
    )?;
    assert_eq!(native_id, "row:2");
    Ok(())
}

#[test]
fn graph_import_requires_identity_candidates_and_exact_bound_scope() -> anyhow::Result<()> {
    let root = tempfile::tempdir()?;
    let mut db = store::open(&root.path().join("store.sqlite"))?;
    let mut manifest = common::manifest(
        Source::CommonCrawlWebGraph,
        Format::CommonCrawlDomainRanksTsv,
        WEBGRAPH.as_bytes(),
    )?;
    manifest.scope = format!("candidate-domains:{}", "0".repeat(64));
    let input = root.path().join("graph.tsv");
    std::fs::write(&input, WEBGRAPH)?;
    assert!(store::import(&mut db, &manifest, &input).is_err());

    import_identity_candidates(&mut db, root.path())?;
    assert!(store::import(&mut db, &manifest, &input).is_err());
    manifest.scope = store::web_graph_selection(&db)?.scope;
    store::import(&mut db, &manifest, &input)?;
    Ok(())
}

#[test]
fn domain_rank_parser_rejects_schema_and_value_drift() -> anyhow::Result<()> {
    for bad in [
        "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\n",
        "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\n",
        "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n0\t1\t1\t1\tcom.example\t1\n",
        "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\tNaN\t1\t1\tcom.example\t1\n",
        "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\t1\t1\t1\tcom.example\t0\n",
    ] {
        let root = tempfile::tempdir()?;
        let mut db = store::open(&root.path().join("store.sqlite"))?;
        import_identity_candidates(&mut db, root.path())?;
        assert!(
            import_graph(&mut db, root.path(), bad.as_bytes()).is_err(),
            "accepted malformed Web Graph input {bad:?}"
        );
    }
    Ok(())
}

#[test]
fn non_dns_provider_rows_are_skipped_without_losing_source_coordinates() -> anyhow::Result<()> {
    let root = tempfile::tempdir()?;
    let mut db = store::open(&root.path().join("store.sqlite"))?;
    import_identity_candidates(&mut db, root.path())?;
    let input = "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\
1\t1\t1\t1\tcom.your_domain\t15\n\
2\t1\t2\t1\tcom.facebook\t18795\n";
    import_graph(&mut db, root.path(), input.as_bytes())?;

    let retained: Vec<(String, String)> = {
        let mut statement = db.prepare(
            "SELECT r.native_id,f.value FROM records r JOIN facts f ON f.source_id=r.source_id AND f.ordinal=r.ordinal WHERE f.predicate='popularity' ORDER BY r.native_id",
        )?;
        statement
            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
            .collect::<Result<_, _>>()?
    };
    assert_eq!(retained.len(), 1);
    assert_eq!(retained[0].0, "row:2");
    assert!(retained[0].1.contains("facebook.com"));
    Ok(())
}

#[test]
fn domain_rank_source_url_is_exactly_allowlisted() -> anyhow::Result<()> {
    let good = "https://data.commoncrawl.org/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/domain/cc-main-2022-may-jun-aug-domain-ranks.txt.gz";
    validate_source_url(Source::CommonCrawlWebGraph, good)?;
    for bad in [
        "http://data.commoncrawl.org/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/domain/cc-main-2022-may-jun-aug-domain-ranks.txt.gz",
        "https://data.commoncrawl.org.evil.example/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/domain/cc-main-2022-may-jun-aug-domain-ranks.txt.gz",
        "https://data.commoncrawl.org/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/host/cc-main-2022-may-jun-aug-host-ranks.txt.gz",
        "https://data.commoncrawl.org/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/domain/other-domain-ranks.txt.gz",
        "https://data.commoncrawl.org/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/domain/cc-main-2022-may-jun-aug-domain-ranks.txt.gz?x=1",
    ] {
        assert!(
            validate_source_url(Source::CommonCrawlWebGraph, bad).is_err(),
            "accepted unreviewed Web Graph URL {bad}"
        );
    }
    Ok(())
}

#[test]
fn gzip_source_is_consumed_and_authenticated_end_to_end() -> anyhow::Result<()> {
    let root = tempfile::tempdir()?;
    let compressed = common::gzip(WEBGRAPH.as_bytes())?;
    let input = root.path().join("domain-ranks.txt.gz");
    std::fs::write(&input, &compressed)?;
    let mut db = store::open(&root.path().join("store.sqlite"))?;
    import_identity_candidates(&mut db, root.path())?;
    let mut manifest = graph_manifest(&db, &compressed)?;
    manifest.compression = Compression::Gzip;
    store::import(&mut db, &manifest, &input)?;
    let registry = common::build(&db, root.path(), "gzip")?;
    assert_eq!(registry.popularity("facebook.com", 10)?.total, 1);
    Ok(())
}

#[test]
#[ignore = "explicit resource benchmark"]
fn hundred_thousand_rows_import_within_engineering_budget() -> anyhow::Result<()> {
    let mut input =
        String::from("#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n");
    for rank in 1..=100_000_u32 {
        let reversed_domain = if rank == 50_000 {
            "com.facebook".to_owned()
        } else {
            format!("com.example-{rank}")
        };
        writeln!(
            input,
            "{rank}\t{}\t{rank}\t{}\t{reversed_domain}\t1",
            100_001 - rank,
            0.1
        )?;
    }
    let root = tempfile::tempdir()?;
    let mut db = store::open(&root.path().join("store.sqlite"))?;
    import_identity_candidates(&mut db, root.path())?;
    let started = Instant::now();
    import_graph(&mut db, root.path(), input.as_bytes())?;
    assert!(
        started.elapsed().as_secs_f64() < 10.0,
        "100k Web Graph rows exceeded the 10-second engineering budget"
    );
    let retained: i64 = db.query_row(
        "SELECT count(*) FROM facts WHERE predicate='popularity'",
        [],
        |row| row.get(0),
    )?;
    assert_eq!(retained, 1);
    Ok(())
}