diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d83e1a..ecd31f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.6.1 - 2026-09-22 + +- Accept complete Common Crawl domain-rank releases containing provider rows + that are not valid DNS hostnames. Such rows remain authenticated and counted + in source coordinates but cannot match or enter the official-domain catalog. +- Keep malformed graph schemas and numeric fields fail-closed, with a regression + derived from the real `com.your_domain` provider row. + ## 0.6.0 - 2026-09-22 - Add a streaming Common Crawl domain Web Graph adapter for harmonic-centrality, diff --git a/Cargo.lock b/Cargo.lock index afcb5e2..1ec3b30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,14 +78,14 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "argand-atomic" -version = "0.6.0" +version = "0.6.1" dependencies = [ "tempfile", ] [[package]] name = "argand-site-registry" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "argand-atomic", diff --git a/Cargo.toml b/Cargo.toml index 6bfc5f0..a4f7ba4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "3" members = ["crates/argand-atomic", "crates/argand-site-registry"] [workspace.package] -version = "0.6.0" +version = "0.6.1" authors = ["Nic Weyand"] edition = "2024" license = "AGPL-3.0-or-later" diff --git a/crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs b/crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs index 71c835c..96d8505 100644 --- a/crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs +++ b/crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs @@ -43,9 +43,15 @@ impl SourceAdapter for DomainRanks { let harmonic_value = nonnegative_finite(fields[1], "harmonic value")?; let pagerank_rank = positive_integer(fields[2], "PageRank rank")?; let pagerank_value = nonnegative_finite(fields[3], "PageRank value")?; - let target = reverse_domain(fields[4])?; let member_hosts = positive_integer(fields[5], "member host count")?; source_row += 1; + let Some(target) = reverse_domain(fields[4]) else { + // The provider graph contains a small amount of underscore and + // otherwise non-DNS host material. It cannot match the registry's + // normalized public identity domains, but it remains part of the + // authenticated input stream and source-row coordinate space. + continue; + }; if !self.targets.contains(&target) { continue; } @@ -118,18 +124,13 @@ fn nonnegative_finite(value: &str, field: &str) -> anyhow::Result { Ok(parsed) } -fn reverse_domain(value: &str) -> anyhow::Result { - ensure!( - !value.is_empty() && value.len() <= 253 && value == value.trim(), - "invalid reversed domain" - ); +fn reverse_domain(value: &str) -> Option { + if value.is_empty() || value.len() > 253 || value != value.trim() { + return None; + } let labels = value.split('.').collect::>(); - ensure!( - labels.len() >= 2, - "reversed domain needs at least two labels" - ); - ensure!( - labels.iter().all(|label| { + if labels.len() < 2 + || !labels.iter().all(|label| { !label.is_empty() && label.len() <= 63 && label @@ -143,8 +144,9 @@ fn reverse_domain(value: &str) -> anyhow::Result { .as_bytes() .last() .is_some_and(u8::is_ascii_alphanumeric) - }), - "invalid reversed domain label" - ); - Ok(labels.into_iter().rev().collect::>().join(".")) + }) + { + return None; + } + Some(labels.into_iter().rev().collect::>().join(".")) } diff --git a/crates/argand-site-registry/tests/webgraph.rs b/crates/argand-site-registry/tests/webgraph.rs index 6f3e8a8..f827ca1 100644 --- a/crates/argand-site-registry/tests/webgraph.rs +++ b/crates/argand-site-registry/tests/webgraph.rs @@ -153,7 +153,6 @@ fn domain_rank_parser_rejects_schema_and_value_drift() -> anyhow::Result<()> { "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\n", "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n0\t1\t1\t1\tcom.example\t1\n", "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\tNaN\t1\t1\tcom.example\t1\n", - "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\t1\t1\t1\tcom..example\t1\n", "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\t1\t1\t1\tcom.example\t0\n", ] { let root = tempfile::tempdir()?; @@ -167,6 +166,30 @@ fn domain_rank_parser_rejects_schema_and_value_drift() -> anyhow::Result<()> { Ok(()) } +#[test] +fn non_dns_provider_rows_are_skipped_without_losing_source_coordinates() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let mut db = store::open(&root.path().join("store.sqlite"))?; + import_identity_candidates(&mut db, root.path())?; + let input = "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\ +1\t1\t1\t1\tcom.your_domain\t15\n\ +2\t1\t2\t1\tcom.facebook\t18795\n"; + import_graph(&mut db, root.path(), input.as_bytes())?; + + let retained: Vec<(String, String)> = { + let mut statement = db.prepare( + "SELECT r.native_id,f.value FROM records r JOIN facts f ON f.source_id=r.source_id AND f.ordinal=r.ordinal WHERE f.predicate='popularity' ORDER BY r.native_id", + )?; + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::>()? + }; + assert_eq!(retained.len(), 1); + assert_eq!(retained[0].0, "row:2"); + assert!(retained[0].1.contains("facebook.com")); + Ok(()) +} + #[test] fn domain_rank_source_url_is_exactly_allowlisted() -> anyhow::Result<()> { let good = "https://data.commoncrawl.org/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/domain/cc-main-2022-may-jun-aug-domain-ranks.txt.gz"; diff --git a/docs/CONSUMERS.md b/docs/CONSUMERS.md index c072268..1433fdb 100644 --- a/docs/CONSUMERS.md +++ b/docs/CONSUMERS.md @@ -34,7 +34,7 @@ source attribution and application-specific malware/content policy. ## Current contracts -Code version 0.6.0 uses writer schema 5 and `argand.site-rules/v4`. +Code version 0.6.1 uses writer schema 5 and `argand.site-rules/v4`. New compact `COMPLETE.json` files use `argand.site-registry/v3` and bind: - authenticated `registry.sqlite` bytes; diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index eccb950..65e9b7b 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -2,6 +2,12 @@ # Version 0.6.0 Web Graph and updater validation, 2026-09-22 +Patch release 0.6.1 additionally replays the real provider-shaped +`com.your_domain` case: the row remains in authenticated input/coordinate +accounting but is not retained as a DNS target. The following valid +`com.facebook` row is retained at its original `row:2` coordinate. Schema and +numeric corruption continue to fail closed. + The evaluation contract was written before implementation. The new Common Crawl domain-rank integration passed five default integration tests; its sixth test is an explicit resource benchmark. The benchmark parsed 100,000 valid provider-shaped