fix: accept non-DNS Common Crawl graph rows

crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs

@@ -43,9 +43,15 @@ impl SourceAdapter for DomainRanks {
             let harmonic_value = nonnegative_finite(fields[1], "harmonic value")?;
             let pagerank_rank = positive_integer(fields[2], "PageRank rank")?;
             let pagerank_value = nonnegative_finite(fields[3], "PageRank value")?;
-            let target = reverse_domain(fields[4])?;
             let member_hosts = positive_integer(fields[5], "member host count")?;
             source_row += 1;
+            let Some(target) = reverse_domain(fields[4]) else {
+                // The provider graph contains a small amount of underscore and
+                // otherwise non-DNS host material. It cannot match the registry's
+                // normalized public identity domains, but it remains part of the
+                // authenticated input stream and source-row coordinate space.
+                continue;
+            };
             if !self.targets.contains(&target) {
                 continue;
             }
@@ -118,18 +124,13 @@ fn nonnegative_finite(value: &str, field: &str) -> anyhow::Result<f64> {
     Ok(parsed)
 }
 
-fn reverse_domain(value: &str) -> anyhow::Result<String> {
-    ensure!(
-        !value.is_empty() && value.len() <= 253 && value == value.trim(),
-        "invalid reversed domain"
-    );
+fn reverse_domain(value: &str) -> Option<String> {
+    if value.is_empty() || value.len() > 253 || value != value.trim() {
+        return None;
+    }
     let labels = value.split('.').collect::<Vec<_>>();
-    ensure!(
-        labels.len() >= 2,
-        "reversed domain needs at least two labels"
-    );
-    ensure!(
-        labels.iter().all(|label| {
+    if labels.len() < 2
+        || !labels.iter().all(|label| {
             !label.is_empty()
                 && label.len() <= 63
                 && label
@@ -143,8 +144,9 @@ fn reverse_domain(value: &str) -> anyhow::Result<String> {
                     .as_bytes()
                     .last()
                     .is_some_and(u8::is_ascii_alphanumeric)
-        }),
-        "invalid reversed domain label"
-    );
-    Ok(labels.into_iter().rev().collect::<Vec<_>>().join("."))
+        })
+    {
+        return None;
+    }
+    Some(labels.into_iter().rev().collect::<Vec<_>>().join("."))
 }

crates/argand-site-registry/tests/webgraph.rs

@@ -153,7 +153,6 @@ fn domain_rank_parser_rejects_schema_and_value_drift() -> anyhow::Result<()> {
         "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\n",
         "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n0\t1\t1\t1\tcom.example\t1\n",
         "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\tNaN\t1\t1\tcom.example\t1\n",
-        "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\t1\t1\t1\tcom..example\t1\n",
         "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\t1\t1\t1\tcom.example\t0\n",
     ] {
         let root = tempfile::tempdir()?;
@@ -167,6 +166,30 @@ fn domain_rank_parser_rejects_schema_and_value_drift() -> anyhow::Result<()> {
     Ok(())
 }
 
+#[test]
+fn non_dns_provider_rows_are_skipped_without_losing_source_coordinates() -> anyhow::Result<()> {
+    let root = tempfile::tempdir()?;
+    let mut db = store::open(&root.path().join("store.sqlite"))?;
+    import_identity_candidates(&mut db, root.path())?;
+    let input = "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\
+1\t1\t1\t1\tcom.your_domain\t15\n\
+2\t1\t2\t1\tcom.facebook\t18795\n";
+    import_graph(&mut db, root.path(), input.as_bytes())?;
+
+    let retained: Vec<(String, String)> = {
+        let mut statement = db.prepare(
+            "SELECT r.native_id,f.value FROM records r JOIN facts f ON f.source_id=r.source_id AND f.ordinal=r.ordinal WHERE f.predicate='popularity' ORDER BY r.native_id",
+        )?;
+        statement
+            .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
+            .collect::<Result<_, _>>()?
+    };
+    assert_eq!(retained.len(), 1);
+    assert_eq!(retained[0].0, "row:2");
+    assert!(retained[0].1.contains("facebook.com"));
+    Ok(())
+}
+
 #[test]
 fn domain_rank_source_url_is_exactly_allowlisted() -> anyhow::Result<()> {
     let good = "https://data.commoncrawl.org/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/domain/cc-main-2022-may-jun-aug-domain-ranks.txt.gz";