diff --git a/CHANGELOG.md b/CHANGELOG.md index a3bea2f..8d83e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,5 @@ # Changelog -## 0.6.2 - 2026-09-22 - -- Retain the PSL normalization fact in compact catalogs and exercise the review - queue against the compact runtime rather than only the full writer projection. - -## 0.6.1 - 2026-09-22 - -- Accept complete Common Crawl domain-rank releases containing provider rows - that are not valid DNS hostnames. Such rows remain authenticated and counted - in source coordinates but cannot match or enter the official-domain catalog. -- Keep malformed graph schemas and numeric fields fail-closed, with a regression - derived from the real `com.your_domain` provider row. - ## 0.6.0 - 2026-09-22 - Add a streaming Common Crawl domain Web Graph adapter for harmonic-centrality, diff --git a/Cargo.lock b/Cargo.lock index 506a25f..afcb5e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,14 +78,14 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "argand-atomic" -version = "0.6.2" +version = "0.6.0" dependencies = [ "tempfile", ] [[package]] name = "argand-site-registry" -version = "0.6.2" +version = "0.6.0" dependencies = [ "anyhow", "argand-atomic", diff --git a/Cargo.toml b/Cargo.toml index 6f02cbe..6bfc5f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "3" members = ["crates/argand-atomic", "crates/argand-site-registry"] [workspace.package] -version = "0.6.2" +version = "0.6.0" authors = ["Nic Weyand"] edition = "2024" license = "AGPL-3.0-or-later" diff --git a/README.md b/README.md index c225623..93b18eb 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ facebook -> Facebook (Wikidata Q355) -> https://www.facebook.com/ The repository contains the library, CLI, schemas, migrations, synthetic fixtures, and independently authenticated public trust roots. It does not place a mutable production database in Git. Publishers import source evidence, collect -signed reviews, and distribute immutable signed registry generations. The current +signed reviews, and distribute immutable signed registry generations. The first public catalog generation is available as a release asset; see [Public catalog](docs/PUBLIC_CATALOG.md). diff --git a/crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs b/crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs index 96d8505..71c835c 100644 --- a/crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs +++ b/crates/argand-site-registry/src/adapters/common_crawl_web_graph.rs @@ -43,15 +43,9 @@ impl SourceAdapter for DomainRanks { let harmonic_value = nonnegative_finite(fields[1], "harmonic value")?; let pagerank_rank = positive_integer(fields[2], "PageRank rank")?; let pagerank_value = nonnegative_finite(fields[3], "PageRank value")?; + let target = reverse_domain(fields[4])?; let member_hosts = positive_integer(fields[5], "member host count")?; source_row += 1; - let Some(target) = reverse_domain(fields[4]) else { - // The provider graph contains a small amount of underscore and - // otherwise non-DNS host material. It cannot match the registry's - // normalized public identity domains, but it remains part of the - // authenticated input stream and source-row coordinate space. - continue; - }; if !self.targets.contains(&target) { continue; } @@ -124,13 +118,18 @@ fn nonnegative_finite(value: &str, field: &str) -> anyhow::Result { Ok(parsed) } -fn reverse_domain(value: &str) -> Option { - if value.is_empty() || value.len() > 253 || value != value.trim() { - return None; - } +fn reverse_domain(value: &str) -> anyhow::Result { + ensure!( + !value.is_empty() && value.len() <= 253 && value == value.trim(), + "invalid reversed domain" + ); let labels = value.split('.').collect::>(); - if labels.len() < 2 - || !labels.iter().all(|label| { + ensure!( + labels.len() >= 2, + "reversed domain needs at least two labels" + ); + ensure!( + labels.iter().all(|label| { !label.is_empty() && label.len() <= 63 && label @@ -144,9 +143,8 @@ fn reverse_domain(value: &str) -> Option { .as_bytes() .last() .is_some_and(u8::is_ascii_alphanumeric) - }) - { - return None; - } - Some(labels.into_iter().rev().collect::>().join(".")) + }), + "invalid reversed domain label" + ); + Ok(labels.into_iter().rev().collect::>().join(".")) } diff --git a/crates/argand-site-registry/src/audit.rs b/crates/argand-site-registry/src/audit.rs index 392b308..a9d270f 100644 --- a/crates/argand-site-registry/src/audit.rs +++ b/crates/argand-site-registry/src/audit.rs @@ -616,8 +616,7 @@ pub(crate) fn compact_runtime(db: &Connection) -> anyhow::Result<()> { INSERT OR IGNORE INTO runtime_facts SELECT fact FROM names; INSERT OR IGNORE INTO runtime_facts SELECT fact FROM popularity; INSERT OR IGNORE INTO runtime_facts SELECT fact FROM rejected; - INSERT OR IGNORE INTO runtime_facts SELECT item.value FROM edges,json_each(edges.facts) item; - INSERT OR IGNORE INTO runtime_facts SELECT id FROM facts WHERE predicate='psl';", + INSERT OR IGNORE INTO runtime_facts SELECT item.value FROM edges,json_each(edges.facts) item;", )?; db.execute( "DELETE FROM facts WHERE NOT EXISTS(SELECT 1 FROM runtime_facts r WHERE r.id=facts.id)", diff --git a/crates/argand-site-registry/tests/v05.rs b/crates/argand-site-registry/tests/v05.rs index 023abee..793e231 100644 --- a/crates/argand-site-registry/tests/v05.rs +++ b/crates/argand-site-registry/tests/v05.rs @@ -267,9 +267,6 @@ fn compact_generation_keeps_runtime_results_and_authenticates_cold_history() -> serde_json::to_value(full.lookup("Facebook", 20)?.candidates)?, serde_json::to_value(compact.lookup("Facebook", 20)?.candidates)? ); - let compact_queue = - argand_site_registry::queue::review_queue(&compact, common::timestamp()?, 100, 1_000)?; - assert!(!compact_queue.items.is_empty()); assert!( std::fs::metadata(compact_path.join("registry.sqlite"))?.len() < std::fs::metadata(root.path().join("full/registry.sqlite"))?.len() diff --git a/crates/argand-site-registry/tests/webgraph.rs b/crates/argand-site-registry/tests/webgraph.rs index f827ca1..6f3e8a8 100644 --- a/crates/argand-site-registry/tests/webgraph.rs +++ b/crates/argand-site-registry/tests/webgraph.rs @@ -153,6 +153,7 @@ fn domain_rank_parser_rejects_schema_and_value_drift() -> anyhow::Result<()> { "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\n", "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n0\t1\t1\t1\tcom.example\t1\n", "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\tNaN\t1\t1\tcom.example\t1\n", + "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\t1\t1\t1\tcom..example\t1\n", "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n1\t1\t1\t1\tcom.example\t0\n", ] { let root = tempfile::tempdir()?; @@ -166,30 +167,6 @@ fn domain_rank_parser_rejects_schema_and_value_drift() -> anyhow::Result<()> { Ok(()) } -#[test] -fn non_dns_provider_rows_are_skipped_without_losing_source_coordinates() -> anyhow::Result<()> { - let root = tempfile::tempdir()?; - let mut db = store::open(&root.path().join("store.sqlite"))?; - import_identity_candidates(&mut db, root.path())?; - let input = "#harmonicc_pos\t#harmonicc_val\t#pr_pos\t#pr_val\t#host_rev\t#n_hosts\n\ -1\t1\t1\t1\tcom.your_domain\t15\n\ -2\t1\t2\t1\tcom.facebook\t18795\n"; - import_graph(&mut db, root.path(), input.as_bytes())?; - - let retained: Vec<(String, String)> = { - let mut statement = db.prepare( - "SELECT r.native_id,f.value FROM records r JOIN facts f ON f.source_id=r.source_id AND f.ordinal=r.ordinal WHERE f.predicate='popularity' ORDER BY r.native_id", - )?; - statement - .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))? - .collect::>()? - }; - assert_eq!(retained.len(), 1); - assert_eq!(retained[0].0, "row:2"); - assert!(retained[0].1.contains("facebook.com")); - Ok(()) -} - #[test] fn domain_rank_source_url_is_exactly_allowlisted() -> anyhow::Result<()> { let good = "https://data.commoncrawl.org/projects/hyperlinkgraph/cc-main-2022-may-jun-aug/domain/cc-main-2022-may-jun-aug-domain-ranks.txt.gz"; diff --git a/docs/CONSUMERS.md b/docs/CONSUMERS.md index f108cdc..c072268 100644 --- a/docs/CONSUMERS.md +++ b/docs/CONSUMERS.md @@ -34,7 +34,7 @@ source attribution and application-specific malware/content policy. ## Current contracts -Code version 0.6.2 uses writer schema 5 and `argand.site-rules/v4`. +Code version 0.6.0 uses writer schema 5 and `argand.site-rules/v4`. New compact `COMPLETE.json` files use `argand.site-registry/v3` and bind: - authenticated `registry.sqlite` bytes; @@ -102,11 +102,9 @@ revocation history. Never mutate a complete generation to migrate it. See `UPSTREAM.json` records the original Argand extraction baseline and file hashes. Argand's prior integration pinned signed v0.5.0 revision -`3d3e08cdfd303df9fbd347a9bab2ba52ad575759`. Argand now pins the public signed -v0.6.2 source release at `3c89a540d910cb298ba752880efeb1ed157dab43` in -downstream commit `224616eb9f6685d1a656b113b8460fb80c0c5a6b`. -The public beta uses Site Registry as Navigate's authoritative auto-route -catalog. Its native `navigation-catalog/v2` +`3d3e08cdfd303df9fbd347a9bab2ba52ad575759`; the v0.6 downstream pin is recorded +by the Argand integration commit after this source release. The public beta uses Site Registry as +Navigate's authoritative auto-route catalog. Its native `navigation-catalog/v2` file is only a collection- and content-policy-bound serving projection compiled from one exact registry generation; it is not a second independently curated destination catalog. diff --git a/docs/PUBLIC_CATALOG.md b/docs/PUBLIC_CATALOG.md index c723bee..dbbdf89 100644 --- a/docs/PUBLIC_CATALOG.md +++ b/docs/PUBLIC_CATALOG.md @@ -1,14 +1,14 @@ # Public signed catalog -The v0.6.2 Forgejo release publishes the current immutable data generation that -Argand and any other Site Registry consumer can verify and resolve: +The v0.5.0 Forgejo release publishes the first immutable data generation that any +Site Registry consumer can verify and resolve: -- release: -- asset: `argand-site-registry-catalog-v0.6.2.tar.gz` +- release: +- asset: `argand-site-registry-catalog-20260920-v1.tar.gz` - asset SHA-256: - `48b0cdf453862d858c4bec6c564360e1309605e30af9aba1f54a9446b9bdbe41` + `d878fa057397effa5dc729d2fa3a689c8edd1f4112ef1326dd6131b3fdeab63e` - generation pin: - `5e5d8fd5dc1864dc3f4c53ec71cb5ac64f6db592cfbc8cc56f48a444378e2309` + `ede14746da8817aafdf705dd88cfeabbe8d23e1991e43a304acd8eca9249b18a` The release also carries a checksum file and an OpenSSH signature under namespace `argand-site-registry-release`. Verify it against @@ -17,13 +17,13 @@ The signed Git history is the independent channel for the trust root; do not lea the only trusted key from the archive it authenticates. ```bash -sha256sum --check argand-site-registry-catalog-v0.6.2.tar.gz.sha256 +sha256sum --check argand-site-registry-catalog-20260920-v1.tar.gz.sha256 ssh-keygen -Y verify \ -f trust/public-catalog-20260920/publisher-allowed-signers \ -I argand-site-registry-publisher-v1 \ -n argand-site-registry-release \ - -s argand-site-registry-catalog-v0.6.2.tar.gz.sig \ - < argand-site-registry-catalog-v0.6.2.tar.gz + -s argand-site-registry-catalog-20260920-v1.tar.gz.sig \ + < argand-site-registry-catalog-20260920-v1.tar.gz ``` After extraction, verify every member with `SHA256SUMS`, then authenticate the @@ -31,36 +31,25 @@ generation and exact reviewer trust root: ```bash argand-site-registry activate \ - --generation public-release-v0.6.2/catalog \ + --generation public-release-v0.5.0/catalog \ --current current.json \ --allowed-signers trust/public-catalog-20260920/publisher-allowed-signers \ --allowed-reviewers trust/public-catalog-20260920/reviewer-allowed-signers \ --identity argand-site-registry-publisher-v1 argand-site-registry resolve \ - --generation public-release-v0.6.2/catalog \ - --pin 5e5d8fd5dc1864dc3f4c53ec71cb5ac64f6db592cfbc8cc56f48a444378e2309 \ - --query "facebook" + --generation public-release-v0.5.0/catalog \ + --pin ede14746da8817aafdf705dd88cfeabbe8d23e1991e43a304acd8eca9249b18a \ + --query "yahoo mail" ``` ## Scope and trust -The v0.6.2 catalog contains 976 entities, 1,062 official-site edges, 20,178 -multilingual name facts, and Common Crawl Web Graph evidence for 840 domains that -already had imported identity assertions. Graph authority can prioritize review -and disambiguation, but cannot create an identity, official-site assertion, -review, vote, or redirect. The archive includes all 33 authenticated cold audit -objects referenced by the compact runtime generation. - -The bounded Wikidata discovery input is broad but not a representative or -high-demand sample. Its query and selection metadata are included for audit; raw -discovery output is never approval. - -The disclosed policy uses one automated evidence-gate reviewer group rather than -claiming human-review quorum. Fresh exact endpoint observations are required, and -source conflicts or dangerous drift need two groups, so the single automated -reviewer must abstain on those risks. Sticky revocations and -publisher/reviewer-key separation remain enabled. +This first catalog is deliberately small. Its disclosed policy uses one automated +evidence-gate reviewer group rather than claiming human-review quorum. Fresh exact +endpoint observations are required, and source conflicts or dangerous drift need +two groups, so the single automated reviewer must abstain on those risks. Sticky +revocations and publisher/reviewer key separation remain enabled. Consumers decide whether this policy is appropriate for their use. Preserve typed abstentions, retain attribution, and apply independent malware and content policy. @@ -71,12 +60,3 @@ The generation's approvals expire. Installing an immutable archive is not a prom that every decision stays valid forever: use the resolver's requested time, consume cumulative signed revocation feeds when published, and move to a newly signed full generation before relying on renewed decisions. - -## Regular updates - -The public source repository includes the same updater used to refresh candidate -generations. The example systemd timer runs weekly. It can download, authenticate, -import and build, but it holds no publisher key and cannot approve, sign or -activate a candidate. That separation lets any consumer automate evidence updates -without allowing a compromised downloader or changed upstream dataset to silently -change redirects. diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 309ad3f..eccb950 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -2,14 +2,6 @@ # Version 0.6.0 Web Graph and updater validation, 2026-09-22 -Patch release 0.6.1 additionally replays the real provider-shaped -`com.your_domain` case: the row remains in authenticated input/coordinate -accounting but is not retained as a DNS target. The following valid -`com.facebook` row is retained at its original `row:2` coordinate. Schema and -numeric corruption continue to fail closed. -The compact-generation lifecycle now also executes the review queue, proving its -PSL normalization input remains in the runtime catalog. - The evaluation contract was written before implementation. The new Common Crawl domain-rank integration passed five default integration tests; its sixth test is an explicit resource benchmark. The benchmark parsed 100,000 valid provider-shaped