From 2a0fe1714b8ffb2e80203722bcb7987c630f432d Mon Sep 17 00:00:00 2001 From: nicweyand Date: Sat, 12 Sep 2026 21:38:59 -0400 Subject: [PATCH] feat: establish standalone Argand Site Registry --- .forgejo/workflows/ci.yml | 58 + .gitignore | 17 + AGENTS.md | 15 + CONTRIBUTING.md | 41 + Cargo.lock | 2359 +++++++++++++++++ Cargo.toml | 44 + GOVERNANCE.md | 30 + LICENSE | 661 +++++ LICENSE_SOURCES.md | 10 + README.md | 110 + SECURITY.md | 29 + UPSTREAM.json | 228 ++ crates/argand-atomic/Cargo.toml | 16 + crates/argand-atomic/src/lib.rs | 211 ++ crates/argand-site-registry/.gitignore | 8 + crates/argand-site-registry/Cargo.toml | 35 + .../argand-site-registry/LICENSE_SOURCES.md | 60 + crates/argand-site-registry/README.md | 348 +++ .../examples/argand-site-registry.service | 20 + .../examples/argand-site-registry.timer | 12 + .../argand-site-registry/examples/lookup.rs | 24 + .../argand-site-registry/examples/update.toml | 50 + .../argand-site-registry/migrations/001.sql | 43 + .../src/adapters/csv_sources.rs | 139 + .../src/adapters/curlie.rs | 171 ++ .../argand-site-registry/src/adapters/mod.rs | 76 + .../src/adapters/wikidata.rs | 186 ++ crates/argand-site-registry/src/build.rs | 359 +++ crates/argand-site-registry/src/cli.rs | 380 +++ crates/argand-site-registry/src/crux.rs | 353 +++ crates/argand-site-registry/src/download.rs | 430 +++ crates/argand-site-registry/src/evidence.rs | 125 + crates/argand-site-registry/src/identity.rs | 220 ++ crates/argand-site-registry/src/json.rs | 76 + crates/argand-site-registry/src/lib.rs | 58 + crates/argand-site-registry/src/main.rs | 8 + crates/argand-site-registry/src/model.rs | 214 ++ crates/argand-site-registry/src/normalize.rs | 163 ++ .../argand-site-registry/src/observation.rs | 60 + crates/argand-site-registry/src/query.rs | 332 +++ crates/argand-site-registry/src/release.rs | 237 ++ crates/argand-site-registry/src/review.rs | 116 + crates/argand-site-registry/src/store.rs | 209 ++ crates/argand-site-registry/src/update.rs | 96 + crates/argand-site-registry/tests/cli.rs | 349 +++ .../argand-site-registry/tests/common/mod.rs | 179 ++ crates/argand-site-registry/tests/failures.rs | 272 ++ crates/argand-site-registry/tests/identity.rs | 173 ++ crates/argand-site-registry/tests/registry.rs | 322 +++ docs/CONSUMERS.md | 57 + docs/INDEX.md | 13 + docs/RELEASING.md | 92 + docs/TRUST.md | 63 + .../plans/2026-09-12-standalone.md | 37 + .../specs/2026-09-12-standalone-design.md | 37 + examples/lookup.py | 27 + scripts/check.sh | 28 + scripts/check_consumers.py | 58 + scripts/source_release.py | 211 ++ tests/test_source_release.py | 139 + 60 files changed, 10494 insertions(+) create mode 100644 .forgejo/workflows/ci.yml create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 CONTRIBUTING.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 GOVERNANCE.md create mode 100644 LICENSE create mode 100644 LICENSE_SOURCES.md create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 UPSTREAM.json create mode 100644 crates/argand-atomic/Cargo.toml create mode 100644 crates/argand-atomic/src/lib.rs create mode 100644 crates/argand-site-registry/.gitignore create mode 100644 crates/argand-site-registry/Cargo.toml create mode 100644 crates/argand-site-registry/LICENSE_SOURCES.md create mode 100644 crates/argand-site-registry/README.md create mode 100644 crates/argand-site-registry/examples/argand-site-registry.service create mode 100644 crates/argand-site-registry/examples/argand-site-registry.timer create mode 100644 crates/argand-site-registry/examples/lookup.rs create mode 100644 crates/argand-site-registry/examples/update.toml create mode 100644 crates/argand-site-registry/migrations/001.sql create mode 100644 crates/argand-site-registry/src/adapters/csv_sources.rs create mode 100644 crates/argand-site-registry/src/adapters/curlie.rs create mode 100644 crates/argand-site-registry/src/adapters/mod.rs create mode 100644 crates/argand-site-registry/src/adapters/wikidata.rs create mode 100644 crates/argand-site-registry/src/build.rs create mode 100644 crates/argand-site-registry/src/cli.rs create mode 100644 crates/argand-site-registry/src/crux.rs create mode 100644 crates/argand-site-registry/src/download.rs create mode 100644 crates/argand-site-registry/src/evidence.rs create mode 100644 crates/argand-site-registry/src/identity.rs create mode 100644 crates/argand-site-registry/src/json.rs create mode 100644 crates/argand-site-registry/src/lib.rs create mode 100644 crates/argand-site-registry/src/main.rs create mode 100644 crates/argand-site-registry/src/model.rs create mode 100644 crates/argand-site-registry/src/normalize.rs create mode 100644 crates/argand-site-registry/src/observation.rs create mode 100644 crates/argand-site-registry/src/query.rs create mode 100644 crates/argand-site-registry/src/release.rs create mode 100644 crates/argand-site-registry/src/review.rs create mode 100644 crates/argand-site-registry/src/store.rs create mode 100644 crates/argand-site-registry/src/update.rs create mode 100644 crates/argand-site-registry/tests/cli.rs create mode 100644 crates/argand-site-registry/tests/common/mod.rs create mode 100644 crates/argand-site-registry/tests/failures.rs create mode 100644 crates/argand-site-registry/tests/identity.rs create mode 100644 crates/argand-site-registry/tests/registry.rs create mode 100644 docs/CONSUMERS.md create mode 100644 docs/INDEX.md create mode 100644 docs/RELEASING.md create mode 100644 docs/TRUST.md create mode 100644 docs/superpowers/plans/2026-09-12-standalone.md create mode 100644 docs/superpowers/specs/2026-09-12-standalone-design.md create mode 100644 examples/lookup.py create mode 100644 scripts/check.sh create mode 100644 scripts/check_consumers.py create mode 100644 scripts/source_release.py create mode 100644 tests/test_source_release.py diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..bc69d81 --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,58 @@ +name: Standalone registry checks +on: + push: + branches: [main] + workflow_dispatch: {} + +jobs: + check: + # Register this label only on a disposable isolated runner. Never use an + # Argand production/host runner. Prerequisites are in docs/RELEASING.md. + runs-on: site-registry-isolated + timeout-minutes: 30 + env: + REGISTRY_REPOSITORY_URL: ${{ forgejo.server_url }}/${{ forgejo.repository }}.git + REGISTRY_REVISION: ${{ forgejo.sha }} + CARGO_BUILD_JOBS: '2' + CARGO_TERM_COLOR: never + RUSTC_WRAPPER: '' + steps: + - name: Fetch the exact public source revision without credentials + shell: bash + run: | + set -euo pipefail + [[ "$REGISTRY_REVISION" =~ ^[0-9a-f]{40}$ ]] + mkdir checkout + cd checkout + git init --initial-branch=main + git remote add origin "$REGISTRY_REPOSITORY_URL" + git fetch --depth=1 origin "$REGISTRY_REVISION" + git checkout --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$REGISTRY_REVISION" + - name: Fetch locked build dependencies and run offline acceptance + shell: bash + run: | + set -euo pipefail + cd checkout + export CARGO_TARGET_DIR="$PWD/target" + export CARGO_BUILD_BUILD_DIR="$PWD/build" + cargo fetch --locked + bash scripts/check.sh + - name: Verify deterministic source packaging and rebuild the archive + shell: bash + run: | + set -euo pipefail + cd checkout + python3 scripts/source_release.py create --output ../source-release-a > ../release-a.json + python3 scripts/source_release.py create --output ../source-release-b > ../release-b.json + cmp ../source-release-a/source.tar.gz ../source-release-b/source.tar.gz + cmp ../source-release-a/RELEASE.json ../source-release-b/RELEASE.json + registry_pin="$(python3 -c 'import json; print(json.load(open("../release-a.json"))["pin"])')" + python3 scripts/source_release.py verify --release ../source-release-a --pin "$registry_pin" + mkdir ../unpacked + tar -xzf ../source-release-a/source.tar.gz -C ../unpacked + registry_prefix="$(python3 -c 'import json; print(json.load(open("../source-release-a/RELEASE.json"))["prefix"])')" + cd "../unpacked/$registry_prefix" + export CARGO_TARGET_DIR="$PWD/target" + export CARGO_BUILD_BUILD_DIR="$PWD/build" + bash scripts/check.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5f5d642 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +/target/ +/build/ +/dist/ +/data/ +/cache/ +*.sqlite +*.sqlite-* +*.db +*.input +*.sig +*.pem +*.key +.env +.env.* +__pycache__/ +*.pyc +!Cargo.lock diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2939c16 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,15 @@ +# Site Registry contributor guidance + +Preserve the Rust library/CLI architecture and the source/derivation/review boundary. +No entity joins solely from names or host similarity. Keep source licenses, +conflicting evidence, immutable generations and revocations intact. + +Use `docs/INDEX.md` for the documentation map. Keep code modules focused and reuse +existing types. Run `cargo fetch --locked` then `bash scripts/check.sh`; do not +weaken a gate or invent provider schemas. No datasets, keys, private deployment +configuration or approval logs belong in this code repository. + +Keep changes bounded. Do not spawn subagents or modify Argand's shared checkout +or build caches from this repository. Coordinate an explicit downstream handoff +before changing Argand's consumer. Preserve signed source history and use +fast-forward Git pushes. CI and update jobs do not authorize dataset promotion. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..896d20e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Contributing + +Explain the observed problem and the proposed behavior in an issue or pull request. +Include a small authored fixture that reproduces a correctness or security failure. +Keep each module focused, preserve existing Rust style and run `bash scripts/check.sh` +after `cargo fetch --locked`. Changes to contracts or normalization need a documented +migration and proof that stale approvals cannot silently survive changed evidence. + +## Code and dependency changes + +Submit code you are authorized to contribute under AGPL-3.0-or-later. Preserve +copyright/license notices for reused material and identify its origin. Do not +submit private datasets, credentials, signing keys, production configuration or +real user query logs. Synthetic fixtures should say that they are synthetic. + +Dependency changes must include the lockfile diff, reason, source/license review +and complete offline gates. Keep builds practical on limited CPU and memory. +Do not replace an existing algorithm or lower a threshold simply to pass a test. + +## Source adapters and website corrections + +A new source requires primary documentation of commercial reuse rights, exact +consumed fields, required attribution, documented current format and distribution +URLs. Add a common-interface adapter, bounded streaming/resume behavior, input +validation and adversarial fixtures. Update LICENSE_SOURCES and attribution before +acceptance. An attractive dataset with unverified terms is not admissible. + +For a website correction, include the entity ID, exact URL, relation/role, country +or locale scope, source record/revision, conflicting claims and dated immutable +evidence. Explain how the evidence supports identity and scope. Popularity, TLS, +hostname resemblance and an unauthenticated ownership claim are insufficient alone. +Do not include executable HTML, authenticated sessions or unnecessary personal data. + +Submissions are proposals. The software repository does not contain a production +approval log or publisher keys. A publisher must independently review evidence, +append an expiring decision, build and inspect the candidate, then sign and activate +it under its own trust policy. Source changes cannot silently renew an approval. + +Disclose relevant ownership or commercial conflicts. A reviewer should not approve +their own disputed website claim. Explain disagreements with evidence; preserve +prior decisions and corrections. Report exploitable failures through SECURITY.md. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..b2cefd0 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2359 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +# engine/Cargo.lock +# By Nic Weyand! +# This file is automatically @generated by Cargo and is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "argand-atomic" +version = "0.1.0" +dependencies = [ + "tempfile", +] + +[[package]] +name = "argand-site-registry" +version = "0.1.0" +dependencies = [ + "anyhow", + "argand-atomic", + "bzip2", + "chrono", + "clap", + "csv", + "flate2", + "http", + "publicsuffix", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "sha2", + "tar", + "tempfile", + "tokio", + "toml", + "unicode-normalization", + "url", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +dependencies = [ + "idna", + "psl-types", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..eec1733 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,44 @@ +# By Nic Weyand! +[workspace] +resolver = "3" +members = ["crates/argand-atomic", "crates/argand-site-registry"] + +[workspace.package] +version = "0.1.0" +authors = ["Nic Weyand"] +edition = "2024" +license = "AGPL-3.0-or-later" +rust-version = "1.97" + +[workspace.dependencies] +anyhow = "1.0.102" +chrono = { version = "0.4.44", features = ["serde"] } +clap = { version = "4.5.60", features = ["derive"] } +flate2 = "1.1.9" +http = "1.4.0" +reqwest = { version = "0.13.2", default-features = false, features = ["json", "query", "rustls", "stream"] } +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.149" +sha2 = "0.10.9" +tempfile = "3.27.0" +tokio = { version = "1.52.1", features = ["full"] } +toml = "1.0.7" +unicode-normalization = "0.1.25" +url = { version = "2.5.8", features = ["serde"] } + +[workspace.lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" + +[workspace.lints.clippy] +all = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } +unwrap_used = "deny" +expect_used = "deny" +panic = "deny" +float_cmp = "deny" + +[profile.release] +codegen-units = 1 +lto = "thin" +strip = "symbols" diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..084769a --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,30 @@ +# Governance + +Nic Weyand is the initial code maintainer. Maintainers review implementation, +dependency, format, provenance and source-license changes. There is no established +multi-party review board or centrally operated public dataset implied by this repo. +Maintainer additions and changes to this policy should be reviewed in public Git +history, with conflicts of interest disclosed. + +Dataset publishers operate independently. Each publisher names its reviewers, +publishes its evidence and review policy, and distributes trusted signing keys +through a channel separate from the dataset. Consumers decide which publisher +identities they accept. Code-maintainer status does not grant authority to change +a consumer's accepted destinations or signing keys. + +Policy, license, normalization, source-allowlist and signature changes receive +explicit maintainer review and complete acceptance checks. Additional independent +review is appropriate for trust-boundary changes when another qualified reviewer +is available. This is a governance expectation; the current software does not +enforce a multi-reviewer quorum or authenticate a free-text reviewer name. + +Corrections and appeals must identify the exact assertion or review fingerprint +and supply contrary evidence. Retain the original claim and decision, append the +correction or revocation, and explain its scope. Urgent suspected malicious +destinations may be revoked pending investigation. A release signer must not hide +revocations by selecting an old generation. + +No payment, source popularity or contributor reputation buys destination approval. +Repeated deceptive submissions can be rejected while their supporting incident +record remains available to affected publishers. Avoid public exposure of secrets +or personal information when documenting abuse; follow SECURITY.md. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/LICENSE_SOURCES.md b/LICENSE_SOURCES.md new file mode 100644 index 0000000..b823b75 --- /dev/null +++ b/LICENSE_SOURCES.md @@ -0,0 +1,10 @@ +# Site Registry source licenses + +The canonical, complete source license document is +[crates/argand-site-registry/LICENSE_SOURCES.md](crates/argand-site-registry/LICENSE_SOURCES.md). +It is compiled into the CLI and included in every registry generation. + +This repository's code is AGPL-3.0-or-later; see [LICENSE](LICENSE). Imported data +keeps its own source licenses. Preserve provenance, source identifiers, timestamps, +notices and attribution when selecting, deriving or redistributing facts. A source +archive contains implementation and authored tests, not a provider dataset. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7344c2e --- /dev/null +++ b/README.md @@ -0,0 +1,110 @@ +# Argand Site Registry + +Evidence-backed website identity and regional resolution, as a Rust library and CLI. + +Look up an entity's names, aliases and websites; inspect each source assertion; +resolve an explicitly reviewed primary or regional destination. Sources, conflicting +claims, licenses and review history remain available for audit. Everything runs +locally with SQLite. No search engine, hosted account or GPU is required. + +`facebook → Facebook → facebook.com` is a name-to-entity-to-registrable-domain +lookup. The retained real Wikidata Facebook record has `www.facebook.com` and +`m.facebook.com` as distinct properties. Its Amazon record includes `amazon.com`, +`amazon.co.uk` and `amazon.de` on the same entity. Similar hostname spelling never +establishes common ownership. Imported links await review before `resolve` can +return a destination; source confidence is not a malware-safety guarantee. + +## Build and try it + +Linux is the currently validated platform. Install Rust (tested with 1.98.1; the +inherited minimum is 1.97), a C compiler, CMake, Perl and OpenSSH (`ssh-keygen`). +Python 3.11+ is needed for release tooling and the Python example. SQLite is built +with the binary. From the root of this source checkout or extracted release: + +```bash +cargo fetch --locked +cargo build --release --locked --offline -p argand-site-registry +./target/release/argand-site-registry --help +``` + +If your Cargo configuration sets a different target directory, set +`CARGO_TARGET_DIR="$PWD/target"` before these commands. To install the CLI: + +```bash +cargo install --path crates/argand-site-registry --locked --offline +argand-site-registry --help +``` + +Run the native all-source example in a **new directory outside the checkout**: + +```bash +ARGAND_REGISTRY_E2E_OUTPUT=/tmp/site-registry-example \ + cargo test -p argand-site-registry --test cli --locked --offline -- --nocapture +``` + +It imports synthetic Wikidata, Majestic, CrUX, Curlie and PSL fixtures, checks +idempotency, joins explicitly reviewed identities, approves a destination, signs +and activates it, revokes it, and rejects rollback past that revocation. Fixture +signing keys and approvals are disposable test material. All production inputs +belong in a configurable data/cache directory outside the source checkout. + +Follow the [operator guide](crates/argand-site-registry/README.md) for actual +downloads, imports, regional reviews, signed datasets and weekly candidate updates. +CrUX acquisition requires explicit credentials and a billing cap. No source +downloads or scheduled jobs run during installation or tests. + +## Use it in another project + +The [Rust example](crates/argand-site-registry/examples/lookup.rs) opens a generation +once with an externally trusted receipt hash and returns the full lookup envelope. +The [Python example](examples/lookup.py) calls the same native CLI; it does not +implement a second resolver. Both preserve attribution and fact provenance. + +```bash +cargo run --locked --offline -p argand-site-registry --example lookup -- \ + --generation /data/registry/generation --pin "$REGISTRY_TRUSTED_PIN" --query facebook + +python3 examples/lookup.py --binary ./target/release/argand-site-registry \ + --generation /data/registry/generation --pin "$REGISTRY_TRUSTED_PIN" --query facebook +``` + +Obtain the pin from a trusted publisher channel or verify the release signature +against an independently configured key. Reading a hash from the same untrusted +download does not authenticate it. Never turn `lookup.candidates[0]` into an +automatic destination: use the native `resolve` result and your application's +own admission policy. A null destination is a meaningful abstention. + +The [consumer contract](docs/CONSUMERS.md) covers Rust dependencies, CLI JSON, +SQLite/JSONL distribution, compatibility and Argand's eventual upstream cutover. + +## Sources and trust + +Wikidata (CC0), Majestic Million (CC BY 3.0), CrUX (CC BY 4.0), Curlie (CC BY 3.0) +and the Public Suffix List (MPL 2.0) remain logically separate. Read the exact +[source licenses and attribution rules](LICENSE_SOURCES.md) before redistribution. +Curlie attribution applies to names and categories as well as descriptions. + +The [trust policy](docs/TRUST.md) explains enforced checks, publisher responsibilities, +evidence standards, expiry and revocation. [CONTRIBUTING.md](CONTRIBUTING.md), +[GOVERNANCE.md](GOVERNANCE.md) and [SECURITY.md](SECURITY.md) cover contributions, +decisions, disputes and incidents. Pull requests cannot directly approve destinations. + +## Development and releases + +```bash +cargo fetch --locked +bash scripts/check.sh +``` + +The check runs formatting, all-target compilation, strict Clippy, Rust tests and +documentation, Python release tests, and native Rust/Python consumer parity. +[RELEASING.md](docs/RELEASING.md) covers deterministic source archives, signature +verification and rebuilding outside the checkout. The Forgejo workflow requires +a dedicated isolated runner; it has no signing or dataset-promotion authority. + +The code remains **AGPL-3.0-or-later**; the complete license is in [LICENSE](LICENSE). +Original attribution is retained. [UPSTREAM.json](UPSTREAM.json) records the signed +Argand extraction revision and original file hashes. This initial independent +package preserves the existing registry runtime and schema. Hosted publication, +runner provisioning and changing Argand's dependency require separate completion; +the initial extraction does not itself establish any of those states. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..200049d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,29 @@ +# Security reports and incidents + +For exploitable code failures or sensitive link-poisoning evidence, contact Nic +Weyand at the maintainer address in the verified signed source-release commit +(`git show -s --format=%ae COMMIT`). Check the signature and maintainer identity +first. Do not post credentials or an active exploit in a public issue. Ordinary +data corrections can follow CONTRIBUTING.md. No response-time SLA is offered. + +Include the software commit, generation receipt pin, source snapshot/statement +IDs, exact fingerprint, affected command and a minimal sanitized reproduction. +Describe what was expected, what happened and whether anyone has received the +affected generation. Use inert or local fixtures where possible. + +Publishers should append a revocation for a suspected malicious destination, +build and inspect a replacement generation with the full review history, sign it +with a trusted key and deliver its new pin to consumers. Consumers must refresh +their verified registry and any derived routing catalogue. Merely appending a +local revocation does not notify running applications or invalidate their caches. + +For signer compromise, remove that signer from consumer trust files through an +independent authenticated channel, investigate affected releases and rotate the +key. Do not accept a replacement key solely because it appears inside a suspect +dataset. Preserve affected source bytes and receipts as evidence; publish a +sanitized advisory identifying affected pins and corrective versions. + +Source release CI receives no dataset credentials or signing keys. Configure it +on a disposable isolated runner without production mounts or access to trusted +publisher state. The source archive verifier checks hashes and safe contents; +it is not a malware scanner or a substitute for source review. diff --git a/UPSTREAM.json b/UPSTREAM.json new file mode 100644 index 0000000..6c72131 --- /dev/null +++ b/UPSTREAM.json @@ -0,0 +1,228 @@ +{ + "schema": "argand.site-upstream/v1", + "repository": "git@git.argand.org:nicweyand/argand.git", + "commit": "47911062b00d87f215ba61c41965faf8a7f4b7f7", + "tree": "74d751a78d118a168879013721b131c35277e9d6", + "code_license": "AGPL-3.0-or-later", + "files": [ + { + "path": "crates/argand-atomic/Cargo.toml", + "source_path": "engine/crates/argand-atomic/Cargo.toml", + "sha256": "f2acc19a4c8dbfff9fa4a0bffca01118b4be63463a0d18299fea73932084524f", + "git_blob": "0a1bf7d31ce3a08e06e6b1432fa440d4dc91323f" + }, + { + "path": "crates/argand-atomic/src/lib.rs", + "source_path": "engine/crates/argand-atomic/src/lib.rs", + "sha256": "014bfe5150d612ecc23ae8317c38364e551b04504620d6860c7a6ec77a03b3a6", + "git_blob": "c1db8f325a67e5d3602d914950250c3bfd7d59b2" + }, + { + "path": "crates/argand-site-registry/.gitignore", + "source_path": "engine/crates/argand-site-registry/.gitignore", + "sha256": "b8ccd3639c8520f0275cf0a1d63dbbf3f760f90dfccd17283f7afb9c566ce1ae", + "git_blob": "eec1fa5b69945e3c3501cacddfaa63058d7c368c" + }, + { + "path": "crates/argand-site-registry/Cargo.toml", + "source_path": "engine/crates/argand-site-registry/Cargo.toml", + "sha256": "b072b76192c1d924865b3bbfcf0765987b3af6f9f9736b0b557fd007d7c22f65", + "git_blob": "0842be5251f5f8d3bb339867df5177cb4c60c561" + }, + { + "path": "crates/argand-site-registry/LICENSE_SOURCES.md", + "source_path": "engine/crates/argand-site-registry/LICENSE_SOURCES.md", + "sha256": "d2200768b03382cc77a6fe69a9197f0e066d29ae26ed3117753a908faae7a6cd", + "git_blob": "057a83991c559b4b1f58817f1d8553583c9a206c" + }, + { + "path": "crates/argand-site-registry/README.md", + "source_path": "engine/crates/argand-site-registry/README.md", + "sha256": "0c8a76bd5fdb35123b86a179c7e2a04c9d03fee016145f9a7323915209f0d61e", + "git_blob": "d5554d1df94710a6f1281b176018a421bdb40815" + }, + { + "path": "crates/argand-site-registry/examples/argand-site-registry.service", + "source_path": "engine/crates/argand-site-registry/examples/argand-site-registry.service", + "sha256": "f9a82191df092ebed18f3109062ec6e32d00a2834722fb6e800088b0c7f8b6b2", + "git_blob": "e7ae9ad21c35a0fae5666a4938ba1dd12b7c1982" + }, + { + "path": "crates/argand-site-registry/examples/argand-site-registry.timer", + "source_path": "engine/crates/argand-site-registry/examples/argand-site-registry.timer", + "sha256": "82d696dc0e01d68275214458e90395a4a4a2bfcb9e76daaf9580783c53fd4bc3", + "git_blob": "d6014854586ff05bfb222f88f0a53d14c6133505" + }, + { + "path": "crates/argand-site-registry/examples/update.toml", + "source_path": "engine/crates/argand-site-registry/examples/update.toml", + "sha256": "bfdf4542fb5d534e3d2f2367c7475ff1d021e718c88b97fd2e3952b44cec4546", + "git_blob": "94a31cf9c92e5590241140d37a9007364f209261" + }, + { + "path": "crates/argand-site-registry/migrations/001.sql", + "source_path": "engine/crates/argand-site-registry/migrations/001.sql", + "sha256": "031c62b7e552b7f5156f372a17a9e6e817973a4ab532035832bf2679e01f63f3", + "git_blob": "48e67f949c101c2ea0fc5d086450f6c071649eab" + }, + { + "path": "crates/argand-site-registry/src/adapters/csv_sources.rs", + "source_path": "engine/crates/argand-site-registry/src/adapters/csv_sources.rs", + "sha256": "68cff89dd144f53e594cdf6690f552518715bd560c3bf267e8b5fd213716875d", + "git_blob": "35ecfbce8b1ebc077f4c19c44764e034d9727d64" + }, + { + "path": "crates/argand-site-registry/src/adapters/curlie.rs", + "source_path": "engine/crates/argand-site-registry/src/adapters/curlie.rs", + "sha256": "175ad21374d6786ea0d2a835d2f4c1c53ee93a0107d9fb73b22c3696fd8cf170", + "git_blob": "d2ac16773e574ddd5be75fe4206a0dfff5f680a7" + }, + { + "path": "crates/argand-site-registry/src/adapters/mod.rs", + "source_path": "engine/crates/argand-site-registry/src/adapters/mod.rs", + "sha256": "7ad554b3794aa47af0fbe89e557808e375fd90421c701000d6bcf68589c0016f", + "git_blob": "7c900761c0ad602cc455419a62fcf86846425224" + }, + { + "path": "crates/argand-site-registry/src/adapters/wikidata.rs", + "source_path": "engine/crates/argand-site-registry/src/adapters/wikidata.rs", + "sha256": "ad5ce312b881857227d3dfdc18a7633969cd63d7d191c7b1c97cfba59a8d2ad0", + "git_blob": "bbc151f0ed5dac12efc3320ce45667046d84d7e6" + }, + { + "path": "crates/argand-site-registry/src/build.rs", + "source_path": "engine/crates/argand-site-registry/src/build.rs", + "sha256": "e1381e13a9d32afffde27a56dc6ee44b658208ea2051b4fb62052594160754dc", + "git_blob": "081c7151e6d293cb1be72051ff50c690ddcb1093" + }, + { + "path": "crates/argand-site-registry/src/cli.rs", + "source_path": "engine/crates/argand-site-registry/src/cli.rs", + "sha256": "d64b45e4735e027bb1e3ebe6f3f1d2fabc90820680349fd919bcb69b7ecc99b4", + "git_blob": "45b11676ca3c003f18369bc3fb9cfcb0bcd6a7ec" + }, + { + "path": "crates/argand-site-registry/src/crux.rs", + "source_path": "engine/crates/argand-site-registry/src/crux.rs", + "sha256": "c2d7aa2937e62defdeef629caa58492f935e658042353ad628be3625829e60b9", + "git_blob": "00248edf4a1c75aa24bb36475c4201a2b8e53158" + }, + { + "path": "crates/argand-site-registry/src/download.rs", + "source_path": "engine/crates/argand-site-registry/src/download.rs", + "sha256": "b9c0593b274a23b887d2f62cd7e79758e4e45bb68dc30399eeb701a47e65f5d7", + "git_blob": "e0f0083d4c63a6a1677fa844a9e238a7af728d1b" + }, + { + "path": "crates/argand-site-registry/src/evidence.rs", + "source_path": "engine/crates/argand-site-registry/src/evidence.rs", + "sha256": "d990b9a5bc8a04d39b8453bac97699ed607a9284978f2cb5c08c834bcb1dc543", + "git_blob": "e057b51fec039edc21a65f794298c36e536b5df5" + }, + { + "path": "crates/argand-site-registry/src/identity.rs", + "source_path": "engine/crates/argand-site-registry/src/identity.rs", + "sha256": "336b541524fef60aaf14df3d2854ed2270f0e6afdf49615b539cea4add96257e", + "git_blob": "c87e52046723015406552c38939c7d2ba17a6211" + }, + { + "path": "crates/argand-site-registry/src/json.rs", + "source_path": "engine/crates/argand-site-registry/src/json.rs", + "sha256": "9b000aa647e1e71aef6f6050ee451089ca6717e8ea839789833215a60a17ad21", + "git_blob": "5dccf4dc98ba7ab0eb91ca9c7fb54836abcad85d" + }, + { + "path": "crates/argand-site-registry/src/lib.rs", + "source_path": "engine/crates/argand-site-registry/src/lib.rs", + "sha256": "20b4ae1f793c58701659e7c3d7ecd7f3c7e77407ab3a0f6f2247b04f08bd30e7", + "git_blob": "d327d778aef7efe39f4ec69f286c94d417bc7aca" + }, + { + "path": "crates/argand-site-registry/src/main.rs", + "source_path": "engine/crates/argand-site-registry/src/main.rs", + "sha256": "ff83c5a169cf963cfadf1d8806cf6796e7f38e194a342b1d02fd0a256cf34c70", + "git_blob": "94a12e3639cc5c6bc69ff6431d125166680595a7" + }, + { + "path": "crates/argand-site-registry/src/model.rs", + "source_path": "engine/crates/argand-site-registry/src/model.rs", + "sha256": "6def11209a141cc1bfcf00d1b2fbb098f286cfe8ba78802dedb8f68927a974f2", + "git_blob": "62b34cc8e3b3153376a7401057e8460879348df7" + }, + { + "path": "crates/argand-site-registry/src/normalize.rs", + "source_path": "engine/crates/argand-site-registry/src/normalize.rs", + "sha256": "3fa4436ef23e45e4528015ac45ec5aa7ee09e729c3312b2575738988a741bf2c", + "git_blob": "171e156addcd75f8b80a1e38c3cac6bfb5adb17a" + }, + { + "path": "crates/argand-site-registry/src/observation.rs", + "source_path": "engine/crates/argand-site-registry/src/observation.rs", + "sha256": "21198045ccb07f2539039263226e6425aa71972345f1b9d34e31c2eed1a82b94", + "git_blob": "c3c54090b34747b3fb53bd2ce54c92a8d04d2076" + }, + { + "path": "crates/argand-site-registry/src/query.rs", + "source_path": "engine/crates/argand-site-registry/src/query.rs", + "sha256": "c4dab3151148925c67b86d88e1fa23b924d520faa4b06cbacffde5ba20a52513", + "git_blob": "d31f2980ec174633a2d27caef772a8df1660a079" + }, + { + "path": "crates/argand-site-registry/src/release.rs", + "source_path": "engine/crates/argand-site-registry/src/release.rs", + "sha256": "4215448520f007b0cc41597eaef3b6f403035a4a36eab11b8f54688a5d355583", + "git_blob": "ad15b4faa56b22e3fa16e5ff37699cf11a31fe0f" + }, + { + "path": "crates/argand-site-registry/src/review.rs", + "source_path": "engine/crates/argand-site-registry/src/review.rs", + "sha256": "bc1f160d36de9ce4bc1215a5befb9d0f34feb021fb8892ee3f27d04e6c2d331a", + "git_blob": "8480ebd2d36d26e72a43ddf2c355e8bcd834956f" + }, + { + "path": "crates/argand-site-registry/src/store.rs", + "source_path": "engine/crates/argand-site-registry/src/store.rs", + "sha256": "8333fdc9ffacb2fdf4ce9291a2337726a3a1c912afbae8c20a2c3942f6a63963", + "git_blob": "d93a41924cbc013e2d24ddea86ab4bad6ab103d1" + }, + { + "path": "crates/argand-site-registry/src/update.rs", + "source_path": "engine/crates/argand-site-registry/src/update.rs", + "sha256": "043902ee10dc0a23c9ca26b8d327fdb5598868c66a180680b1eb499fcd15b9c0", + "git_blob": "f4ce455058152b6051430b21ed5219c4e865fd17" + }, + { + "path": "crates/argand-site-registry/tests/cli.rs", + "source_path": "engine/crates/argand-site-registry/tests/cli.rs", + "sha256": "71911d48f5856ad185ac6f563aee835a6bab8363b540199d2be25384ef6ee1fd", + "git_blob": "21dd87dfe9dd49e8e17a3b0555e68ef0ece0d4a8" + }, + { + "path": "crates/argand-site-registry/tests/common/mod.rs", + "source_path": "engine/crates/argand-site-registry/tests/common/mod.rs", + "sha256": "6c0c013e96155745d6a9daa85629e70f3670f9b25a4faa6d7c11fc3cfb6eaa60", + "git_blob": "9ec6c5440d688b3ed8bde4cdeaa0d23013dbc9fe" + }, + { + "path": "crates/argand-site-registry/tests/failures.rs", + "source_path": "engine/crates/argand-site-registry/tests/failures.rs", + "sha256": "8c432147cc28b10f83dc199dfcbed00556ce98fe30ce052ddd83da7d1b3de629", + "git_blob": "6849c51a6395113a7fcd3198a5348e6ec1bca775" + }, + { + "path": "crates/argand-site-registry/tests/identity.rs", + "source_path": "engine/crates/argand-site-registry/tests/identity.rs", + "sha256": "19c90f05c4fe5ed584e5b09379052492784bc1ebb290ddccc245429ee7839db0", + "git_blob": "e65f7c79b3cbb1b34f6f99b08efafe03232228a3" + }, + { + "path": "crates/argand-site-registry/tests/registry.rs", + "source_path": "engine/crates/argand-site-registry/tests/registry.rs", + "sha256": "36bbf77a316635990a0f54a3c9d6fbde074417d7036a587878ea06dfba6c6108", + "git_blob": "32740a65012a8678a526cf3aff2086699f4d5861" + } + ], + "workspace_manifest_sha256": "ba992b68ced3bbcd8978832e2df6d3c67b192e36263edc3dee3ab03611a2fe71", + "workspace_lock_sha256": "bdc5f164f19c60a8b1453b6d2e0548c6d03e911abf9270112a53bf5b5b3a5ee5", + "note": "Initial extraction from a verified signed commit; file hashes describe that immutable baseline, not subsequent standalone changes." +} diff --git a/crates/argand-atomic/Cargo.toml b/crates/argand-atomic/Cargo.toml new file mode 100644 index 0000000..0a1bf7d --- /dev/null +++ b/crates/argand-atomic/Cargo.toml @@ -0,0 +1,16 @@ +# engine/crates/argand-atomic/Cargo.toml +# By Nic Weyand! + +[package] +name = "argand-atomic" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/argand-atomic/src/lib.rs b/crates/argand-atomic/src/lib.rs new file mode 100644 index 0000000..c1db8f3 --- /dev/null +++ b/crates/argand-atomic/src/lib.rs @@ -0,0 +1,211 @@ +// engine/crates/argand-atomic/src/lib.rs +// By Nic Weyand! + +//! Power-loss-durable atomic publication of small files. + +use std::{ + fs::{File, OpenOptions}, + io::{self, Write}, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, +}; + +static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +/// Replaces `path` atomically after its new contents and parent entry are durable. +/// +/// A process-unique sibling prevents concurrent writers from corrupting each +/// other's temporary file. Publication remains last-writer-wins. +/// +/// # Errors +/// +/// Returns the underlying filesystem error when the temporary file cannot be +/// created, written, synchronized, renamed, or made durable in its directory. +pub fn replace_durable(path: &Path, bytes: &[u8]) -> io::Result<()> { + replace_durable_with(path, |file| file.write_all(bytes)) +} + +/// Replaces `path` atomically with contents written incrementally by `write`. +/// +/// This preserves the same power-loss boundary as [`replace_durable`] without +/// requiring a large generated artifact to exist twice in memory. +/// +/// # Errors +/// +/// Returns the writer's error or an underlying filesystem error when the +/// temporary file cannot be created, synchronized, renamed, or made durable. +pub fn replace_durable_with( + path: &Path, + write: impl FnOnce(&mut File) -> io::Result<()>, +) -> io::Result<()> { + let temporary = temporary_sibling(path); + let mut guard = TemporaryFile::create(&temporary)?; + let file = guard + .file + .as_mut() + .ok_or_else(|| io::Error::other("atomic publication temporary file closed unexpectedly"))?; + write(file)?; + file.sync_all()?; + drop(guard.file.take()); + std::fs::rename(&temporary, path)?; + guard.published = true; + sync_parent(path) +} + +/// Creates `path` atomically after its contents are durable, refusing replacement. +/// +/// # Errors +/// +/// Returns [`io::ErrorKind::AlreadyExists`] if `path` already exists. Other +/// errors describe temporary-file creation, writing, synchronization, +/// publication, or parent-directory synchronization failures. +pub fn create_durable(path: &Path, bytes: &[u8]) -> io::Result<()> { + create_durable_with(path, |file| file.write_all(bytes)) +} + +/// Creates `path` atomically from incrementally written contents, without clobbering. +/// +/// The complete synchronized sibling is published with a hard link, so readers +/// can never observe partial contents and an existing destination wins. +/// +/// # Errors +/// +/// Returns [`io::ErrorKind::AlreadyExists`] if `path` already exists. Other +/// errors describe temporary-file creation, writing, synchronization, +/// publication, or parent-directory synchronization failures. +pub fn create_durable_with( + path: &Path, + write: impl FnOnce(&mut File) -> io::Result<()>, +) -> io::Result<()> { + let temporary = temporary_sibling(path); + let mut guard = TemporaryFile::create(&temporary)?; + let file = guard + .file + .as_mut() + .ok_or_else(|| io::Error::other("atomic publication temporary file closed unexpectedly"))?; + write(file)?; + file.sync_all()?; + drop(guard.file.take()); + std::fs::hard_link(&temporary, path)?; + std::fs::remove_file(&temporary)?; + guard.published = true; + sync_parent(path) +} + +fn temporary_sibling(path: &Path) -> PathBuf { + let sequence = TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let mut name = path.file_name().unwrap_or_default().to_os_string(); + name.push(format!(".{}.{}.tmp", std::process::id(), sequence)); + path.with_file_name(name) +} + +fn sync_parent(path: &Path) -> io::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + File::open(parent)?.sync_all() +} + +struct TemporaryFile { + file: Option, + path: PathBuf, + published: bool, +} + +impl TemporaryFile { + fn create(path: &Path) -> io::Result { + let file = OpenOptions::new() // atomic-writes: allow canonical atomic publication primitive + .create_new(true) + .write(true) + .open(path)?; + Ok(Self { + file: Some(file), + path: path.to_owned(), + published: false, + }) + } +} + +impl Drop for TemporaryFile { + fn drop(&mut self) { + if !self.published { + let _ = std::fs::remove_file(&self.path); + } + } +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use super::{create_durable, create_durable_with, replace_durable, replace_durable_with}; + + #[test] + fn creates_complete_contents_without_clobbering() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let destination = directory.path().join("receipt.json"); + + create_durable(&destination, b"first")?; + let error = match create_durable(&destination, b"second") { + Ok(()) => return Err(std::io::Error::other("existing destination was clobbered")), + Err(error) => error, + }; + + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + assert_eq!(std::fs::read(&destination)?, b"first"); + assert_eq!(std::fs::read_dir(directory.path())?.count(), 1); + Ok(()) + } + + #[test] + fn failed_create_removes_unpublished_temporary_file() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let destination = directory.path().join("receipt.json"); + + let error = match create_durable_with(&destination, |file| { + file.write_all(b"partial")?; + Err(std::io::Error::other("fixture failure")) + }) { + Ok(()) => { + return Err(std::io::Error::other( + "fixture writer unexpectedly succeeded", + )); + } + Err(error) => error, + }; + + assert_eq!(error.kind(), std::io::ErrorKind::Other); + assert!(!destination.exists()); + assert_eq!(std::fs::read_dir(directory.path())?.count(), 0); + Ok(()) + } + + #[test] + fn replaces_complete_contents_without_temporary_debris() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let destination = directory.path().join("checkpoint.json"); + + replace_durable(&destination, b"old")?; + replace_durable(&destination, b"complete-new-value")?; + + assert_eq!(std::fs::read(&destination)?, b"complete-new-value"); + assert_eq!(std::fs::read_dir(directory.path())?.count(), 1); + Ok(()) + } + + #[test] + fn streams_complete_contents_without_temporary_debris() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let destination = directory.path().join("streamed"); + + replace_durable_with(&destination, |file| { + file.write_all(b"first-")?; + file.write_all(b"second") + })?; + + assert_eq!(std::fs::read(&destination)?, b"first-second"); + assert_eq!(std::fs::read_dir(directory.path())?.count(), 1); + Ok(()) + } +} diff --git a/crates/argand-site-registry/.gitignore b/crates/argand-site-registry/.gitignore new file mode 100644 index 0000000..eec1fa5 --- /dev/null +++ b/crates/argand-site-registry/.gitignore @@ -0,0 +1,8 @@ +# By Nic Weyand! Local source bytes, databases, generations and credentials. +/data/ +/cache/ +/generations/ +*.sqlite +*.sqlite-* +*.part +*.sig diff --git a/crates/argand-site-registry/Cargo.toml b/crates/argand-site-registry/Cargo.toml new file mode 100644 index 0000000..0842be5 --- /dev/null +++ b/crates/argand-site-registry/Cargo.toml @@ -0,0 +1,35 @@ +# By Nic Weyand! +[package] +name = "argand-site-registry" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow.workspace = true +argand-atomic = { path = "../argand-atomic" } +bzip2 = "0.6.1" +chrono.workspace = true +clap.workspace = true +csv = "1.4.0" +flate2.workspace = true +publicsuffix = "=2.3.0" +reqwest.workspace = true +rusqlite = { version = "=0.40.2", features = ["bundled"] } +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +tar = "0.4.46" +tokio.workspace = true +toml.workspace = true +unicode-normalization.workspace = true +url.workspace = true + +[dev-dependencies] +tempfile.workspace = true +http.workspace = true + +[lints] +workspace = true diff --git a/crates/argand-site-registry/LICENSE_SOURCES.md b/crates/argand-site-registry/LICENSE_SOURCES.md new file mode 100644 index 0000000..057a839 --- /dev/null +++ b/crates/argand-site-registry/LICENSE_SOURCES.md @@ -0,0 +1,60 @@ +# Site Registry source licenses + +Reviewed against the primary distribution and licensing pages on 2026-09-12. +The Rust code uses the workspace's **AGPL-3.0-or-later** license. Imported data keeps +its own licenses; neither the code license nor a merged export relicenses it. +Commercial reuse is supported subject to the following obligations. A provider's +listing is evidence of an assertion, not a guarantee of ownership or safety. + +| Source | Exact data license and evidence | Distribution and consumed fields | +| --- | --- | --- | +| Wikidata | [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/), [`CC0-1.0`](https://www.wikidata.org/wiki/Wikidata:Licensing) | [JSON dumps](https://www.wikidata.org/wiki/Wikidata:Database_download), [entity JSON](https://www.wikidata.org/wiki/Special:EntityData/Q355.json). Entity ID, revision, all labels/aliases, full P856 statements including ranks, qualifiers and references; P17, P159, P407, P1001 and country/language code mappings P297/P218/P219/P220. Relevant raw entity records remain available for audit. | +| Majestic Million | [CC BY 3.0 Unported](https://creativecommons.org/licenses/by/3.0/), [`CC-BY-3.0`](https://majestic.com/reports/majestic-million) | [Official CSV](https://downloads.majestic.com/majestic_million.csv). Domain/IDN/TLD, global/TLD ranks, referring subnet/IP counts and their previous values. Ranks remain source-specific signals; no entity ownership is inferred. | +| Chrome UX Report (CrUX), Google | [CC BY 4.0 International](https://creativecommons.org/licenses/by/4.0/), [`CC-BY-4.0`](https://developer.chrome.com/docs/crux/methodology) | [Monthly BigQuery dataset](https://developer.chrome.com/docs/crux/bigquery/): `origin`, `experimental.popularity.rank`, observation month, optional audience-country dataset code. The adapter produces `origin,rank,yyyymm,country_code` CSV. Rank is a coarse bucket, not a precise visit count. Audience country is not website jurisdiction. No API key or OAuth token is retained. | +| Curlie | [CC BY 3.0 Unported](https://creativecommons.org/licenses/by/3.0/), [`CC-BY-3.0`](https://curlie.org/docs/en/license.html), including the attribution placement prescribed on that page | [Format documentation](https://curlie.org/docs/en/rdf.html), [official download redirect](https://curlie.org/directory-dl), currently [Passau-hosted archive](https://share.innkube.fim.uni-passau.de/curlie-rdf/curlie-rdf-all.tar.gz). Despite its RDF name, the current archive contains **literal TSV**. Content: URL, title, description, category ID. Structure: category ID, full category path, entry count, description, latitude, longitude. Archive notices are retained. | +| Public Suffix List contributors | [Mozilla Public License 2.0](https://mozilla.org/MPL/2.0/), [`MPL-2.0`](https://publicsuffix.org/list/public_suffix_list.dat) | [Official list](https://publicsuffix.org/list/public_suffix_list.dat). All ICANN and PRIVATE rules, wildcard/exception rules, version/commit comments and notices. Used for hostname, registrable-domain and public-suffix derivations. Download at most once per day. | + +## Attribution and distribution + +* **Wikidata:** CC0 imposes no attribution condition. Keep Wikidata IDs, source + links and revision evidence for traceability. CC0 structured data does not + extend to unrelated Wikipedia prose, images or linked websites. +* **Majestic:** credit “Majestic Million, Majestic”, link the source and CC BY 3.0, + retain supplied notices, and identify Argand's changes. The current distribution + page governs this import; older blog posts describe different historic terms. +* **CrUX:** credit “Chrome UX Report, Google”, link the source and CC BY 4.0, + retain notices and indicate the projection/normalization. BigQuery access and + billing are separate from the data license. This adapter requires an explicit + project and positive maximum-bytes-billed limit. +* **Curlie:** the requirement applies to **names, categories and descriptions**. + Every public page using Curlie content must include its prescribed HTML credit: + + ```html +
+ With content from Curlie.org - the largest human-edited directory of the web. Contribute by submitting a website or becoming an editor. +
+ ``` + + Also retain the source and license links and indicate modifications. Imports + retain descriptions as audit data. Lookup/resolve never expose descriptions; + JSONL export omits them by default. `--include-descriptions` is an explicit + distribution choice: receiving applications must satisfy these obligations + before displaying the text. Treat directory descriptions as untrusted text, + never as executable HTML. A JSON attribution object alone does not satisfy + Curlie's public-page placement requirement. +* **PSL:** retain the list's notices, MPL license and access to its source form + when redistributing it. Generations retain the unmodified list in `records` + and `facts`; exports include the PSL fact and original download locator. + Changes to covered PSL source files must remain available under MPL 2.0. + The Rust `publicsuffix` parser is MIT/Apache-2.0; that is separate from the list. + +Every generation contains this document and `ATTRIBUTION.json`, both hash-bound +by its signed completion receipt. Exports carry source manifests, licenses, +retrieval times, assertion selectors and confidence, plus a required attribution +envelope. Consumers must preserve these when extracting subsets or redistributing +derived facts. Source credits do not imply endorsement by any provider. + +Cloudflare Radar, default Tranco, Cisco Umbrella, arbitrary mirrors and any other +unreviewed source are not supported. Adding a source requires verified commercial +reuse rights, its own adapter, provenance and attribution policy. A source format +or distribution-host change fails validation until the adapter is reviewed. diff --git a/crates/argand-site-registry/README.md b/crates/argand-site-registry/README.md new file mode 100644 index 0000000..2a4c5e4 --- /dev/null +++ b/crates/argand-site-registry/README.md @@ -0,0 +1,348 @@ +# Argand Site Registry + +A Rust library and CLI for an entity ↔ website/domain dataset. SQLite stores +source assertions separately and builds indexed, immutable registry generations. +The code follows the engine workspace's **AGPL-3.0-or-later** license; see the +[GNU AGPL](https://www.gnu.org/licenses/agpl-3.0.html). Data licenses and required +credits are in [LICENSE_SOURCES.md](LICENSE_SOURCES.md). + +`facebook → Facebook → facebook.com` is a name-to-entity-to-registrable-domain +lookup. The actual retained Wikidata destination is `https://www.facebook.com/`; +normalization does not silently replace it with an apex URL. A mobile website is +a separate property. The current retained Amazon entity, Q3884, has 13 P856 +properties, including `amazon.com`, `amazon.co.uk`, and `amazon.de`. + +Imported assertions enter the review queue. Only explicit, unexpired reviews +can produce a `resolve` destination. Automatic updates build candidates; signing +and activation are separate operator actions. This crate is a dataset component; +existing Argand Navigate policy and collection admission still apply when a +consumer integrates it into public search. + +## Install and run the offline acceptance example + +Rust 1.97+ and OpenSSH (`ssh-keygen`) are required. From the standalone repository root: + +```bash +cargo install --path crates/argand-site-registry --locked +argand-site-registry --help +cargo test -p argand-site-registry --all-targets --locked --offline +``` + +The native CLI test imports small source-shaped fixtures for **all five sources**, +repeats the imports, resolves aliases, signs and activates an approved generation, +revokes the destination, and rejects rollback past the revocation. Synthetic +fixtures are authored in Rust test code; no provider datasets or signing keys +are committed. To retain a local example for inspection, choose a **new** path: + +```bash +ARGAND_REGISTRY_E2E_OUTPUT=/tmp/argand-site-registry-example \ + cargo test -p argand-site-registry --test cli --locked --offline -- --nocapture +``` + +The example contains `candidate/`, `approved/`, `revoked/`, JSON manifests, an +attributed export and a disposable test key. Do not use that test key or those +synthetic approvals for a real release. + +## Acquire and import sources + +All paths are explicit. These commands use `jq` only to read CLI JSON output. +They create data outside the checkout. Byte caps are upper bounds, not estimates +of current source sizes. Increase a cap only after checking available storage. + +```bash +export ARGAND_SITE_DATA="$HOME/.local/share/argand-site-registry" +mkdir -p "$ARGAND_SITE_DATA" + +argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \ + --source psl --format psl-text \ + --url https://publicsuffix.org/list/public_suffix_list.dat \ + --snapshot "$(date -u +%F)" --scope full --maximum-bytes 1000000 \ + > "$ARGAND_SITE_DATA/psl-download.json" + +argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \ + --source wikidata --format wikidata-entities \ + --url 'https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q355%7CQ3884&format=json&maxlag=5' \ + --snapshot "$(date -u +%F)" --scope selection:facebook-amazon \ + --maximum-bytes 5000000 > "$ARGAND_SITE_DATA/wikidata-download.json" + +argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \ + --source majestic --format majestic-csv \ + --url https://downloads.majestic.com/majestic_million.csv \ + --snapshot "$(date -u +%F)" --scope full --maximum-bytes 250000000 \ + > "$ARGAND_SITE_DATA/majestic-download.json" + +argand-site-registry download --cache "$ARGAND_SITE_DATA/cache" \ + --source curlie --format curlie-tar-gz \ + --url https://curlie.org/directory-dl \ + --snapshot "$(date -u +%F)" --scope full --maximum-bytes 1000000000 \ + > "$ARGAND_SITE_DATA/curlie-download.json" + +for source in psl wikidata majestic curlie; do + argand-site-registry import --database "$ARGAND_SITE_DATA/import.sqlite" \ + --input "$(jq -r .input "$ARGAND_SITE_DATA/$source-download.json")" \ + --manifest "$(jq -r .manifest "$ARGAND_SITE_DATA/$source-download.json")" +done +``` + +For a full Wikidata dump, select a real dump URL from the official +[download index](https://dumps.wikimedia.org/wikidatawiki/entities/), then use +`--format wikidata-dump --compression gzip` (or `bzip2`) and `--scope full`. +The parser handles the documented one-entity-per-line JSON array and concatenated +compressed streams. Do not use truthy RDF: it loses statement evidence. Full +dumps need substantial disk space and a long sequential scan even though memory +is bounded. A small entity selection is useful on limited hardware. + +To reuse an already acquired file, retain its **original** retrieval time, +source URL and snapshot/revision. First verify its acquisition receipt, then: + +```bash +argand-site-registry manifest --input /data/Q355.json \ + --output /data/Q355.source.json --source wikidata --format wikidata-entities \ + --source-url https://www.wikidata.org/wiki/Special:EntityData/Q355.json \ + --snapshot retained-Q355-revision --scope selection:Q355 \ + --retrieved-at 2026-09-10T13:40:20.446514Z +argand-site-registry import --database "$ARGAND_SITE_DATA/import.sqlite" \ + --input /data/Q355.json --manifest /data/Q355.source.json +``` + +Replace paths, snapshot and time with the actual acquisition details. A manifest +declares provenance; making one does not authenticate arbitrary file contents. + +### CrUX + +The adapter queries the documented monthly BigQuery table and streams paginated +results into this exact CSV projection: + +```sql +SELECT DISTINCT origin, experimental.popularity.rank AS rank, + '202608' AS yyyymm, '' AS country_code +FROM `chrome-ux-report.all.202608` +WHERE experimental.popularity.rank IS NOT NULL +ORDER BY origin, rank +``` + +The month above is an example of the documented table naming. Confirm that the +desired month exists. For an audience-country dataset, use `country: "GB"` in +the request; the adapter selects `chrome-ux-report.country_gb.202608` and emits +`GB`. The rank is a bucket; do not mix it numerically with Majestic's exact rank. + +Create `crux-request.json` with your project and explicit limits: + +```json +{ + "project": "your-billing-project", + "month": "202608", + "country": null, + "maximum_bytes_billed": 1000000000, + "maximum_output_bytes": 500000000 +} +``` + +Supply an authorized OAuth access token through `GOOGLE_OAUTH_ACCESS_TOKEN` +using your credential manager, then run: + +```bash +argand-site-registry crux-download --cache "$ARGAND_SITE_DATA/cache" \ + --request crux-request.json > "$ARGAND_SITE_DATA/crux-download.json" +argand-site-registry import --database "$ARGAND_SITE_DATA/import.sqlite" \ + --input "$(jq -r .input "$ARGAND_SITE_DATA/crux-download.json")" \ + --manifest "$(jq -r .manifest "$ARGAND_SITE_DATA/crux-download.json")" +``` + +No default billing project or unbounded query is provided. An interrupted job +reuses its content-derived BigQuery job ID; result pages replay from the same +query result. Keep `job.json` with the acquisition records. Expired server results +require an operator to inspect the existing job. Pinned local exports of the +exact CSV projection can instead use `manifest --source crux --format crux-csv +--source-url https://developer.chrome.com/docs/crux/bigquery/` with their actual +retrieval time, query/snapshot identity and appropriate `monthly:YYYYMM:country` +scope. The token is never written into a manifest. + +## Build, look up and review + +```bash +argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \ + --output "$ARGAND_SITE_DATA/generation-1" > "$ARGAND_SITE_DATA/build-1.json" +export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-1.json")" +argand-site-registry lookup --generation "$ARGAND_SITE_DATA/generation-1" \ + --pin "$ARGAND_SITE_PIN" --query facebook +argand-site-registry lookup --generation "$ARGAND_SITE_DATA/generation-1" \ + --pin "$ARGAND_SITE_PIN" --query amazon --limit 100 +``` + +The real-source acceptance run produced: + +| Query | Entity | Example properties | Registrable domains | +| --- | --- | --- | --- | +| `facebook` | Facebook (Q355) | `https://www.facebook.com/`, `https://m.facebook.com/` | `facebook.com` | +| `amazon` | Amazon (Q3884) | `https://www.amazon.com/`, `https://www.amazon.co.uk/`, `https://www.amazon.de/` | `amazon.com`, `amazon.co.uk`, `amazon.de` | + +These properties were asserted on the **same Wikidata entity**. Hostname +resemblance did not establish the relationship. Imported qualifiers remain in +`evidence` and `property_scopes`; unknown locale/country remains null. Names, +aliases and entity metadata carry their own fact-level source declarations. +Regional locale/country and role are explicit reviewed assertions. They are +separate from entity headquarters, ccTLD spelling and CrUX audience country. + +Inspect the full statements, references, names/aliases, hostname spelling and +independent current ownership/role evidence. A review JSON has this shape: + +```json +{ + "fingerprint": "COPY_THE_EXACT_64_CHARACTER_FINGERPRINT_FROM_LOOKUP", + "decision": "approve", + "reviewer": "operator identity", + "reason": "How entity ownership and this exact destination role were verified", + "evidence": "An immutable capture identifier or evidence digest", + "reviewed_at": "2026-09-12T12:00:00Z", + "expires_at": "2026-10-12T12:00:00Z", + "role": "regional", + "locale": "", + "country": "GB" +} +``` + +Replace the example evidence and dates; approvals expire within 90 days. Use +`role: "primary"` for the independently verified default and `country: "DE"` +for a separately verified German regional property. A country-scoped review +can leave locale empty. If both are specified, both must match the request. + +```bash +argand-site-registry review --database "$ARGAND_SITE_DATA/import.sqlite" \ + --generation "$ARGAND_SITE_DATA/generation-1" --pin "$ARGAND_SITE_PIN" \ + --decision review.json +argand-site-registry build --database "$ARGAND_SITE_DATA/import.sqlite" \ + --output "$ARGAND_SITE_DATA/generation-2" > "$ARGAND_SITE_DATA/build-2.json" +export ARGAND_SITE_PIN="$(jq -r .pin "$ARGAND_SITE_DATA/build-2.json")" +argand-site-registry resolve --generation "$ARGAND_SITE_DATA/generation-2" \ + --pin "$ARGAND_SITE_PIN" --query amazon --country GB +``` + +After the corresponding real reviews, GB selects the reviewed UK property; +DE selects the reviewed German property; otherwise an explicitly reviewed +primary may be used. Unknown, expired, tied or entity-ambiguous requests return +`"destination": null`. Result limits never hide ambiguity. Name/alias changes, +changed statements/revisions, URLs or normalization evidence invalidate reviews. +Deprecated, end-dated and non-value statements remain audit evidence and cannot +be admitted. An unchanged PSL file with a new retrieval time preserves reviews. + +## Release, export, update and recovery + +When two providers describe the same navigational entity, `lookup` deliberately +shows both source IDs. Connect them only after reviewing their identities: + +```bash +argand-site-registry equivalence --generation "$ARGAND_SITE_DATA/generation-2" \ + --pin "$ARGAND_SITE_PIN" --left SOURCE_ENTITY_ID --right OTHER_SOURCE_ENTITY_ID +``` + +Use the returned fingerprint in a review JSON with `role: "unspecified"`, empty +locale/country, a reason, immutable identity evidence and an expiry. Then repeat +the command with `--database "$ARGAND_SITE_DATA/import.sqlite" --decision +identity-review.json` and rebuild. `resolve` follows only active, explicitly +reviewed equivalences and includes their provenance. Each destination still +needs its own review. The original IDs, raw ambiguity counts and conflicting +assertions remain visible. Changed names or website assertions invalidate the +identity decision; identity revocations use the same append-only release log. +Operator-authored decisions are published under CC0-1.0, separately from source +data licenses. + +Each generation contains `registry.sqlite`, `LICENSE_SOURCES.md`, +`ATTRIBUTION.json` and a hash-binding `COMPLETE.json`. Distribute all four together. +Keep the import database, review history and cached source bytes for recovery. +The SQLite file includes raw relevant records and descriptions for audit; public +consumers must obey the source attribution requirements. `export` streams +source-bearing JSONL and omits descriptions by default: + +```bash +argand-site-registry export --generation "$ARGAND_SITE_DATA/generation-2" \ + --pin "$ARGAND_SITE_PIN" --output "$ARGAND_SITE_DATA/assertions.jsonl" +argand-site-registry sign --generation "$ARGAND_SITE_DATA/generation-2" \ + --pin "$ARGAND_SITE_PIN" --key /secure/registry-signing-key +argand-site-registry activate --generation "$ARGAND_SITE_DATA/generation-2" \ + --current "$ARGAND_SITE_DATA/current.json" \ + --allowed-signers /secure/registry-allowed-signers --identity registry-publisher +``` + +Use an existing operator-controlled SSH signing key. The external allowed-signers +file follows OpenSSH syntax: `registry-publisher ssh-ed25519 PUBLIC_KEY`. Neither +keys nor the trust file should come from the downloaded dataset. Consumers can +open `Registry::open(path, trusted_receipt_sha256)` once and reuse its indexed +queries, or verify a publisher with `release::verify_signed` first. A hash proves +integrity only relative to a trusted pin. Signature verification authenticates +the publisher, not the truth of a source assertion. + +`diff --old PATH --old-pin HASH --new PATH --new-pin HASH` streams added/removed +edge fingerprints. `verify --generation PATH --pin HASH` checks every artifact +bound by the receipt. To revoke, append a review with `decision: "revoke"`, then +rebuild, sign and activate. Re-activating an older signed generation supports +rollback **only if it retains every distributed revocation**. Otherwise rebuild +the older source selection with the current review log; never edit generations. + +The [update configuration](examples/update.toml) and [systemd service/timer](examples/) +provide weekly candidate refreshes without a resident daemon. Set absolute paths +and an installed executable path. TOML paths do not expand environment variables. +`update --config /etc/argand-site-registry.toml` downloads/imports all configured +sources and builds only after they succeed. The same inputs and review log reuse +the same generation. A nonzero exit is a failed refresh; the active pointer stays +intact. Failed `pending-*` builds can be inspected before explicitly removing +that incomplete directory. Acquisition, import and update operations take local +locks; use one writer and keep old complete generations for rollback. + +Downloads permit only the reviewed HTTPS source endpoints, validate each +redirect, bound bytes and bind range resumes to strong ETags. Chunked/validatorless +responses safely restart on interruption. PSL network attempts are limited to +once per 24 hours per cache. CrUX is opt-in and may use `{previous_month}` in +update configuration; monthly data may not yet be published on the first day. +Wikidata source snapshot labels support `{date}` and `{month}` in scheduled +downloads. No scheduled job signs, approves, renews approvals or activates links. + +## Storage and operating limits + +Migration `migrations/001.sql` owns schema version 1. `sources`, `records` and +`facts` preserve snapshot/native IDs, licenses, retrieval times and confidence; +`reviews` is append-only. Complete source selection is latest retrieval time per +provider/scope, with digest as the deterministic tie break. Use the **same scope** +for a replacement snapshot, and separate scopes for deliberate independent +selections. History and conflicts remain stored. A failed source cannot replace +a complete one. Avoid overlapping full/partial scopes unless both evidences are +intended to remain active. + +Derived tables are `selected_sources`, `entities`, `names`, `properties`, `edges`, +`popularity` and `rejected`. Entity IDs derive from source/native IDs; URL IDs +derive from strict normalized URLs. Equal source entities merge across snapshots +and equal URLs share a property. Explicit `equivalences` connect reviewed +cross-source identities while preserving both IDs. Names are never an identity +join. The canonical label rule prefers labels, then English, +then language/text order. Original labels/aliases are kept. Popularity has its +own source, target, observation period and audience scope and never creates an +ownership edge. All derivations bind input fact IDs, PSL identity and the +`argand.site-rules/v1` contract through their generation receipt. + +Imports use transactions of 256 relevant records with durable replay checkpoints. +Restart replays the compressed stream and skips committed records. Large source +records are capped at 16 MiB; SQLite has an 8 MiB page cache and disk-backed sorts. +Builds stream a canonical sorted copy and never load the full registry into RAM. +Names and entity metadata exposed by lookup are capped at 256 facts each, with +uncapped totals; the complete assertions remain available in the database/export. +Lookup returns at most 100 edges and reports all pre-limit ambiguity counts. +The full store/history and each generation consume disk; there is no automatic +pruning. The original compressed source is hashed before/after import, so expect +extra sequential disk reads. These bounds are not a full-dump throughput claim. + +The `observation` module defines future crawler evidence for redirects, canonical +links, hreflang, JSON-LD sameAs, sitemaps and country selectors, with capture IDs, +hashes, rights and confidence. It does not crawl or automatically infer ownership. + +Source vandalism, compromised publishers and a domain changing ownership cannot +be eliminated by hashes or popularity. Review expiration, exact evidence binding, +signed releases, explicit revocations and conservative abstention contain those +risks. Protect the writer database, signing key and consumer trust configuration. +Do not feed raw `lookup` candidates straight into an automatic redirect consumer. + +If an import fails, fix the input/format or reuse the matching original source +manifest, then rerun the same import. Do not edit digests to make corrupted data +pass. A source host/schema change needs an adapter review. HTTP 403/429 is a +source-access failure; reuse an authorized retained snapshot or retry according +to the provider's policy. A null resolution means evidence/review is missing, +expired or ambiguous; `lookup` explains which assertions are involved. diff --git a/crates/argand-site-registry/examples/argand-site-registry.service b/crates/argand-site-registry/examples/argand-site-registry.service new file mode 100644 index 0000000..e7ae9ad --- /dev/null +++ b/crates/argand-site-registry/examples/argand-site-registry.service @@ -0,0 +1,20 @@ +# By Nic Weyand! Install the executable/config and create this service user first. +[Unit] +Description=Build an Argand Site Registry candidate +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=argand-site-registry +Group=argand-site-registry +StateDirectory=argand-site-registry +ExecStart=/usr/local/bin/argand-site-registry update --config /etc/argand-site-registry.toml +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/argand-site-registry +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +TimeoutStartSec=infinity diff --git a/crates/argand-site-registry/examples/argand-site-registry.timer b/crates/argand-site-registry/examples/argand-site-registry.timer new file mode 100644 index 0000000..d601485 --- /dev/null +++ b/crates/argand-site-registry/examples/argand-site-registry.timer @@ -0,0 +1,12 @@ +# By Nic Weyand! Creates review candidates; does not sign or activate them. +[Unit] +Description=Refresh the Argand Site Registry weekly + +[Timer] +OnCalendar=Sun *-*-* 03:00:00 UTC +RandomizedDelaySec=30m +Persistent=true +Unit=argand-site-registry.service + +[Install] +WantedBy=timers.target diff --git a/crates/argand-site-registry/examples/lookup.rs b/crates/argand-site-registry/examples/lookup.rs new file mode 100644 index 0000000..0ab7843 --- /dev/null +++ b/crates/argand-site-registry/examples/lookup.rs @@ -0,0 +1,24 @@ +// By Nic Weyand! +//! Open an externally pinned registry once and emit the complete lookup envelope. + +use argand_site_registry::query::Registry; +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser)] +struct Args { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + query: String, +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse(); + let registry = Registry::open(&args.generation, &args.pin)?; + let result = registry.lookup(&args.query, 100)?; + println!("{}", serde_json::to_string(&result)?); + Ok(()) +} diff --git a/crates/argand-site-registry/examples/update.toml b/crates/argand-site-registry/examples/update.toml new file mode 100644 index 0000000..94a31cf --- /dev/null +++ b/crates/argand-site-registry/examples/update.toml @@ -0,0 +1,50 @@ +# By Nic Weyand! Operator example; use absolute writable paths. +cache = "/var/lib/argand-site-registry/cache" +database = "/var/lib/argand-site-registry/import.sqlite" +generations = "/var/lib/argand-site-registry/generations" + +[[downloads]] +source = "psl" +format = "psl_text" +url = "https://publicsuffix.org/list/public_suffix_list.dat" +snapshot = "{date}" +scope = "full" +maximum_bytes = 1000000 + +[[downloads]] +source = "wikidata" +format = "wikidata_entities" +url = "https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q355%7CQ3884&format=json&maxlag=5" +snapshot = "{date}" +scope = "selection:facebook-amazon" +maximum_bytes = 5000000 + +[[downloads]] +source = "majestic" +format = "majestic_csv" +url = "https://downloads.majestic.com/majestic_million.csv" +snapshot = "{date}" +scope = "full" +maximum_bytes = 250000000 + +[[downloads]] +source = "curlie" +format = "curlie_tar_gz" +url = "https://curlie.org/directory-dl" +snapshot = "{date}" +scope = "full" +maximum_bytes = 1000000000 + +# Optional pinned acquisitions; repeat [[inputs]] for each source. +# [[inputs]] +# input = "/data/source-object.gz" +# manifest = "/data/source.json" + +# CrUX is deliberately opt-in: provide your authorized project, positive billing +# cap, and GOOGLE_OAUTH_ACCESS_TOKEN through the scheduler credential environment. +# [[crux]] +# project = "your-billing-project" +# month = "{previous_month}" +# country = "GB" +# maximum_bytes_billed = 1000000000 +# maximum_output_bytes = 500000000 diff --git a/crates/argand-site-registry/migrations/001.sql b/crates/argand-site-registry/migrations/001.sql new file mode 100644 index 0000000..48e67f9 --- /dev/null +++ b/crates/argand-site-registry/migrations/001.sql @@ -0,0 +1,43 @@ +-- By Nic Weyand! Immutable assertions; only import progress and append-only review grow. +CREATE TABLE registry_metadata (singleton INTEGER PRIMARY KEY CHECK(singleton=1), rules TEXT NOT NULL) STRICT; +INSERT INTO registry_metadata VALUES(1,'argand.site-rules/v1'); +CREATE TABLE sources ( + id TEXT PRIMARY KEY, source TEXT NOT NULL, scope TEXT NOT NULL, + retrieved_at TEXT NOT NULL, manifest TEXT NOT NULL, + checkpoint INTEGER NOT NULL DEFAULT 0 CHECK(checkpoint >= 0), + complete INTEGER NOT NULL DEFAULT 0 CHECK(complete IN (0,1)) +) STRICT; +CREATE INDEX source_latest ON sources(source,scope,complete,retrieved_at,id); +CREATE TABLE records ( + source_id TEXT NOT NULL REFERENCES sources(id), ordinal INTEGER NOT NULL, + native_id TEXT NOT NULL, raw_json TEXT NOT NULL, + PRIMARY KEY(source_id,ordinal) +) STRICT; +CREATE TABLE facts ( + id TEXT PRIMARY KEY, source_id TEXT NOT NULL, ordinal INTEGER NOT NULL, + subject TEXT NOT NULL, predicate TEXT NOT NULL, value TEXT NOT NULL, + selector TEXT NOT NULL, confidence INTEGER NOT NULL CHECK(confidence BETWEEN 0 AND 10000), + FOREIGN KEY(source_id,ordinal) REFERENCES records(source_id,ordinal) +) STRICT; +CREATE INDEX fact_source ON facts(source_id,ordinal); +CREATE INDEX fact_subject ON facts(subject,predicate); +CREATE TABLE reviews ( + sequence INTEGER PRIMARY KEY, fingerprint TEXT NOT NULL, + decision TEXT NOT NULL CHECK(decision IN ('approve','revoke')), + reviewer TEXT NOT NULL, reason TEXT NOT NULL, evidence TEXT NOT NULL, + reviewed_at TEXT NOT NULL, expires_at TEXT NOT NULL, + role TEXT NOT NULL CHECK(role IN ('primary','regional','unspecified')), + locale TEXT NOT NULL, country TEXT NOT NULL +) STRICT; +CREATE INDEX review_fingerprint ON reviews(fingerprint,sequence DESC); +CREATE TABLE equivalences ( + fingerprint TEXT PRIMARY KEY, left_entity TEXT NOT NULL, right_entity TEXT NOT NULL, + left_signature TEXT NOT NULL, right_signature TEXT NOT NULL +) STRICT; +CREATE INDEX equivalence_left ON equivalences(left_entity); +CREATE INDEX equivalence_right ON equivalences(right_entity); +CREATE TRIGGER equivalence_no_update BEFORE UPDATE ON equivalences BEGIN SELECT RAISE(ABORT,'equivalences are immutable'); END; +CREATE TRIGGER equivalence_no_delete BEFORE DELETE ON equivalences BEGIN SELECT RAISE(ABORT,'equivalences are immutable'); END; +CREATE TRIGGER review_no_update BEFORE UPDATE ON reviews BEGIN SELECT RAISE(ABORT,'reviews are append-only'); END; +CREATE TRIGGER review_no_delete BEFORE DELETE ON reviews BEGIN SELECT RAISE(ABORT,'reviews are append-only'); END; +PRAGMA user_version=1; diff --git a/crates/argand-site-registry/src/adapters/csv_sources.rs b/crates/argand-site-registry/src/adapters/csv_sources.rs new file mode 100644 index 0000000..35ecfbc --- /dev/null +++ b/crates/argand-site-registry/src/adapters/csv_sources.rs @@ -0,0 +1,139 @@ +// By Nic Weyand! +//! Source-specific CSV projections; popularity never creates an entity edge. + +use super::{RecordSink, SourceAdapter, bounded_line}; +use crate::model::{Fact, Record}; +use anyhow::{Context, ensure}; +use serde_json::{Value, json}; +use std::io::BufRead; + +pub(super) struct CsvSource { + pub crux: bool, +} + +impl SourceAdapter for CsvSource { + fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> { + let mut line = String::new(); + ensure!(bounded_line(input, &mut line)? > 0, "empty CSV"); + let headers = parse_line(&line)?; + let required = if self.crux { + vec!["origin", "rank", "yyyymm", "country_code"] + } else { + vec![ + "GlobalRank", + "TldRank", + "Domain", + "TLD", + "RefSubNets", + "RefIPs", + "IDN_Domain", + "IDN_TLD", + "PrevGlobalRank", + "PrevTldRank", + "PrevRefSubNets", + "PrevRefIPs", + ] + }; + ensure!( + headers.iter().map(String::as_str).collect::>() == required, + "CSV schema differs from documented projection" + ); + let mut ordinal = 0; + while bounded_line(input, &mut line)? > 0 { + ensure!(!line.trim().is_empty(), "blank CSV row"); + let cells = parse_line(&line)?; + ensure!(cells.len() == headers.len(), "CSV column count changed"); + let raw: Value = headers + .iter() + .zip(&cells) + .map(|(k, v)| (k.clone(), json!(v))) + .collect(); + let value = if self.crux { + crux(&raw)? + } else { + majestic(&raw)? + }; + ordinal += 1; + let subject = value["target"] + .as_str() + .context("missing popularity target")? + .to_owned(); + sink.emit(Record { + native_id: format!("row:{ordinal}"), + raw, + facts: vec![Fact { + subject, + predicate: "popularity".into(), + value, + selector: format!("row:{ordinal}"), + confidence: 10000, + }], + })?; + } + ensure!(ordinal > 0, "empty CSV dataset"); + Ok(()) + } +} + +fn parse_line(line: &str) -> anyhow::Result> { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(line.as_bytes()); + let row = reader.records().next().context("missing CSV record")??; + ensure!( + reader.records().next().is_none(), + "multiline CSV unsupported by these source contracts" + ); + Ok(row.iter().map(str::to_owned).collect()) +} + +fn integer(raw: &Value, key: &str) -> anyhow::Result { + Ok(raw[key].as_str().context("CSV field missing")?.parse()?) +} + +fn majestic(raw: &Value) -> anyhow::Result { + let rank = integer(raw, "GlobalRank")?; + ensure!(rank > 0, "rank must be positive"); + Ok( + json!({"target":raw["Domain"],"target_kind":"hostname","rank":rank, + "tld_rank":integer(raw,"TldRank")?,"referring_subnets":integer(raw,"RefSubNets")?, + "referring_ips":integer(raw,"RefIPs")?,"previous_global_rank":integer(raw,"PrevGlobalRank")?, + "previous_tld_rank":integer(raw,"PrevTldRank")?,"previous_referring_subnets":integer(raw,"PrevRefSubNets")?, + "previous_referring_ips":integer(raw,"PrevRefIPs")?,"country_code":null,"period":null}), + ) +} + +fn crux(raw: &Value) -> anyhow::Result { + let month = raw["yyyymm"].as_str().context("missing month")?; + validate_month(month)?; + let country = raw["country_code"] + .as_str() + .context("missing audience country")?; + ensure!( + country.is_empty() + || (country.len() == 2 && country.bytes().all(|b| b.is_ascii_alphabetic())), + "invalid CrUX audience country" + ); + let rank = integer(raw, "rank")?; + ensure!(rank > 0, "rank must be positive"); + let url = url::Url::parse(raw["origin"].as_str().context("missing origin")?)?; + ensure!( + url.path() == "/" && url.query().is_none() && url.fragment().is_none(), + "CrUX target must be origin" + ); + Ok( + json!({"target":raw["origin"],"target_kind":"origin","rank":rank,"coarse_rank":true,"period":month,"country_code":country.to_ascii_uppercase()}), + ) +} + +pub(crate) fn validate_month(month: &str) -> anyhow::Result<()> { + ensure!( + month.len() == 6 && month.bytes().all(|b| b.is_ascii_digit()), + "month must be YYYYMM" + ); + ensure!( + (1..=12).contains(&month[4..].parse::()?), + "invalid month" + ); + Ok(()) +} diff --git a/crates/argand-site-registry/src/adapters/curlie.rs b/crates/argand-site-registry/src/adapters/curlie.rs new file mode 100644 index 0000000..d2ac167 --- /dev/null +++ b/crates/argand-site-registry/src/adapters/curlie.rs @@ -0,0 +1,171 @@ +// By Nic Weyand! +//! Curlie v2 (2025-05): unquoted 4-column content and 6-column structure TSV. + +use super::{RecordSink, SourceAdapter, bounded_line}; +use crate::model::{Fact, Record, Source, entity_id}; +use anyhow::{Context, ensure}; +use serde_json::json; +use std::{ + io::{BufRead, BufReader, Read}, + path::Component, +}; + +pub(super) struct Curlie; + +impl SourceAdapter for Curlie { + fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> { + let gzip = flate2::read::MultiGzDecoder::new(input); + let mut archive = tar::Archive::new(gzip); + let mut entries = 0_u64; + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.path()?.into_owned(); + ensure!( + path.components().all(|c| matches!(c, Component::Normal(_))), + "unsafe archive path" + ); + let name = path.to_str().context("non-UTF8 archive member")?.to_owned(); + ensure!( + name.starts_with("curlie-rdf/") || name == "curlie-rdf", + "unexpected Curlie archive root" + ); + if entry.header().entry_type().is_dir() { + continue; + } + ensure!( + entry.header().entry_type().is_file(), + "archive links and special members rejected" + ); + ensure!( + entry.size() <= 4 * 1024 * 1024 * 1024, + "archive member exceeds 4 GiB" + ); + let content = name.ends_with("-c.tsv"); + let structure = name.ends_with("-s.tsv"); + if !content && !structure { + ensure!( + path.extension() + .is_some_and(|e| e.eq_ignore_ascii_case("txt")), + "unrecognized archive member" + ); + let mut text = String::new(); + entry.take(1024 * 1024 + 1).read_to_string(&mut text)?; + ensure!(text.len() <= 1024 * 1024, "Curlie metadata too large"); + sink.emit(Record { + native_id: name, + raw: json!(text), + facts: vec![], + })?; + continue; + } + let mut reader = BufReader::new(&mut entry); + let mut line = String::new(); + let mut ordinal = 0; + while bounded_line(&mut reader, &mut line)? > 0 { + let fields: Vec<&str> = line.trim_end_matches(['\n', '\r']).split('\t').collect(); + ensure!( + fields.len() == if content { 4 } else { 6 }, + "Curlie TSV schema changed: {name}" + ); + ordinal += 1; + entries += 1; + let locator = format!("{name}:{ordinal}"); + let facts = if content { + content_facts(&fields)? + } else { + category_facts(&fields)? + }; + sink.emit(Record { + native_id: locator, + raw: json!(fields), + facts, + })?; + } + } + // Read through the gzip trailer: tar EOF alone must not hide corruption. + let mut gzip = archive.into_inner(); + let mut tail = [0; 8192]; + let mut trailing = 0usize; + loop { + let n = gzip.read(&mut tail)?; + if n == 0 { + break; + } + trailing += n; + ensure!( + trailing <= 1024 * 1024 && tail[..n].iter().all(|b| *b == 0), + "unexpected archive trailing data" + ); + } + ensure!(entries > 0, "Curlie archive has no entries"); + Ok(()) + } +} + +fn content_facts(fields: &[&str]) -> anyhow::Result> { + ensure!(fields[3].parse::()? > 0, "invalid category ID"); + // Curlie identifies a site, not the legal organization operating it. + let subject = entity_id(Source::Curlie, fields[0]); + let values = [ + ( + "name", + json!({"text":fields[1],"language":"und","kind":"label"}), + "column:2", + ), + ( + "website", + json!({"url":fields[0],"relation":"directory_listing","category_id":fields[3],"statement":null}), + "column:1", + ), + ( + "description", + json!({"text":fields[2],"category_id":fields[3]}), + "column:3", + ), + ( + "category_membership", + json!({"category_id":fields[3]}), + "column:4", + ), + ]; + Ok(values + .into_iter() + .map(|(p, v, s)| Fact { + subject: subject.clone(), + predicate: p.into(), + value: v, + selector: s.into(), + confidence: 5000, + }) + .collect()) +} + +fn category_facts(fields: &[&str]) -> anyhow::Result> { + ensure!(fields[0].parse::()? > 0, "invalid category ID"); + let count: u64 = fields[2].parse()?; + let latitude = coordinate(fields[4], 90.0)?; + let longitude = coordinate(fields[5], 180.0)?; + ensure!( + latitude.is_some() == longitude.is_some(), + "incomplete coordinates" + ); + Ok(vec![Fact { + subject: format!("curlie:category:{}", fields[0]), + predicate: "category".into(), + value: json!({"category_id":fields[0],"path":fields[1],"entry_count":count,"description":fields[3],"latitude":latitude,"longitude":longitude}), + selector: "columns:1-6".into(), + confidence: 5000, + }]) +} + +fn coordinate(value: &str, maximum: f64) -> anyhow::Result> { + if value.is_empty() { + return Ok(None); + } + let coordinate: f64 = value.parse()?; + ensure!( + coordinate.is_finite() && coordinate.abs() <= maximum, + "invalid geographic coordinate" + ); + Ok(Some(coordinate)) +} diff --git a/crates/argand-site-registry/src/adapters/mod.rs b/crates/argand-site-registry/src/adapters/mod.rs new file mode 100644 index 0000000..7c90076 --- /dev/null +++ b/crates/argand-site-registry/src/adapters/mod.rs @@ -0,0 +1,76 @@ +// By Nic Weyand! +//! Bounded source-specific readers behind one ingestion interface. + +use crate::model::{Fact, Format, Record}; +use anyhow::ensure; +use serde_json::json; +use std::io::{BufRead, Read}; +pub(crate) mod csv_sources; +mod curlie; +mod wikidata; + +/// Maximum decompressed size of one entity, row, or metadata document. +pub const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; + +/// Transactional sink; an emitted record and all its facts share one checkpoint. +pub trait RecordSink { + /// Persists one source record. + /// + /// # Errors + /// Returns storage or record-validation errors. + fn emit(&mut self, record: Record) -> anyhow::Result<()>; +} + +/// Common streaming import interface. Readers never fetch website assertions. +pub trait SourceAdapter { + /// Reads the complete input, rejecting malformed or truncated records. + /// + /// # Errors + /// Returns parser, input, or sink errors. + fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()>; +} + +/// Chooses the explicit source format adapter. +#[must_use] +pub fn adapter(format: Format) -> Box { + match format { + Format::WikidataDump => Box::new(wikidata::Wikidata { dump: true }), + Format::WikidataEntities => Box::new(wikidata::Wikidata { dump: false }), + Format::MajesticCsv => Box::new(csv_sources::CsvSource { crux: false }), + Format::CruxCsv => Box::new(csv_sources::CsvSource { crux: true }), + Format::CurlieTarGz => Box::new(curlie::Curlie), + Format::PslText => Box::new(PslAdapter), + } +} + +pub(crate) fn bounded_line(input: &mut dyn BufRead, buffer: &mut String) -> anyhow::Result { + buffer.clear(); + let n = input + .take((MAX_RECORD_BYTES + 1) as u64) + .read_line(buffer)?; + ensure!(n <= MAX_RECORD_BYTES, "record exceeds 16 MiB"); + Ok(n) +} + +struct PslAdapter; +impl SourceAdapter for PslAdapter { + fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> { + let mut text = String::new(); + input + .take((MAX_RECORD_BYTES + 1) as u64) + .read_to_string(&mut text)?; + ensure!(text.len() <= MAX_RECORD_BYTES, "PSL exceeds bound"); + crate::normalize::Normalizer::new(text.as_bytes(), String::new())?; + sink.emit(Record { + native_id: "public_suffix_list.dat".into(), + raw: json!(text), + facts: vec![Fact { + subject: "psl".into(), + predicate: "psl".into(), + value: json!(text), + selector: String::new(), + confidence: 10000, + }], + }) + } +} diff --git a/crates/argand-site-registry/src/adapters/wikidata.rs b/crates/argand-site-registry/src/adapters/wikidata.rs new file mode 100644 index 0000000..bbc151f --- /dev/null +++ b/crates/argand-site-registry/src/adapters/wikidata.rs @@ -0,0 +1,186 @@ +// By Nic Weyand! +//! Wikibase JSON, retaining complete statement qualifiers, references, and rank. + +use super::{MAX_RECORD_BYTES, RecordSink, SourceAdapter, bounded_line}; +use crate::model::{Fact, Record, Source, entity_id}; +use anyhow::{Context, ensure}; +use serde_json::{Value, json}; +use std::io::{BufRead, Read}; + +pub(super) struct Wikidata { + pub dump: bool, +} + +impl SourceAdapter for Wikidata { + fn ingest(&self, input: &mut dyn BufRead, sink: &mut dyn RecordSink) -> anyhow::Result<()> { + if !self.dump { + let mut raw = Vec::new(); + input + .take((MAX_RECORD_BYTES + 1) as u64) + .read_to_end(&mut raw)?; + ensure!( + raw.len() <= MAX_RECORD_BYTES, + "entity response exceeds 16 MiB; use dump format" + ); + let value = crate::json::parse(&raw)?; + for (key, entity) in value["entities"] + .as_object() + .context("missing entities object")? + { + ensure!( + entity["id"].as_str() == Some(key), + "entity map key disagrees with native ID" + ); + project(entity, sink, true)?; + } + return Ok(()); + } + let mut line = String::new(); + bounded_line(input, &mut line)?; + ensure!(line.trim() == "[", "dump must begin with ["); + let mut seen = false; + let mut comma = false; + loop { + ensure!( + bounded_line(input, &mut line)? > 0, + "truncated Wikidata dump" + ); + let row = line.trim(); + if row.is_empty() { + continue; + } + if row == "]" { + ensure!(!comma && seen, "empty dump or trailing comma"); + while bounded_line(input, &mut line)? > 0 { + ensure!(line.trim().is_empty(), "data after dump"); + } + return Ok(()); + } + ensure!(!seen || comma, "missing entity separator"); + comma = row.ends_with(','); + let entity = crate::json::parse(row.strip_suffix(',').unwrap_or(row).as_bytes())?; + project(&entity, sink, false)?; + seen = true; + } + } +} + +fn project(raw: &Value, sink: &mut dyn RecordSink, explicit_selection: bool) -> anyhow::Result<()> { + let id = raw["id"].as_str().context("missing Wikidata ID")?; + ensure!( + id.len() > 1 + && matches!(id.as_bytes()[0], b'Q' | b'P' | b'L') + && id.as_bytes()[1] != b'0' + && id[1..].bytes().all(|b| b.is_ascii_digit()), + "invalid Wikidata ID" + ); + let claims = raw.get("claims").and_then(Value::as_object); + let mut facts = Vec::new(); + let subject = entity_id(Source::Wikidata, id); + // Preserve all names of relevant entities and all small code-mapping records. + let relevant = claims.is_some_and(|c| { + ["P856", "P297", "P218", "P219", "P220"] + .iter() + .any(|p| c.contains_key(*p)) + }); + if !relevant { + // A valid entity response with removed websites/deleted entity is a + // retirement observation. It must supersede the old selection instead + // of looking like an empty failed download that keeps old links active. + if explicit_selection { + sink.emit(Record { + native_id: id.into(), + raw: raw.clone(), + facts: Vec::new(), + })?; + } + return Ok(()); + } + for (field, kind) in [("labels", "label"), ("aliases", "alias")] { + if let Some(names) = raw.get(field).and_then(Value::as_object) { + for (language, values) in names { + let names: Vec<&Value> = if kind == "alias" { + values + .as_array() + .context("aliases must be arrays")? + .iter() + .collect() + } else { + vec![values] + }; + for (i, value) in names.into_iter().enumerate() { + ensure!( + value["language"].as_str() == Some(language), + "name language mismatch" + ); + let text = value["value"].as_str().context("name must be text")?; + let pointer_language = language.replace('~', "~0").replace('/', "~1"); + facts.push(Fact { + subject: subject.clone(), + predicate: "name".into(), + value: json!({"text":text,"language":language,"kind":kind,"native_id":id}), + selector: if kind == "alias" { + format!("/{field}/{pointer_language}/{i}") + } else { + format!("/{field}/{pointer_language}") + }, + confidence: 8000, + }); + } + } + } + } + if let Some(claims) = claims { + for property in [ + "P856", "P17", "P159", "P407", "P1001", "P297", "P218", "P219", "P220", + ] { + if let Some(statements) = claims.get(property) { + for (i, statement) in statements + .as_array() + .context("claims must be arrays")? + .iter() + .enumerate() + { + let predicate = if property == "P856" { + "website" + } else { + property + }; + let value = if property == "P856" { + website_value(statement, id, raw)? + } else { + json!({"native_id":id,"statement":statement}) + }; + facts.push(Fact { + subject: subject.clone(), + predicate: predicate.into(), + value, + selector: format!("/claims/{property}/{i}"), + confidence: 5000, + }); + } + } + } + } + sink.emit(Record { + native_id: id.into(), + raw: raw.clone(), + facts, + }) +} + +fn website_value(statement: &Value, id: &str, raw: &Value) -> anyhow::Result { + let snak = &statement["mainsnak"]; + ensure!( + snak["property"].as_str() == Some("P856"), + "mismatched website property" + ); + let url = if snak["snaktype"] == "value" + && snak.pointer("/datavalue/type").and_then(Value::as_str) == Some("string") + { + snak.pointer("/datavalue/value") + } else { + None + }; + Ok(json!({"url":url,"native_id":id,"statement":statement,"revision":raw.get("lastrevid")})) +} diff --git a/crates/argand-site-registry/src/build.rs b/crates/argand-site-registry/src/build.rs new file mode 100644 index 0000000..081c715 --- /dev/null +++ b/crates/argand-site-registry/src/build.rs @@ -0,0 +1,359 @@ +// By Nic Weyand! +//! Immutable SQLite projections, with source evidence left intact. + +use crate::{ + model::SourceManifest, + normalize::{Normalizer, name_key}, + store, +}; +use anyhow::{Context, ensure}; +use rusqlite::{Connection, params}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::{ + fs::{self, File}, + path::Path, +}; + +const PROJECTIONS: &str = " +CREATE TABLE selected_sources(id TEXT PRIMARY KEY REFERENCES sources(id)) STRICT; +INSERT INTO selected_sources SELECT id FROM ( + SELECT id,row_number() OVER(PARTITION BY source,scope ORDER BY retrieved_at DESC,id DESC) AS position + FROM sources WHERE complete=1) WHERE position=1; +CREATE TABLE names(entity TEXT NOT NULL,key TEXT NOT NULL,text TEXT NOT NULL,language TEXT NOT NULL,kind TEXT NOT NULL,fact TEXT NOT NULL REFERENCES facts(id),PRIMARY KEY(entity,fact)) STRICT; +CREATE INDEX name_lookup ON names(key,entity); +CREATE TABLE entities(id TEXT PRIMARY KEY,canonical_name TEXT NOT NULL,names_fingerprint TEXT NOT NULL) STRICT; +CREATE TABLE properties(id TEXT PRIMARY KEY,url TEXT NOT NULL UNIQUE,hostname TEXT NOT NULL,domain TEXT NOT NULL,suffix TEXT NOT NULL,derived_json TEXT NOT NULL) STRICT; +CREATE INDEX property_host ON properties(hostname); +CREATE TABLE edges(fingerprint TEXT PRIMARY KEY,entity TEXT NOT NULL REFERENCES entities(id),property TEXT NOT NULL REFERENCES properties(id),relation TEXT NOT NULL,facts TEXT NOT NULL,evidence TEXT NOT NULL,eligible INTEGER NOT NULL CHECK(eligible IN(0,1))) STRICT; +CREATE INDEX edge_entity ON edges(entity,property); +CREATE TABLE popularity(fact TEXT PRIMARY KEY REFERENCES facts(id),source TEXT NOT NULL,target TEXT NOT NULL,hostname TEXT NOT NULL,domain TEXT NOT NULL,value TEXT NOT NULL,derived_json TEXT NOT NULL) STRICT; +CREATE INDEX popularity_host ON popularity(hostname,source); +CREATE TABLE rejected(fact TEXT PRIMARY KEY REFERENCES facts(id),reason TEXT NOT NULL) STRICT; +"; + +/// Completion receipt. Authenticity needs an external digest or trusted signature. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Receipt { + /// `argand.site-registry/v1`. + pub schema: String, + /// Parser/derivation contract. + pub rules: String, + /// Database content digest. + pub database_sha256: String, + /// Digest of the shipped source-license document. + pub licenses_sha256: String, + /// Digest of the shipped machine-readable attribution envelope. + pub attribution_sha256: String, + /// PSL source manifest identity. + pub psl_source: String, + /// Active source snapshot declarations. + pub sources: Vec, + /// Distinct entity count. + pub entities: u64, + /// Strict URL identity count. + pub properties: u64, + /// Edge assertion groups, including ineligible evidence. + pub edges: u64, + /// Rejected projection facts, retained in the assertion tables. + pub rejected: u64, +} + +/// Builds a fresh generation from a consistent snapshot of committed imports. +/// +/// # Errors +/// Rejects existing destinations, incomplete PSL, corrupt stores, and I/O failures. +pub fn build(db: &Connection, output: &Path) -> anyhow::Result { + fs::create_dir(output).context("generation destination must not exist")?; + let path = output.join("registry.sqlite"); + let snapshot = store::open(&path)?; + copy_canonical(db, &snapshot)?; + snapshot.execute_batch(PROJECTIONS)?; + let (psl_id,text): (String,String)=snapshot.query_row("SELECT f.source_id,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl' ORDER BY f.source_id LIMIT 1",[],|r| Ok((r.get(0)?,r.get(1)?))).context("import a complete PSL snapshot first")?; + let psl_count:u64=snapshot.query_row("SELECT count(*) FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='psl'",[],|r|store::unsigned(r,0))?; + ensure!(psl_count == 1, "exactly one active PSL snapshot required"); + let psl_text: String = serde_json::from_str(&text)?; + let normalizer = Normalizer::new(psl_text.as_bytes(), psl_id.clone())?; + snapshot.execute_batch("BEGIN")?; + project_names(&snapshot)?; + project_facts(&snapshot, &normalizer)?; + snapshot.execute_batch("COMMIT; ANALYZE;")?; + let check: String = snapshot.query_row("PRAGMA integrity_check", [], |r| r.get(0))?; + ensure!(check == "ok", "registry integrity failed"); + let foreign_count: u64 = + snapshot.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| { + store::unsigned(r, 0) + })?; + ensure!(foreign_count == 0, "registry foreign keys failed"); + let mut statement = snapshot + .prepare("SELECT manifest FROM sources JOIN selected_sources USING(id) ORDER BY id")?; + let sources = statement + .query_map([], |r| r.get::<_, String>(0))? + .map(|s| Ok(serde_json::from_str(&s?)?)) + .collect::>>()?; + drop(statement); + let mut receipt = Receipt { + schema: "argand.site-registry/v1".into(), + rules: store::RULE_VERSION.into(), + database_sha256: String::new(), + licenses_sha256: crate::digest(crate::release::LICENSES.as_bytes()), + attribution_sha256: crate::digest(&serde_json::to_vec_pretty( + &crate::release::attribution(), + )?), + psl_source: psl_id, + sources, + entities: count(&snapshot, "entities")?, + properties: count(&snapshot, "properties")?, + edges: count(&snapshot, "edges")?, + rejected: count(&snapshot, "rejected")?, + }; + snapshot.close().map_err(|(_, e)| e)?; + File::open(&path)?.sync_all()?; + receipt.database_sha256 = crate::file_digest(&path)?; + argand_atomic::create_durable( + &output.join("LICENSE_SOURCES.md"), + crate::release::LICENSES.as_bytes(), + )?; + argand_atomic::create_durable( + &output.join("ATTRIBUTION.json"), + &serde_json::to_vec_pretty(&crate::release::attribution())?, + )?; + argand_atomic::create_durable( + &output.join("COMPLETE.json"), + &serde_json::to_vec_pretty(&receipt)?, + )?; + Ok(receipt) +} + +fn copy_canonical(source: &Connection, destination: &Connection) -> anyhow::Result<()> { + // Sorted logical copy prevents physical insertion order or failed imports + // from changing generation bytes. Hold one read transaction across tables. + let source_transaction = source.unchecked_transaction()?; + let destination_transaction = destination.unchecked_transaction()?; + let tables = [ + ( + "sources", + "SELECT * FROM sources WHERE complete=1 ORDER BY id", + 7, + ), + ( + "records", + "SELECT r.* FROM records r JOIN sources s ON s.id=r.source_id WHERE s.complete=1 ORDER BY r.source_id,r.ordinal", + 4, + ), + ( + "facts", + "SELECT f.* FROM facts f JOIN sources s ON s.id=f.source_id WHERE s.complete=1 ORDER BY f.id", + 8, + ), + ("reviews", "SELECT * FROM reviews ORDER BY sequence", 11), + ( + "equivalences", + "SELECT * FROM equivalences ORDER BY fingerprint", + 5, + ), + ]; + for (table, select, columns) in tables { + let placeholders = (1..=columns) + .map(|i| format!("?{i}")) + .collect::>() + .join(","); + let mut insert = + destination.prepare(&format!("INSERT INTO {table} VALUES({placeholders})"))?; + let mut select = source.prepare(select)?; + let mut rows = select.query([])?; + while let Some(row) = rows.next()? { + let values = (0..columns) + .map(|i| row.get::<_, rusqlite::types::Value>(i)) + .collect::, _>>()?; + insert.execute(rusqlite::params_from_iter(values))?; + } + } + destination_transaction.commit()?; + source_transaction.commit()?; + Ok(()) +} + +fn count(db: &Connection, table: &str) -> anyhow::Result { + Ok( + db.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| { + store::unsigned(r, 0) + })?, + ) +} + +fn project_names(db: &Connection) -> anyhow::Result<()> { + let mut stmt=db.prepare("SELECT f.id,f.subject,f.value FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate='name' ORDER BY f.subject,f.id")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let (id, subject, raw): (String, String, String) = (row.get(0)?, row.get(1)?, row.get(2)?); + let value: Value = serde_json::from_str(&raw)?; + let text = value["text"].as_str().context("name text missing")?; + match name_key(text) { + Ok(key) => { + db.execute( + "INSERT INTO names VALUES(?1,?2,?3,?4,?5,?6)", + params![ + subject, + key, + text, + value["language"].as_str().unwrap_or("und"), + value["kind"].as_str().unwrap_or("label"), + id + ], + )?; + } + Err(error) => { + db.execute( + "INSERT INTO rejected VALUES(?1,?2)", + params![id, error.to_string()], + )?; + } + } + } + // All entities with website assertions exist even when names are absent. + let mut entities=db.prepare("SELECT DISTINCT f.subject FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.predicate IN('website','name') ORDER BY f.subject")?; + for subject in entities.query_map([], |r| r.get::<_, String>(0))? { + let subject = subject?; + let mut names=db.prepare("SELECT DISTINCT text,language,kind FROM names WHERE entity=?1 ORDER BY CASE WHEN kind='label' THEN 0 ELSE 1 END,CASE WHEN language='en' THEN 0 ELSE 1 END,language,text")?; + let values = names + .query_map([&subject], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + )) + })? + .collect::, _>>()?; + let canonical = values.first().map_or(subject.as_str(), |v| v.0.as_str()); + let fingerprint = crate::digest(&serde_json::to_vec(&values)?); + db.execute( + "INSERT INTO entities VALUES(?1,?2,?3)", + params![subject, canonical, fingerprint], + )?; + } + Ok(()) +} + +fn project_facts(db: &Connection, normalizer: &Normalizer) -> anyhow::Result<()> { + let mut stmt=db.prepare("SELECT f.id,f.subject,f.predicate,f.value,s.source,f.selector,r.native_id FROM facts f JOIN sources s ON s.id=f.source_id JOIN selected_sources a ON a.id=s.id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE f.predicate IN('website','popularity') ORDER BY f.subject,f.id")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: String = row.get(0)?; + let subject: String = row.get(1)?; + let predicate: String = row.get(2)?; + let raw: String = row.get(3)?; + let value: Value = serde_json::from_str(&raw)?; + let source: String = row.get(4)?; + let result = if predicate == "website" { + project_edge( + db, + normalizer, + &id, + &subject, + &value, + &source, + &row.get::<_, String>(5)?, + &row.get::<_, String>(6)?, + ) + } else { + project_popularity(db, normalizer, &id, &source, &value) + }; + if let Err(error) = result { + // SQL errors are operational failures, never silently quarantined data. + if error.downcast_ref::().is_some() { + return Err(error); + } + db.execute( + "INSERT INTO rejected VALUES(?1,?2)", + params![id, error.to_string()], + )?; + } + } + Ok(()) +} + +fn project_popularity( + db: &Connection, + n: &Normalizer, + id: &str, + source: &str, + value: &Value, +) -> anyhow::Result<()> { + let target = value["target"] + .as_str() + .context("missing popularity target")?; + let domain = if value["target_kind"] == "origin" { + n.url(target)?.domain + } else { + n.domain(target)? + }; + db.execute( + "INSERT INTO popularity VALUES(?1,?2,?3,?4,?5,?6,?7)", + params![ + id, + source, + target, + domain.hostname, + domain.registrable_domain, + serde_json::to_string(value)?, + serde_json::to_string(&domain)? + ], + )?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] // One complete source assertion, kept explicit for auditability. +fn project_edge( + db: &Connection, + n: &Normalizer, + id: &str, + entity: &str, + value: &Value, + source: &str, + selector: &str, + native: &str, +) -> anyhow::Result<()> { + let url = value["url"] + .as_str() + .context("website statement has no concrete URL")?; + let property = n.url(url)?; + db.execute( + "INSERT OR IGNORE INTO properties VALUES(?1,?2,?3,?4,?5,?6)", + params![ + property.id, + property.url, + property.domain.hostname, + property.domain.registrable_domain, + property.domain.public_suffix, + serde_json::to_string(&property)? + ], + )?; + let names: String = db.query_row( + "SELECT names_fingerprint FROM entities WHERE id=?1", + [entity], + |r| r.get(0), + )?; + let relation = if source == "wikidata" { + "asserted_official" + } else { + "directory_listing" + }; + let evidence = json!({"source":source,"native_id":native,"selector":selector,"assertion":value,"names_fingerprint":names,"normalization":store::RULE_VERSION}); + let mut identity_property = property.clone(); + // Bind review to actual PSL bytes and derived fields, not a fresh timestamp + // for an otherwise identical list. Full retrieval provenance stays on property. + identity_property.domain.psl_source.clear(); + let fingerprint = crate::digest(&serde_json::to_vec(&( + entity, + &identity_property, + &evidence, + ))?); + // End-dated assertions stay as historical evidence, never current destinations. + // Future/partial starts require the operator to inspect the retained qualifiers. + let eligible = value.pointer("/statement/rank").and_then(Value::as_str) != Some("deprecated") + && value.pointer("/statement/qualifiers/P582").is_none(); + db.execute("INSERT INTO edges VALUES(?1,?2,?3,?4,?5,?6,?7) ON CONFLICT(fingerprint) DO UPDATE SET facts=json_insert(edges.facts,'$[#]',?8)",params![fingerprint,entity,property.id,relation,serde_json::to_string(&vec![id])?,serde_json::to_string(&evidence)?,eligible,id])?; + Ok(()) +} diff --git a/crates/argand-site-registry/src/cli.rs b/crates/argand-site-registry/src/cli.rs new file mode 100644 index 0000000..45b1167 --- /dev/null +++ b/crates/argand-site-registry/src/cli.rs @@ -0,0 +1,380 @@ +// By Nic Weyand! +//! Explicit source acquisition, import, review, and immutable generation commands. + +use anyhow::ensure; +use argand_site_registry as registry; +use clap::{Parser, Subcommand}; +use registry::{ + model::{Compression, Format, Source, SourceManifest}, + query::Registry, +}; +use std::{io::Write, path::PathBuf}; + +#[derive(Parser)] +#[command( + version, + about = "Provenance-preserving, reviewed entity ↔ website registry" +)] +struct Args { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Preview an explicit entity equivalence, or record its exact review JSON. + Equivalence { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + left: String, + #[arg(long)] + right: String, + #[arg(long, requires = "database")] + decision: Option, + #[arg(long)] + database: Option, + }, + /// Download one allowlisted source into an immutable local cache. + Download { + #[arg(long)] + cache: PathBuf, + #[arg(long, value_enum)] + source: Source, + #[arg(long, value_enum)] + format: Format, + #[arg(long, value_enum, default_value = "none")] + compression: Compression, + #[arg(long)] + url: String, + #[arg(long)] + snapshot: String, + #[arg(long)] + scope: String, + #[arg(long)] + maximum_bytes: u64, + }, + /// Download paginated `CrUX` data using an explicit billing configuration JSON. + CruxDownload { + #[arg(long)] + cache: PathBuf, + #[arg(long)] + request: PathBuf, + }, + /// Declare a pinned local source; does not certify ownership or publisher trust. + Manifest { + #[arg(long)] + input: PathBuf, + #[arg(long)] + output: PathBuf, + #[arg(long, value_enum)] + source: Source, + #[arg(long, value_enum)] + format: Format, + #[arg(long, value_enum, default_value = "none")] + compression: Compression, + #[arg(long)] + source_url: String, + #[arg(long)] + snapshot: String, + #[arg(long)] + scope: String, + #[arg(long)] + retrieved_at: chrono::DateTime, + }, + /// Import a complete pinned source, resuming committed record batches. + Import { + #[arg(long)] + database: PathBuf, + #[arg(long)] + input: PathBuf, + #[arg(long)] + manifest: PathBuf, + }, + /// Build a new immutable generation; output must not exist. + Build { + #[arg(long)] + database: PathBuf, + #[arg(long)] + output: PathBuf, + }, + /// Audit an exact name or alias; includes ambiguity counts and attribution. + Lookup { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + query: String, + #[arg(long, default_value_t = 20)] + limit: u32, + }, + /// Resolve only an unambiguous, explicitly reviewed, unexpired property. + Resolve { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + query: String, + #[arg(long)] + locale: Option, + #[arg(long)] + country: Option, + }, + /// Append an exact assertion approval or revocation from a review JSON file. + Review { + #[arg(long)] + database: PathBuf, + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + decision: PathBuf, + }, + /// Export facts as streaming JSONL with source licenses and attribution. + Export { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + output: PathBuf, + #[arg(long)] + include_descriptions: bool, + }, + /// Show added/removed evidence fingerprints without loading either registry. + Diff { + #[arg(long)] + old: PathBuf, + #[arg(long)] + old_pin: String, + #[arg(long)] + new: PathBuf, + #[arg(long)] + new_pin: String, + }, + /// Verify complete database bytes against an external receipt pin. + Verify { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + }, + /// Sign an independently reviewed generation with an SSH signing key. + Sign { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + pin: String, + #[arg(long)] + key: PathBuf, + }, + /// Verify publisher signature and atomically activate (also supports rollback). + Activate { + #[arg(long)] + generation: PathBuf, + #[arg(long)] + current: PathBuf, + #[arg(long)] + allowed_signers: PathBuf, + #[arg(long)] + identity: String, + }, + /// Run configured acquisition/import/build; never approves or activates. + Update { + #[arg(long)] + config: PathBuf, + }, +} + +pub(super) async fn run() -> anyhow::Result<()> { + let value = match Args::parse().command { + command @ (Command::Download { .. } + | Command::CruxDownload { .. } + | Command::Manifest { .. } + | Command::Equivalence { .. }) => acquire(command).await?, + Command::Import { + database, + input, + manifest, + } => { + serde_json::json!({"source_id":registry::store::import(&mut registry::store::open(&database)?,®istry::read_json(&manifest)?,&input)?}) + } + Command::Build { database, output } => { + ensure!(database.is_file(), "import database does not exist"); + let receipt = registry::build::build(®istry::store::open(&database)?, &output)?; + serde_json::json!({"receipt":receipt,"pin":registry::file_digest(&output.join("COMPLETE.json"))?,"generation":output}) + } + Command::Lookup { + generation, + pin, + query, + limit, + } => serde_json::to_value(Registry::open(&generation, &pin)?.lookup(&query, limit)?)?, + Command::Resolve { + generation, + pin, + query, + locale, + country, + } => { + serde_json::json!({"destination":Registry::open(&generation,&pin)?.resolve(&query,locale.as_deref(),country.as_deref(),chrono::Utc::now())?,"attribution":registry::release::attribution()}) + } + Command::Review { + database, + generation, + pin, + decision, + } => { + serde_json::json!({"review_sequence":registry::review::record(®istry::store::open(&database)?,&Registry::open(&generation,&pin)?,®istry::read_json(&decision)?)?,"rebuild_required":true}) + } + Command::Export { + generation, + pin, + output, + include_descriptions, + } => { + registry::release::export( + &Registry::open(&generation, &pin)?, + &output, + include_descriptions, + )?; + serde_json::json!({"export":output}) + } + Command::Diff { + old, + old_pin, + new, + new_pin, + } => { + registry::release::diff( + &Registry::open(&old, &old_pin)?, + &Registry::open(&new, &new_pin)?, + &mut std::io::stdout().lock(), + )?; + return Ok(()); + } + Command::Verify { generation, pin } => { + serde_json::to_value(Registry::open(&generation, &pin)?.receipt)? + } + Command::Sign { + generation, + pin, + key, + } => { + registry::release::sign(&generation, &key, &pin)?; + serde_json::json!({"signed":generation}) + } + Command::Activate { + generation, + current, + allowed_signers, + identity, + } => { + registry::release::activate(&generation, ¤t, &allowed_signers, &identity)?; + serde_json::json!({"current":current}) + } + Command::Update { config } => { + let config = read_config(&config)?; + serde_json::json!({"candidate":registry::update::run(&config).await?}) + } + }; + writeln!( + std::io::stdout().lock(), + "{}", + serde_json::to_string_pretty(&value)? + )?; + Ok(()) +} + +async fn acquire(command: Command) -> anyhow::Result { + Ok(match command { + Command::Equivalence { + generation, + pin, + left, + right, + decision, + database, + } => { + let registry = Registry::open(&generation, &pin)?; + let pair = registry::identity::propose(®istry, &left, &right)?; + if let Some(decision) = decision { + let database = database + .ok_or_else(|| anyhow::anyhow!("identity review needs a writer database"))?; + serde_json::json!({"review_sequence":registry::identity::record(®istry::store::open(&database)?,®istry,&pair,®istry::read_json(&decision)?)?,"rebuild_required":true}) + } else { + serde_json::to_value(pair)? + } + } + Command::Download { + cache, + source, + format, + compression, + url, + snapshot, + scope, + maximum_bytes, + } => serde_json::to_value( + registry::download::download( + &cache, + ®istry::download::Download { + source, + format, + compression, + url, + snapshot, + scope, + maximum_bytes, + }, + ) + .await?, + )?, + Command::CruxDownload { cache, request } => serde_json::to_value( + registry::crux::download(&cache, ®istry::read_json(&request)?).await?, + )?, + Command::Manifest { + input, + output, + source, + format, + compression, + source_url, + snapshot, + scope, + retrieved_at, + } => { + let manifest = SourceManifest { + schema: "argand.site-source/v1".into(), + source, + format, + compression, + source_url, + snapshot, + scope, + retrieved_at, + license: source.license().into(), + license_url: source.license_url().into(), + sha256: registry::file_digest(&input)?, + bytes: input.metadata()?.len(), + }; + manifest.validate()?; + argand_atomic::create_durable(&output, &serde_json::to_vec_pretty(&manifest)?)?; + serde_json::to_value(manifest)? + } + _ => anyhow::bail!("expected an acquisition command"), + }) +} + +fn read_config(path: &std::path::Path) -> anyhow::Result { + ensure!( + path.metadata()?.len() <= 1024 * 1024, + "config exceeds 1 MiB" + ); + Ok(toml::from_str(&std::fs::read_to_string(path)?)?) +} diff --git a/crates/argand-site-registry/src/crux.rs b/crates/argand-site-registry/src/crux.rs new file mode 100644 index 0000000..00248ed --- /dev/null +++ b/crates/argand-site-registry/src/crux.rs @@ -0,0 +1,353 @@ +// By Nic Weyand! +//! Official `BigQuery` `CrUX` projection, bounded pagination and idempotent job IDs. + +use crate::{ + download::{CachedSource, client}, + model::{Compression, Format, Source, SourceManifest}, +}; +use anyhow::{Context, ensure}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::{ + fs::{self, File, OpenOptions}, + io::Write, + path::Path, + time::Duration, +}; + +/// Explicit authenticated `BigQuery` request. Never run without a billing cap. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CruxDownload { + /// Operator's billing project. + pub project: String, + /// Observation month, YYYYMM. + pub month: String, + /// Optional two-letter audience country. + pub country: Option, + /// Maximum bytes billed by this query. + pub maximum_bytes_billed: u64, + /// Maximum exported CSV bytes. + pub maximum_output_bytes: u64, +} + +/// Acquires a complete `CrUX` projection using `GOOGLE_OAUTH_ACCESS_TOKEN`. +/// The token is never persisted, logged, or sent outside bigquery.googleapis.com. +/// A durable job ID avoids duplicate submissions on interruption. Completed +/// pages replay from the same server-side result; output is published atomically. +/// +/// # Errors +/// Returns authentication, billing, schema, size, expiry, or transport failures. +pub async fn download(cache: &Path, request: &CruxDownload) -> anyhow::Result { + let query = query(request)?; + let key = crate::digest(&serde_json::to_vec(request)?); + let root = cache.join("crux").join(&key); + fs::create_dir_all(&root)?; + let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(root.join("download.lock"))?; + lock.try_lock() + .context("CrUX acquisition already running")?; + let manifest_path = root.join("source.json"); + if manifest_path.exists() { + let manifest: SourceManifest = crate::read_json(&manifest_path)?; + manifest.validate()?; + let input = root.join(&manifest.sha256); + ensure!( + crate::file_digest(&input)? == manifest.sha256 + && input.metadata()?.len() == manifest.bytes + && manifest.source == Source::Crux + && manifest.bytes <= request.maximum_output_bytes, + "cached CrUX corruption" + ); + return Ok(CachedSource { + input, + manifest: manifest_path, + }); + } + let token = std::env::var("GOOGLE_OAUTH_ACCESS_TOKEN") + .context("set GOOGLE_OAUTH_ACCESS_TOKEN for the explicitly configured billing project")?; + let client = client()?; + let job_id = format!("argand_site_registry_{key}"); + ensure_job(&client, &token, request, &job_id, &query, &root).await?; + let part = root.join("projection.part"); + download_pages(&client, &token, request, &job_id, &part).await?; + let sha256 = crate::file_digest(&part)?; + let input = root.join(&sha256); + let manifest = SourceManifest { + schema: "argand.site-source/v1".into(), + source: Source::Crux, + format: Format::CruxCsv, + compression: Compression::None, + snapshot: format!( + "{}:{}:{job_id}", + request.month, + request.country.as_deref().unwrap_or("global") + ), + scope: format!( + "monthly:{}:{}", + request.month, + request.country.as_deref().unwrap_or("global") + ), + source_url: "https://developer.chrome.com/docs/crux/bigquery/".into(), + license: Source::Crux.license().into(), + license_url: Source::Crux.license_url().into(), + retrieved_at: Utc::now(), + sha256, + bytes: part.metadata()?.len(), + }; + fs::rename(part, &input)?; + File::open(&root)?.sync_all()?; + argand_atomic::create_durable(&manifest_path, &serde_json::to_vec_pretty(&manifest)?)?; + Ok(CachedSource { + input, + manifest: manifest_path, + }) +} + +async fn ensure_job( + client: &reqwest::Client, + token: &str, + request: &CruxDownload, + job_id: &str, + query: &str, + root: &Path, +) -> anyhow::Result<()> { + let job = json!({"jobReference":{"projectId":request.project,"jobId":job_id,"location":"US"},"configuration":{"query":{"query":query,"useLegacySql":false,"maximumBytesBilled":request.maximum_bytes_billed.to_string()}}}); + let job_path = root.join("job.json"); + if !job_path.exists() { + argand_atomic::create_durable(&job_path, &serde_json::to_vec_pretty(&job)?)?; + } + let endpoint = format!( + "https://bigquery.googleapis.com/bigquery/v2/projects/{}/jobs", + request.project + ); + let response = client + .post(&endpoint) + .bearer_auth(token) + .json(&job) + .send() + .await?; + ensure!( + response.status().is_success() || response.status() == reqwest::StatusCode::CONFLICT, + "BigQuery job submission HTTP {}", + response.status() + ); + // A conflict must be our exact query and budget, not an unrelated prior job. + let remote = bounded_json( + client + .get(format!("{endpoint}/{job_id}")) + .query(&[("location", "US")]) + .bearer_auth(token) + .send() + .await?, + ) + .await?; + ensure!( + remote + .pointer("/configuration/query/query") + .and_then(Value::as_str) + == Some(query), + "existing BigQuery job query differs" + ); + ensure!( + remote + .pointer("/configuration/query/maximumBytesBilled") + .and_then(Value::as_str) + == Some(&request.maximum_bytes_billed.to_string()), + "existing BigQuery job billing limit differs" + ); + ensure!( + remote.pointer("/status/errorResult").is_none(), + "BigQuery job failed; inspect job metadata using your account" + ); + Ok(()) +} + +async fn download_pages( + client: &reqwest::Client, + token: &str, + request: &CruxDownload, + job_id: &str, + part: &Path, +) -> anyhow::Result<()> { + let result_url = format!( + "https://bigquery.googleapis.com/bigquery/v2/projects/{}/queries/{job_id}", + request.project + ); + let mut output = File::create(part)?; // atomic-writes: allow unpublished replay file, synced and renamed before manifest publication + output.write_all(b"origin,rank,yyyymm,country_code\n")?; + let mut written = 29_u64; + let mut count = 0_u64; + let mut page_token = String::new(); + let mut pending = 0_u32; + loop { + let mut get = client.get(&result_url).bearer_auth(token).query(&[ + ("location", "US"), + ("maxResults", "10000"), + ("timeoutMs", "10000"), + ]); + if !page_token.is_empty() { + get = get.query(&[("pageToken", &page_token)]); + } + let page = bounded_json(get.send().await?).await?; + if page["jobComplete"] != true { + pending += 1; + ensure!( + pending <= 180, + "BigQuery job still pending; rerun to resume the same job" + ); + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + ensure!(page.get("errors").is_none(), "BigQuery returned job errors"); + let bytes = page_csv(&page)?; + written += u64::try_from(bytes.len())?; + ensure!( + written <= request.maximum_output_bytes, + "CrUX output byte cap exceeded" + ); + output.write_all(&bytes)?; + count += u64::try_from( + page.get("rows") + .and_then(Value::as_array) + .map_or(0, Vec::len), + )?; + let next = page + .get("pageToken") + .and_then(Value::as_str) + .unwrap_or_default(); + if next.is_empty() { + let total: u64 = page["totalRows"] + .as_str() + .context("BigQuery totalRows missing")? + .parse()?; + ensure!( + count == total && count > 0, + "incomplete or empty CrUX result" + ); + break; + } + ensure!(next != page_token, "BigQuery repeated pagination token"); + page_token = next.into(); + } + output.sync_all()?; + drop(output); + Ok(()) +} + +fn query(request: &CruxDownload) -> anyhow::Result { + crate::adapters::csv_sources::validate_month(&request.month)?; + ensure!( + !request.project.is_empty() + && request.project.len() <= 63 + && request + .project + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'), + "invalid Google project ID" + ); + ensure!( + request.maximum_bytes_billed > 0 && request.maximum_output_bytes > 0, + "explicit positive query and output byte caps required" + ); + let (table, country) = if let Some(country) = &request.country { + ensure!( + country.len() == 2 && country.bytes().all(|b| b.is_ascii_alphabetic()), + "invalid country" + ); + ( + format!( + "chrome-ux-report.country_{}.{}", + country.to_ascii_lowercase(), + request.month + ), + country.to_ascii_uppercase(), + ) + } else { + ( + format!("chrome-ux-report.all.{}", request.month), + String::new(), + ) + }; + Ok(format!( + "SELECT DISTINCT origin, experimental.popularity.rank AS rank, '{}' AS yyyymm, '{}' AS country_code FROM `{table}` WHERE experimental.popularity.rank IS NOT NULL ORDER BY origin, rank", + request.month, country + )) +} + +async fn bounded_json(mut response: reqwest::Response) -> anyhow::Result { + ensure!( + response.status().is_success(), + "BigQuery HTTP {}", + response.status() + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + bytes.len() + chunk.len() <= 24 * 1024 * 1024, + "BigQuery page exceeds 24 MiB" + ); + bytes.extend_from_slice(&chunk); + } + crate::json::parse(&bytes) +} + +fn page_csv(page: &Value) -> anyhow::Result> { + let schema = page + .pointer("/schema/fields") + .and_then(Value::as_array) + .context("missing BigQuery schema")?; + ensure!( + schema + .iter() + .filter_map(|f| f["name"].as_str()) + .collect::>() + == ["origin", "rank", "yyyymm", "country_code"], + "BigQuery projection schema changed" + ); + let mut writer = csv::Writer::from_writer(Vec::new()); + if let Some(rows) = page.get("rows").and_then(Value::as_array) { + for row in rows { + let fields = row["f"].as_array().context("missing BigQuery row fields")?; + ensure!(fields.len() == 4, "BigQuery field count mismatch"); + let values = fields + .iter() + .map(|f| f["v"].as_str().context("null or non-string BigQuery cell")) + .collect::>>()?; + writer.write_record(values)?; + } + } + Ok(writer.into_inner()?) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn documented_projection_billing_bound_and_page_schema() -> anyhow::Result<()> { + let mut request = CruxDownload { + project: "example-project".into(), + month: "202608".into(), + country: Some("GB".into()), + maximum_bytes_billed: 1_000_000, + maximum_output_bytes: 1_000_000, + }; + let sql = query(&request)?; + assert!(sql.contains("`chrome-ux-report.country_gb.202608`")); + assert!(sql.contains("experimental.popularity.rank")); + request.maximum_bytes_billed = 0; + assert!(query(&request).is_err()); + let mut page = json!({"schema":{"fields":[{"name":"origin"},{"name":"rank"},{"name":"yyyymm"},{"name":"country_code"}]},"rows":[{"f":[{"v":"https://example.co.uk"},{"v":"1000"},{"v":"202608"},{"v":"GB"}]}]}); + assert_eq!(page_csv(&page)?, b"https://example.co.uk,1000,202608,GB\n"); + page["rows"][0]["f"][1]["v"] = Value::Null; + assert!(page_csv(&page).is_err()); + page["schema"]["fields"][0]["name"] = json!("changed"); + assert!(page_csv(&page).is_err()); + Ok(()) + } +} diff --git a/crates/argand-site-registry/src/download.rs b/crates/argand-site-registry/src/download.rs new file mode 100644 index 0000000..e0f0083 --- /dev/null +++ b/crates/argand-site-registry/src/download.rs @@ -0,0 +1,430 @@ +// By Nic Weyand! +//! Allowlisted source acquisition with validator-bound range resume. + +use crate::model::{Compression, Format, Source, SourceManifest}; +use anyhow::{Context, ensure}; +use chrono::Utc; +use reqwest::{Client, StatusCode, header}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{self, File, OpenOptions}, + io::Write, + path::{Path, PathBuf}, + time::Duration, +}; + +/// Explicit download request. The byte cap is mandatory for large objects. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Download { + /// Provider whose license and transport allowlist apply. + pub source: Source, + /// Supported source format. + pub format: Format, + /// Outer compression. + #[serde(default)] + pub compression: Compression, + /// Official distribution URL. + pub url: String, + /// Source-native revision or dump date. + pub snapshot: String, + /// Source replacement scope. + pub scope: String, + /// Maximum downloaded object bytes. + pub maximum_bytes: u64, +} + +/// Result points to immutable cached bytes and their manifest. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CachedSource { + /// Local input path. + pub input: PathBuf, + /// Local manifest path. + pub manifest: PathBuf, +} + +#[derive(Deserialize, Serialize)] +struct Resume { + url: String, + etag: String, + total: u64, +} + +/// Validates source URLs against reviewed data endpoints, including redirects. +/// No arbitrary URL or website assertion is ever fetched through this path. +/// +/// # Errors +/// Rejects unknown endpoints/parameters, credentials, fragments and custom ports. +#[allow(clippy::case_sensitive_file_extension_comparisons)] // HTTPS endpoint paths are case-sensitive. +pub fn validate_source_url(source: Source, input: &str) -> anyhow::Result<()> { + let url = url::Url::parse(input)?; + ensure!( + url.scheme() == "https" + && url.port().is_none() + && url.username().is_empty() + && url.password().is_none() + && url.fragment().is_none(), + "source requires HTTPS without credentials/fragment/custom port" + ); + let host = url.host_str().context("missing source host")?; + let path = url.path(); + if source == Source::Wikidata && host == "www.wikidata.org" && path == "/w/api.php" { + return validate_entity_query(&url); + } + ensure!(url.query().is_none(), "unexpected source query parameters"); + let allowed = match source { + Source::Wikidata => { + (host == "dumps.wikimedia.org" + && path.starts_with("/wikidatawiki/entities/") + && (path.ends_with(".json.gz") || path.ends_with(".json.bz2"))) + || (host == "www.wikidata.org" + && path.starts_with("/wiki/Special:EntityData/") + && path.ends_with(".json")) + } + Source::Majestic => host == "downloads.majestic.com" && path == "/majestic_million.csv", + Source::Crux => host == "developer.chrome.com" && path == "/docs/crux/bigquery/", + Source::Curlie => { + (host == "curlie.org" && path == "/directory-dl") + || (host == "share.innkube.fim.uni-passau.de" + && path == "/curlie-rdf/curlie-rdf-all.tar.gz") + } + Source::Psl => host == "publicsuffix.org" && path == "/list/public_suffix_list.dat", + }; + ensure!(allowed, "unreviewed source endpoint: {host}{path}"); + Ok(()) +} + +pub(crate) fn client() -> anyhow::Result { + Ok(Client::builder() + .user_agent("Argand-Site-Registry/0.1 (+https://argand.org)") + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .connect_timeout(Duration::from_secs(15)) + .read_timeout(Duration::from_secs(60)) + .build()?) +} + +/// Downloads and seals source bytes. Re-running the same request reuses them. +/// Change `snapshot` to request a refresh. Incomplete downloads resume only with +/// a strong `ETag`, matching Content-Range, and the same final URL. +/// +/// # Errors +/// Returns URL policy, transport, size, locking, integrity, or filesystem errors. +pub async fn download(cache: &Path, request: &Download) -> anyhow::Result { + ensure!( + request.source != Source::Crux, + "CrUX acquisition uses crux-download and authenticated BigQuery pagination" + ); + validate_source_url(request.source, &request.url)?; + ensure!(request.maximum_bytes > 0, "maximum bytes must be positive"); + let key = crate::digest(&serde_json::to_vec(request)?); + let dir = cache.join(request.source.key()).join(key); + fs::create_dir_all(&dir)?; + let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(dir.join("download.lock"))?; + lock.try_lock().context("source download already running")?; + let complete = dir.join("source.json"); + if complete.exists() { + let manifest: SourceManifest = crate::read_json(&complete)?; + manifest.validate()?; + ensure!( + manifest.source == request.source + && manifest.format == request.format + && manifest.source_url == request.url + && manifest.snapshot == request.snapshot + && manifest.scope == request.scope + && manifest.bytes <= request.maximum_bytes, + "cached source declaration differs from request" + ); + let input = dir.join(&manifest.sha256); + ensure!( + crate::file_digest(&input)? == manifest.sha256 + && input.metadata()?.len() == manifest.bytes, + "cached object corrupted" + ); + return Ok(CachedSource { + input, + manifest: complete, + }); + } + if request.source == Source::Psl { + reserve_psl_refresh(cache)?; + } + let part = dir.join("download.part"); + let state = dir.join("resume.json"); + let client = client()?; + let mut last_error = None; + let attempts = if request.source == Source::Psl { 1 } else { 3 }; + for attempt in 0..attempts { + match transfer(&client, request, &part, &state).await { + Ok(()) => { + last_error = None; + break; + } + Err(error) => { + last_error = Some(error); + if attempt + 1 < attempts { + tokio::time::sleep(Duration::from_secs(1 << attempt)).await; + } + } + } + } + if let Some(error) = last_error { + return Err(error); + } + let sha256 = crate::file_digest(&part)?; + let bytes = part.metadata()?.len(); + let manifest = SourceManifest { + schema: "argand.site-source/v1".into(), + source: request.source, + format: request.format, + compression: request.compression, + snapshot: request.snapshot.clone(), + scope: request.scope.clone(), + source_url: request.url.clone(), + license: request.source.license().into(), + license_url: request.source.license_url().into(), + retrieved_at: Utc::now(), + sha256: sha256.clone(), + bytes, + }; + manifest.validate()?; + let input = dir.join(sha256); + fs::rename(&part, &input)?; + File::open(&dir)?.sync_all()?; + argand_atomic::create_durable(&complete, &serde_json::to_vec_pretty(&manifest)?)?; + Ok(CachedSource { + input, + manifest: complete, + }) +} + +async fn transfer( + client: &Client, + request: &Download, + part: &Path, + state: &Path, +) -> anyhow::Result<()> { + let resume: Option = if state.exists() && part.exists() { + Some(crate::read_json(state)?) + } else { + None + }; + let offset = part.metadata().map_or(0, |m| m.len()); + let resume = resume.filter(|r| { + !r.etag.is_empty() && !r.etag.starts_with("W/") && offset > 0 && offset < r.total + }); + let mut url = request.url.clone(); + let mut response = None; + for _ in 0..5 { + validate_source_url(request.source, &url)?; + let mut get = client.get(&url).header(header::ACCEPT_ENCODING, "identity"); + if let Some(r) = resume.as_ref().filter(|r| r.url == url) { + get = get + .header(header::RANGE, format!("bytes={offset}-")) + .header(header::IF_RANGE, &r.etag); + } + let reply = get.send().await?; + if reply.status().is_redirection() { + let location = reply + .headers() + .get(header::LOCATION) + .context("redirect missing Location")? + .to_str()?; + url = url::Url::parse(&url)?.join(location)?.to_string(); + } else { + response = Some(reply); + break; + } + } + let mut response = response.context("source redirect limit exceeded")?; + let (start, total, etag) = response_extent(&response, resume.as_ref(), offset, &url)?; + ensure!( + total <= request.maximum_bytes, + "source exceeds configured byte budget" + ); + let mut output = OpenOptions::new() // atomic-writes: allow resumable unpublished partial file; seal by rename and receipt + .create(true) + .write(true) + .truncate(start == 0) + .append(start > 0) + .open(part)?; + argand_atomic::replace_durable(state, &serde_json::to_vec(&Resume { url, etag, total })?)?; + let mut written = start; + while let Some(chunk) = response.chunk().await? { + written += u64::try_from(chunk.len())?; + ensure!( + (total == 0 || written <= total) && written <= request.maximum_bytes, + "source exceeded declared byte length" + ); + output.write_all(&chunk)?; + } + output.sync_all()?; + ensure!( + written > 0 && (total == 0 || written == total), + "source transfer truncated or empty" + ); + Ok(()) +} + +fn response_extent( + response: &reqwest::Response, + resume: Option<&Resume>, + offset: u64, + url: &str, +) -> anyhow::Result<(u64, u64, String)> { + let status = response.status(); + ensure!( + status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT, + "source returned HTTP {status}" + ); + ensure!( + response + .headers() + .get(header::CONTENT_ENCODING) + .is_none_or(|v| v == "identity"), + "unexpected HTTP content encoding" + ); + let etag = response + .headers() + .get(header::ETAG) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_owned(); + let (start, total) = if status == StatusCode::PARTIAL_CONTENT { + let range = response + .headers() + .get(header::CONTENT_RANGE) + .context("missing Content-Range")? + .to_str()?; + let (start, end, total) = parse_range(range)?; + let old = resume.context("unsolicited partial response")?; + ensure!( + start == offset + && end + 1 == total + && total == old.total + && old.url == url + && etag == old.etag, + "range validator or offsets changed" + ); + (start, total) + } else { + (0, response.content_length().unwrap_or(0)) + }; + Ok((start, total, etag)) +} + +fn validate_entity_query(url: &url::Url) -> anyhow::Result<()> { + let pairs: Vec<_> = url.query_pairs().collect(); + let mut params = std::collections::BTreeMap::new(); + for (key, value) in pairs { + ensure!( + params.insert(key, value).is_none(), + "duplicate source query parameter" + ); + } + ensure!( + params.len() == 4 + && params.get("action").is_some_and(|s| s == "wbgetentities") + && params.get("format").is_some_and(|s| s == "json") + && params.get("maxlag").is_some_and(|s| s == "5"), + "only the reviewed wbgetentities JSON query is supported" + ); + let ids = params.get("ids").context("missing Wikidata entity IDs")?; + ensure!( + ids.len() <= 1024 + && ids.split('|').count() <= 50 + && ids.split('|').all(|id| { + id.len() > 1 + && id.starts_with('Q') + && id.as_bytes()[1] != b'0' + && id[1..].bytes().all(|b| b.is_ascii_digit()) + }), + "invalid entity selection" + ); + Ok(()) +} + +pub(crate) fn parse_range(value: &str) -> anyhow::Result<(u64, u64, u64)> { + let (bounds, total) = value + .strip_prefix("bytes ") + .context("invalid range unit")? + .split_once('/') + .context("invalid range total")?; + let (start, end) = bounds.split_once('-').context("invalid range bounds")?; + let result = (start.parse()?, end.parse()?, total.parse()?); + ensure!( + result.0 <= result.1 && result.1 < result.2, + "invalid range interval" + ); + Ok(result) +} + +fn reserve_psl_refresh(cache: &Path) -> anyhow::Result<()> { + let daily_lock = OpenOptions::new() // atomic-writes: allow serialize the shared refresh timestamp + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(cache.join("psl/refresh.lock"))?; + daily_lock + .try_lock() + .context("PSL refresh already running")?; + let stamp = cache.join("psl/last-attempt.json"); + if stamp.exists() { + let last: chrono::DateTime = crate::read_json(&stamp)?; + ensure!( + Utc::now() - last >= chrono::Duration::days(1), + "PSL network refresh limited to once per day; reuse the existing pinned snapshot" + ); + } + argand_atomic::replace_durable(&stamp, &serde_json::to_vec(&Utc::now())?)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validator_bound_ranges_and_source_allowlist() -> anyhow::Result<()> { + let url = "https://downloads.majestic.com/majestic_million.csv"; + validate_source_url(Source::Majestic, url)?; + for bad in [ + "http://downloads.majestic.com/majestic_million.csv", + "https://downloads.majestic.com.evil.org/majestic_million.csv", + "https://downloads.majestic.com/other.csv", + "https://user@downloads.majestic.com/majestic_million.csv", + ] { + assert!(validate_source_url(Source::Majestic, bad).is_err()); + } + let response = reqwest::Response::from( + http::Response::builder() + .status(206) + .header("content-range", "bytes 5-9/10") + .header("etag", "\"version1\"") + .body("12345")?, + ); + let state = Resume { + url: url.into(), + etag: "\"version1\"".into(), + total: 10, + }; + assert_eq!(response_extent(&response, Some(&state), 5, url)?.0, 5); + assert!(response_extent(&response, Some(&state), 4, url).is_err()); + assert!(response_extent(&response, None, 5, url).is_err()); + let changed = Resume { + etag: "\"version2\"".into(), + ..state + }; + assert!(response_extent(&response, Some(&changed), 5, url).is_err()); + for bad in ["items 1-2/3", "bytes 4-2/3", "bytes 1-3/3", "bytes 1-2/*"] { + assert!(parse_range(bad).is_err()); + } + Ok(()) + } +} diff --git a/crates/argand-site-registry/src/evidence.rs b/crates/argand-site-registry/src/evidence.rs new file mode 100644 index 0000000..e057b51 --- /dev/null +++ b/crates/argand-site-registry/src/evidence.rs @@ -0,0 +1,125 @@ +// By Nic Weyand! +//! Bounded, source-bearing entity and property metadata for registry consumers. + +use anyhow::Context; +use rusqlite::Connection; +use serde::Serialize; +use serde_json::{Value, json}; + +/// Stable entity identity with language-tagged labels, aliases and metadata. +#[derive(Clone, Debug, Serialize)] +pub struct Entity { + /// Stable source-derived Argand identity. + pub id: String, + /// Deterministic label selection; see the versioned build rule. + pub canonical_name: String, + /// Retained name/alias facts and their complete source declarations. + pub names: Vec, + /// All active name facts, before the output cap of 256. + pub total_names: u64, + /// Country/headquarters/language metadata, distinct from property scope. + pub metadata: Vec, + /// All active metadata facts, before the output cap of 256. + pub total_metadata: u64, +} + +pub(crate) fn fact(db: &Connection, id: &str) -> anyhow::Result { + let (manifest,native,selector,confidence,value,predicate):(String,String,String,u16,String,String)=db.query_row("SELECT s.manifest,r.native_id,f.selector,f.confidence,f.value,f.predicate FROM facts f JOIN sources s ON s.id=f.source_id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE f.id=?1",[id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?)))?; + Ok( + json!({"fact_id":id,"source":serde_json::from_str::(&manifest)?,"source_identifier":native,"selector":selector,"confidence":confidence,"predicate":predicate,"value":serde_json::from_str::(&value)?}), + ) +} + +pub(crate) fn entity(db: &Connection, id: &str, name: &str) -> anyhow::Result { + let (total_names, names) = fields(db, id, true)?; + let (total_metadata, metadata) = fields(db, id, false)?; + Ok(Entity { + id: id.into(), + canonical_name: name.into(), + names, + total_names, + metadata, + total_metadata, + }) +} + +fn fields(db: &Connection, entity: &str, names: bool) -> anyhow::Result<(u64, Vec)> { + let predicate = if names { + "f.predicate='name'" + } else { + "f.predicate IN('P17','P159','P407','P1001','P297','P218','P219','P220')" + }; + let from = format!( + "FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.subject=?1 AND {predicate}" + ); + let total = db.query_row(&format!("SELECT count(*) {from}"), [entity], |r| { + crate::store::unsigned(r, 0) + })?; + let mut statement = db.prepare(&format!("SELECT f.id {from} ORDER BY f.id LIMIT 256"))?; + let mut values = Vec::new(); + for id in statement.query_map([entity], |r| r.get::<_, String>(0))? { + values.push(fact(db, &id?)?); + } + Ok((total, values)) +} + +/// A scoped property assertion. Unknown locale/country stays null; entity-country +/// or audience metadata is never substituted for website jurisdiction. +#[derive(Clone, Debug, Serialize)] +pub struct PropertyScope { + /// Unspecified for source assertions, or the explicitly reviewed role. + pub role: String, + /// Reviewed locale, if any. + pub locale: Option, + /// Reviewed country, if any. + pub country: Option, + /// Original language qualifiers as Wikidata entity IDs. + pub language_entities: Vec, + /// Original jurisdiction qualifiers as Wikidata entity IDs. + pub jurisdiction_entities: Vec, + /// Source fact IDs, or complete operator-decision provenance. + pub provenance: Value, +} + +pub(crate) fn scopes( + evidence: &Value, + provenance: &[Value], + review: Option<&Value>, +) -> anyhow::Result> { + let mut scopes = vec![PropertyScope { + role: "unspecified".into(), + locale: None, + country: None, + language_entities: qualifier_ids(evidence, "P407"), + jurisdiction_entities: qualifier_ids(evidence, "P1001"), + provenance: json!({"fact_ids":provenance.iter().map(|p|&p["fact_id"]).collect::>()}), + }]; + if let Some(review) = review { + let text = |field| review[field].as_str().context("invalid review scope"); + let optional = |field| -> anyhow::Result> { + Ok(Some(text(field)?) + .filter(|s| !s.is_empty()) + .map(str::to_owned)) + }; + scopes.push(PropertyScope { + role: text("role")?.into(), + locale: optional("locale")?, + country: optional("country")?, + language_entities: Vec::new(), + jurisdiction_entities: Vec::new(), + provenance: review.clone(), + }); + } + Ok(scopes) +} + +fn qualifier_ids(evidence: &Value, property: &str) -> Vec { + evidence + .pointer(&format!("/assertion/statement/qualifiers/{property}")) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|v| v.pointer("/datavalue/value/id").and_then(Value::as_str)) + .map(str::to_owned) + .collect() +} diff --git a/crates/argand-site-registry/src/identity.rs b/crates/argand-site-registry/src/identity.rs new file mode 100644 index 0000000..c87e520 --- /dev/null +++ b/crates/argand-site-registry/src/identity.rs @@ -0,0 +1,220 @@ +// By Nic Weyand! +//! Explicit identity equivalences, bound to all names and website assertions. + +use crate::{query::Registry, review::Review}; +use anyhow::ensure; +use chrono::{DateTime, Utc}; +use rusqlite::{Connection, params}; +use serde::Serialize; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; + +/// Proposed identity relationship. Original entity IDs and assertions survive. +#[derive(Clone, Debug, Serialize)] +pub struct Equivalence { + /// Exact pair and evidence fingerprint to review. + pub fingerprint: String, + /// Deterministically ordered source entity IDs. + pub entities: [String; 2], + /// Digests of every current name and website assertion on each entity. + pub signatures: [String; 2], + /// Bounded entity metadata with its original source provenance. + pub evidence: [crate::evidence::Entity; 2], +} + +/// Previews an explicit equivalence without changing the registry. +/// +/// # Errors +/// Rejects equal or unknown entities and malformed evidence. +pub fn propose(registry: &Registry, left: &str, right: &str) -> anyhow::Result { + ensure!( + left != right, + "identity equivalence needs distinct entity IDs" + ); + let mut entities = [left.to_owned(), right.to_owned()]; + entities.sort(); + let signatures = [ + signature(®istry.db, &entities[0])?, + signature(®istry.db, &entities[1])?, + ]; + let mut names = Vec::new(); + for entity in &entities { + let name: String = registry.db.query_row( + "SELECT canonical_name FROM entities WHERE id=?1", + [entity], + |r| r.get(0), + )?; + names.push(crate::evidence::entity(®istry.db, entity, &name)?); + } + Ok(Equivalence { + fingerprint: crate::digest(&serde_json::to_vec(&( + &entities, + &signatures, + crate::store::RULE_VERSION, + ))?), + entities, + signatures, + evidence: [names.remove(0), names.remove(0)], + }) +} + +/// Appends a reviewed equivalence or revocation to the same signed decision log. +/// Both complete source identities must exist in the writer store. Rebuild to use. +/// +/// # Errors +/// Rejects stale fingerprints, scoped identity decisions or missing evidence. +pub fn record( + db: &Connection, + registry: &Registry, + pair: &Equivalence, + review: &Review, +) -> anyhow::Result { + crate::review::validate(review)?; + let expected = propose(registry, &pair.entities[0], &pair.entities[1])?; + ensure!( + review.fingerprint == expected.fingerprint && pair.fingerprint == expected.fingerprint, + "identity review fingerprint differs from current evidence" + ); + ensure!( + review.role == "unspecified" && review.locale.is_empty() && review.country.is_empty(), + "identity reviews cannot assert a destination role" + ); + for entity in &expected.entities { + let mut statement = registry.db.prepare("SELECT DISTINCT f.source_id FROM facts f JOIN selected_sources s ON s.id=f.source_id WHERE f.subject=?1")?; + for id in statement.query_map([entity], |r| r.get::<_, String>(0))? { + crate::store::source(db, &id?)?; + } + } + let transaction = db.unchecked_transaction()?; + db.execute( + "INSERT OR IGNORE INTO equivalences VALUES(?1,?2,?3,?4,?5)", + params![ + expected.fingerprint, + expected.entities[0], + expected.entities[1], + expected.signatures[0], + expected.signatures[1] + ], + )?; + let sequence = crate::review::append(db, review)?; + transaction.commit()?; + Ok(sequence) +} + +fn signature(db: &Connection, entity: &str) -> anyhow::Result { + let name: String = db.query_row( + "SELECT names_fingerprint FROM entities WHERE id=?1", + [entity], + |r| r.get(0), + )?; + let mut hash = Sha256::new(); + hash.update(name.as_bytes()); + let mut statement = + db.prepare("SELECT fingerprint FROM edges WHERE entity=?1 ORDER BY fingerprint")?; + for edge in statement.query_map([entity], |r| r.get::<_, String>(0))? { + hash.update(edge?.as_bytes()); + } + Ok(format!("{:x}", hash.finalize())) +} + +pub(crate) fn expand( + registry: &Registry, + initial: &str, + now: DateTime, +) -> anyhow::Result, Vec)>> { + let mut entities = BTreeSet::from([initial.to_owned()]); + let mut pending = vec![initial.to_owned()]; + let mut signatures = BTreeMap::new(); + let mut evidence = BTreeMap::new(); + while let Some(entity) = pending.pop() { + let mut statement=registry.db.prepare("SELECT e.fingerprint,e.left_entity,e.right_entity,e.left_signature,e.right_signature,r.sequence,r.reviewer,r.reason,r.evidence,r.reviewed_at,r.expires_at FROM equivalences e JOIN reviews r ON r.fingerprint=e.fingerprint WHERE (e.left_entity=?1 OR e.right_entity=?1) AND r.sequence=(SELECT max(sequence) FROM reviews WHERE fingerprint=e.fingerprint) AND r.decision='approve' ORDER BY e.fingerprint")?; + let mut rows = statement.query([&entity])?; + while let Some(row) = rows.next()? { + let started: String = row.get(9)?; + let expires: String = row.get(10)?; + if now < DateTime::parse_from_rfc3339(&started)? + || now >= DateTime::parse_from_rfc3339(&expires)? + { + continue; + } + let pair = [row.get::<_, String>(1)?, row.get::<_, String>(2)?]; + let mut valid = true; + for (i, id) in pair.iter().enumerate() { + if !signatures.contains_key(id) { + let current = match signature(®istry.db, id) { + Ok(signature) => Some(signature), + Err(error) + if matches!( + error.downcast_ref::(), + Some(rusqlite::Error::QueryReturnedNoRows) + ) => + { + None + } + Err(error) => return Err(error), + }; + signatures.insert(id.clone(), current); + } + if signatures.get(id).and_then(Option::as_ref) + != Some(&row.get::<_, String>(i + 3)?) + { + valid = false; + break; + } + } + if !valid { + continue; + } + let fingerprint: String = row.get(0)?; + evidence.insert(fingerprint.clone(),json!({"fingerprint":fingerprint,"entities":pair,"source":"argand_operator_review","source_identifier":row.get::<_,i64>(5)?,"reviewer":row.get::<_,String>(6)?,"reason":row.get::<_,String>(7)?,"evidence":row.get::<_,String>(8)?,"retrieved_at":started,"expires_at":expires,"license":"CC0-1.0","confidence":9000})); + for id in pair { + if entities.insert(id.clone()) { + pending.push(id); + } + } + if entities.len() > 64 || evidence.len() > 256 { + return Ok(None); + } + } + } + Ok(Some((entities, evidence.into_values().collect()))) +} + +pub(crate) fn candidates( + registry: &Registry, + query: &str, + now: DateTime, +) -> anyhow::Result> { + let key = crate::normalize::name_key(query)?; + let mut statement = registry + .db + .prepare("SELECT DISTINCT entity FROM names WHERE key=?1 ORDER BY entity LIMIT 65")?; + let matched = statement + .query_map([key], |r| r.get::<_, String>(0))? + .collect::, _>>()?; + if matched.is_empty() || matched.len() > 64 { + return Ok(Vec::new()); + } + let Some((entities, evidence)) = expand(registry, &matched[0], now)? else { + return Ok(Vec::new()); + }; + if matched.iter().any(|id| !entities.contains(id)) { + return Ok(Vec::new()); + } + let mut output = Vec::new(); + for entity in entities { + let mut statement = registry.db.prepare( + "SELECT fingerprint FROM edges WHERE entity=?1 ORDER BY fingerprint LIMIT 101", + )?; + for fingerprint in statement.query_map([entity], |r| r.get::<_, String>(0))? { + let mut candidate = registry.candidate(&fingerprint?)?; + candidate.identity_provenance.clone_from(&evidence); + output.push(candidate); + if output.len() > 100 { + return Ok(Vec::new()); + } + } + } + Ok(output) +} diff --git a/crates/argand-site-registry/src/json.rs b/crates/argand-site-registry/src/json.rs new file mode 100644 index 0000000..5dccf4d --- /dev/null +++ b/crates/argand-site-registry/src/json.rs @@ -0,0 +1,76 @@ +// By Nic Weyand! +//! Reject duplicate object keys instead of silently losing conflicting values. + +use serde::de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}; +use serde_json::{Map, Number, Value}; +use std::fmt; + +struct Unique(Value); + +impl<'de> Deserialize<'de> for Unique { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(UniqueVisitor) + } +} + +struct UniqueVisitor; +impl<'de> Visitor<'de> for UniqueVisitor { + type Value = Unique; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("JSON with unique object keys") + } + fn visit_bool(self, v: bool) -> Result { + Ok(Unique(Value::Bool(v))) + } + fn visit_i64(self, v: i64) -> Result { + Ok(Unique(Value::Number(v.into()))) + } + fn visit_u64(self, v: u64) -> Result { + Ok(Unique(Value::Number(v.into()))) + } + fn visit_f64(self, v: f64) -> Result { + Number::from_f64(v) + .map(|n| Unique(Value::Number(n))) + .ok_or_else(|| E::custom("nonfinite JSON number")) + } + fn visit_str(self, v: &str) -> Result { + Ok(Unique(Value::String(v.into()))) + } + fn visit_unit(self) -> Result { + Ok(Unique(Value::Null)) + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut values = Vec::new(); + while let Some(Unique(v)) = seq.next_element()? { + values.push(v); + } + Ok(Unique(Value::Array(values))) + } + fn visit_map>(self, mut map: A) -> Result { + let mut values = Map::new(); + while let Some((key, Unique(value))) = map.next_entry::()? { + if values.insert(key, value).is_some() { + return Err(de::Error::custom("duplicate JSON object key")); + } + } + Ok(Unique(Value::Object(values))) + } +} + +pub(crate) fn parse(bytes: &[u8]) -> anyhow::Result { + Ok(serde_json::from_slice::(bytes)?.0) +} + +#[cfg(test)] +mod tests { + #[test] + fn duplicate_nested_keys_are_rejected() -> anyhow::Result<()> { + assert!(super::parse(br#"{"a":{"url":"good","url":"bad"}}"#).is_err()); + let bytes = br#"{"a":[true,null,42,-8,0.2,"text"]}"#; + assert_eq!( + super::parse(bytes)?, + serde_json::from_slice::(bytes)? + ); + Ok(()) + } +} diff --git a/crates/argand-site-registry/src/lib.rs b/crates/argand-site-registry/src/lib.rs new file mode 100644 index 0000000..d327d77 --- /dev/null +++ b/crates/argand-site-registry/src/lib.rs @@ -0,0 +1,58 @@ +// By Nic Weyand! +//! Source-separated website assertions and reviewed, immutable registry releases. + +pub mod adapters; +pub mod build; +pub mod crux; +pub mod download; +pub mod evidence; +pub mod identity; +mod json; +pub mod model; +pub mod normalize; +pub mod observation; +pub mod query; +pub mod release; +pub mod review; +pub mod store; +pub mod update; + +use sha2::{Digest, Sha256}; +use std::{fs::File, io::Read, path::Path}; + +/// Hashes bytes with SHA-256, using lowercase hexadecimal. +#[must_use] +pub fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +/// Hashes a file with bounded memory. +/// +/// # Errors +/// Returns filesystem errors. +pub fn file_digest(path: &Path) -> anyhow::Result { + let mut file = File::open(path)?; + let mut hash = Sha256::new(); + let mut buffer = [0; 8192]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + hash.update(&buffer[..count]); + } + Ok(format!("{:x}", hash.finalize())) +} + +/// Reads bounded JSON metadata, never an unbounded dataset. +/// +/// # Errors +/// Returns malformed, oversized, or unreadable input errors. +pub fn read_json(path: &Path) -> anyhow::Result { + let mut bytes = Vec::new(); + File::open(path)? + .take(1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + anyhow::ensure!(bytes.len() <= 1024 * 1024, "metadata exceeds 1 MiB"); + Ok(serde_json::from_value(json::parse(&bytes)?)?) +} diff --git a/crates/argand-site-registry/src/main.rs b/crates/argand-site-registry/src/main.rs new file mode 100644 index 0000000..94a12e3 --- /dev/null +++ b/crates/argand-site-registry/src/main.rs @@ -0,0 +1,8 @@ +// By Nic Weyand! +//! Native Site Registry CLI. +mod cli; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + cli::run().await +} diff --git a/crates/argand-site-registry/src/model.rs b/crates/argand-site-registry/src/model.rs new file mode 100644 index 0000000..62b34cc --- /dev/null +++ b/crates/argand-site-registry/src/model.rs @@ -0,0 +1,214 @@ +// By Nic Weyand! +//! Source identities and source-native assertion envelopes. + +use anyhow::ensure; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Only sources with reviewed commercial reuse terms are implemented. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, clap::ValueEnum, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum Source { + /// Wikidata structured data. + Wikidata, + /// Majestic Million domain ranks. + Majestic, + /// Chrome User Experience Report origin ranks. + Crux, + /// Curlie directory entries and categories. + Curlie, + /// Public Suffix List, including PRIVATE rules. + Psl, +} + +impl Source { + /// Stable source namespace. + #[must_use] + pub const fn key(self) -> &'static str { + match self { + Self::Wikidata => "wikidata", + Self::Majestic => "majestic", + Self::Crux => "crux", + Self::Curlie => "curlie", + Self::Psl => "psl", + } + } + /// Exact SPDX data license. + #[must_use] + pub const fn license(self) -> &'static str { + match self { + Self::Wikidata => "CC0-1.0", + Self::Majestic | Self::Curlie => "CC-BY-3.0", + Self::Crux => "CC-BY-4.0", + Self::Psl => "MPL-2.0", + } + } + /// Authoritative license evidence page. + #[must_use] + pub const fn license_url(self) -> &'static str { + match self { + Self::Wikidata => "https://www.wikidata.org/wiki/Wikidata:Licensing", + Self::Majestic => "https://majestic.com/reports/majestic-million", + Self::Crux => "https://developer.chrome.com/docs/crux/methodology", + Self::Curlie => "https://curlie.org/docs/en/license.html", + Self::Psl => "https://publicsuffix.org/list/public_suffix_list.dat", + } + } +} + +/// Source format is explicit; unsupported revisions fail closed. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, clap::ValueEnum, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum Format { + /// Official JSON array, one entity per line. + WikidataDump, + /// Official Special:EntityData/API entities object. + WikidataEntities, + /// Header-bearing Majestic CSV. + MajesticCsv, + /// Registry's documented `BigQuery` projection: `origin,rank,yyyymm,country_code`. + CruxCsv, + /// Current Curlie tar.gz containing literal TSV files. + CurlieTarGz, + /// UTF-8 PSL text. + PslText, +} + +/// Compression of the downloaded object (Curlie tar.gz uses `None` here). +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, clap::ValueEnum)] +#[serde(rename_all = "snake_case")] +pub enum Compression { + /// Uncompressed, or intrinsically compressed archive format. + #[default] + None, + /// Concatenated gzip members. + Gzip, + /// Concatenated bzip2 streams. + Bzip2, +} + +/// An immutable, externally auditable input declaration; local paths are separate. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct SourceManifest { + /// Must be `argand.site-source/v1`. + pub schema: String, + /// Approved provider. + pub source: Source, + /// Exact source format. + pub format: Format, + /// Outer object compression. + #[serde(default)] + pub compression: Compression, + /// Source-native snapshot/revision identifier. + pub snapshot: String, + /// Replacement scope, e.g. `full` or `selection:facebook`. + pub scope: String, + /// Original distribution URL, not an arbitrary mirror. + pub source_url: String, + /// Exact data license identifier. + pub license: String, + /// Authoritative license evidence URL. + pub license_url: String, + /// Actual retrieval time, distinct from the observation period. + pub retrieved_at: DateTime, + /// Hash over the original compressed bytes. + pub sha256: String, + /// Original compressed object length. + pub bytes: u64, +} + +impl SourceManifest { + /// Checks source, license, identity, and format consistency. + /// + /// # Errors + /// Returns a descriptive error for unsupported source declarations. + pub fn validate(&self) -> anyhow::Result<()> { + ensure!( + self.schema == "argand.site-source/v1", + "unsupported source schema" + ); + ensure!( + self.license == self.source.license() && self.license_url == self.source.license_url(), + "source license evidence mismatch" + ); + ensure!( + self.bytes > 0 && valid_digest(&self.sha256), + "invalid source length or digest" + ); + ensure!( + !self.snapshot.is_empty() + && self.snapshot.len() <= 512 + && !self.scope.is_empty() + && self.scope.len() <= 128, + "invalid snapshot or replacement scope" + ); + let valid = matches!( + (self.source, self.format), + ( + Source::Wikidata, + Format::WikidataDump | Format::WikidataEntities + ) | (Source::Majestic, Format::MajesticCsv) + | (Source::Crux, Format::CruxCsv) + | (Source::Curlie, Format::CurlieTarGz) + | (Source::Psl, Format::PslText) + ); + ensure!(valid, "source/format mismatch"); + crate::download::validate_source_url(self.source, &self.source_url)?; + Ok(()) + } + /// Content identity of the declaration, including original retrieval time. + /// + /// # Errors + /// Returns a serialization error. + pub fn id(&self) -> anyhow::Result { + Ok(crate::digest(&serde_json::to_vec(self)?)) + } +} + +/// A source assertion. `value` retains source-native qualifiers and references. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Fact { + /// Source-namespaced subject; never inferred from a similar name. + pub subject: String, + /// Typed predicate understood by the projection, or a retained metadata key. + pub predicate: String, + /// Structured source value. + pub value: Value, + /// Source record JSON pointer or TSV column selector. + pub selector: String, + /// Evidence confidence on a 0..10000 policy scale, not calibrated probability. + pub confidence: u16, +} + +/// Common adapter output. Records are retained even if every URL is rejected. +#[derive(Clone, Debug)] +pub struct Record { + /// Source-native entity ID or member/row locator. + pub native_id: String, + /// Complete relevant source record; no flattened evidence loss. + pub raw: Value, + /// Assertions derived from this record. + pub facts: Vec, +} + +/// Stable identifier for a source entity independent of names and revisions. +#[must_use] +pub fn entity_id(source: Source, native: &str) -> String { + format!( + "argand:entity:{}:{}", + source.key(), + crate::digest(native.as_bytes()) + ) +} + +/// Whether a string is a full lowercase SHA-256. +#[must_use] +pub fn valid_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} diff --git a/crates/argand-site-registry/src/normalize.rs b/crates/argand-site-registry/src/normalize.rs new file mode 100644 index 0000000..171e156 --- /dev/null +++ b/crates/argand-site-registry/src/normalize.rs @@ -0,0 +1,163 @@ +// By Nic Weyand! +//! Strict registry URL identity, independent of search document equivalences. + +use anyhow::{Context, ensure}; +use publicsuffix::{List, Psl}; +use serde::{Deserialize, Serialize}; +use unicode_normalization::UnicodeNormalization; +use url::{Host, Url}; + +/// PSL-derived information; `psl_source` binds every derived field to its input. +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] +pub struct Domain { + /// ASCII IDNA hostname. + pub hostname: String, + /// Registrable domain using ICANN and PRIVATE sections. + pub registrable_domain: String, + /// Effective public suffix. + pub public_suffix: String, + /// Whether the matched rule is in the PRIVATE section. + pub private_suffix: bool, + /// Immutable source manifest identity. + pub psl_source: String, + /// Content digest, stable when retrieval time changes but list bytes do not. + pub psl_sha256: String, +} + +/// Strict URL identity and its derived domain fields. +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] +pub struct WebProperty { + /// Stable URL-derived Argand ID, unaffected by PSL refresh. + pub id: String, + /// Normalized absolute URL; meaningful paths and queries are preserved. + pub url: String, + /// Domain evidence. + pub domain: Domain, +} + +/// A pinned dynamic PSL. It never refreshes during an import or query. +pub struct Normalizer { + list: List, + source: String, + content_sha256: String, +} + +impl Normalizer { + /// Constructs the parser from an independently pinned source. + /// + /// # Errors + /// Rejects malformed, empty, or unsectioned lists. + pub fn new(bytes: &[u8], source: String) -> anyhow::Result { + let text = std::str::from_utf8(bytes)?; + ensure!( + text.contains("===BEGIN ICANN DOMAINS===") + && text.contains("===END PRIVATE DOMAINS==="), + "incomplete PSL sections" + ); + let list = List::from_bytes(bytes).map_err(|e| anyhow::anyhow!("invalid PSL: {e}"))?; + ensure!(!list.is_empty(), "empty PSL"); + Ok(Self { + list, + source, + content_sha256: crate::digest(bytes), + }) + } + + /// Parses a hostname without inventing a website URL for popularity rows. + /// + /// # Errors + /// Rejects invalid, special-use, IP, and suffix-only hosts. + pub fn domain(&self, input: &str) -> anyhow::Result { + ensure!( + !input.is_empty() + && input.len() <= 1024 + && !input.ends_with("..") + && !input.contains(['/', ':', '@', '\\', '?', '#']) + && !input.chars().any(char::is_whitespace), + "invalid hostname" + ); + let host = match Host::parse(input.trim_end_matches('.'))? { + Host::Domain(host) => host.to_ascii_lowercase(), + _ => anyhow::bail!("IP literals are not registry destinations"), + }; + ensure!( + host.len() <= 253 + && host.split('.').all(|label| !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-')), + "invalid DNS name" + ); + ensure!( + !["localhost", "local", "internal", "test", "invalid", "onion"] + .iter() + .any(|suffix| host == *suffix || host.ends_with(&format!(".{suffix}"))), + "special-use hostname" + ); + let domain = self + .list + .domain(host.as_bytes()) + .context("hostname is a public suffix")?; + ensure!(domain.suffix().is_known(), "unknown public suffix"); + Ok(Domain { + registrable_domain: std::str::from_utf8(domain.as_bytes())?.to_owned(), + public_suffix: std::str::from_utf8(domain.suffix().as_bytes())?.to_owned(), + private_suffix: domain.suffix().typ() == Some(publicsuffix::Type::Private), + hostname: host, + psl_source: self.source.clone(), + psl_sha256: self.content_sha256.clone(), + }) + } + + /// Normalizes syntax without assuming HTTP/HTTPS, www, or path equivalence. + /// + /// # Errors + /// Rejects malformed, credential-bearing, special-use and non-HTTP(S) URLs. + pub fn url(&self, input: &str) -> anyhow::Result { + ensure!( + input + .split_once("://") + .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("http") + || scheme.eq_ignore_ascii_case("https")), + "URL needs an explicit HTTP(S) authority" + ); + ensure!( + input.len() <= 8192 + && !input.chars().any(|c| c.is_control() || c.is_whitespace()) + && !input.contains('\\'), + "invalid URL characters or length" + ); + let mut url = Url::parse(input)?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.username().is_empty() + && url.password().is_none(), + "unsupported scheme or credentials" + ); + let domain = self.domain(url.host_str().context("missing hostname")?)?; + url.set_host(Some(&domain.hostname))?; + url.set_fragment(None); + let normalized = url.to_string(); + Ok(WebProperty { + id: format!("argand:web:{}", crate::digest(normalized.as_bytes())), + url: normalized, + domain, + }) + } +} + +/// Same NFC/lowercase/space rule as the existing Rust navigation consumer. +/// Original names remain in the assertion store. Confusables are never folded. +/// +/// # Errors +/// Rejects controls, invisible directional text, and oversized names. +pub fn name_key(input: &str) -> anyhow::Result { + ensure!(input.len() <= 4096 && !input.chars().any(|c| c.is_control() || matches!(c, '\u{00ad}' | '\u{061c}' | '\u{200b}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2060}'..='\u{206f}' | '\u{feff}')), "unsafe name characters or length"); + let lower: String = input.nfc().flat_map(char::to_lowercase).collect(); + let key = lower.split_whitespace().collect::>().join(" "); + ensure!(!key.is_empty(), "empty name"); + Ok(key) +} diff --git a/crates/argand-site-registry/src/observation.rs b/crates/argand-site-registry/src/observation.rs new file mode 100644 index 0000000..c3c5409 --- /dev/null +++ b/crates/argand-site-registry/src/observation.rs @@ -0,0 +1,60 @@ +// By Nic Weyand! +//! Extension contract for later crawler evidence. No network or ownership inference. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// Observed relationship, distinct from an entity-ownership claim. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ObservationKind { + /// An actual HTTP redirect with its status code. + Redirect { + /// HTTP redirect status. + status: u16, + }, + /// A page's declared canonical link. + Canonical, + /// A page's declared alternate locale. + Hreflang { + /// Unmodified declared locale. + locale: String, + }, + /// A JSON-LD sameAs assertion, not proof of ownership. + JsonLdSameAs, + /// A URL actually present in a fetched sitemap. + Sitemap, + /// A site-provided country-selector link. + CountrySelector { + /// Country as declared by the site. + country: String, + }, +} + +/// Immutable evidence coordinates for a future crawler-source adapter. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Observation { + /// Relationship type. + pub relation: ObservationKind, + /// Actual page that asserted the relationship. + pub from_url: String, + /// Exact observed target, resolved against the recorded document base. + pub to_url: String, + /// Source provider/capture collection. + pub source: String, + /// Immutable source-native capture identifier. + pub source_identifier: String, + /// License or rights declaration verified for this evidence. + pub license: String, + /// License evidence URL. + pub license_url: String, + /// Retrieval instant. + pub retrieved_at: DateTime, + /// Captured source-content hash. + pub content_sha256: String, + /// JSON pointer, header name, or DOM selector for the assertion. + pub selector: String, + /// Confidence on the same 0..10000 policy scale as source facts. + pub confidence: u16, +} diff --git a/crates/argand-site-registry/src/query.rs b/crates/argand-site-registry/src/query.rs new file mode 100644 index 0000000..d31f298 --- /dev/null +++ b/crates/argand-site-registry/src/query.rs @@ -0,0 +1,332 @@ +// By Nic Weyand! +//! Bounded native lookup; ambiguity is counted before limits or policy filtering. + +use crate::{build::Receipt, normalize::name_key}; +use anyhow::{Context, ensure}; +use chrono::{DateTime, Utc}; +use rusqlite::{Connection, OptionalExtension, params}; +use serde::Serialize; +use serde_json::{Value, json}; +use std::path::Path; + +/// Open verified generation. Hashes are checked once, outside the query path. +pub struct Registry { + pub(crate) db: Connection, + /// External receipt pin supplied by the caller. + pub identity: String, + /// Verified manifest. + pub receipt: Receipt, +} + +/// One entity/property assertion with provenance and review state. +#[derive(Clone, Debug, Serialize)] +pub struct Candidate { + /// Explicit active identity reviews used to connect source entities. + pub identity_provenance: Vec, + /// Canonical name, aliases and entity metadata with their source facts. + pub entity: crate::evidence::Entity, + /// Stable entity identity. + pub entity_id: String, + /// Deterministic canonical display name. + pub canonical_name: String, + /// Exact proposed destination. + pub url: String, + /// URL, hostname, registrable-domain and pinned PSL provenance. + pub web_property: Value, + /// Conflicting source/reviewer scope assertions, without destructive merging. + pub property_scopes: Vec, + /// Assertion relation, never a rank-derived ownership inference. + pub relation: String, + /// Conservative minimum source-assertion confidence, separate from review. + pub confidence: u16, + /// Exact evidence fingerprint to review. + pub fingerprint: String, + /// Statement qualifiers and identity evidence. + pub evidence: Value, + /// Every source fact supporting this assertion. + pub provenance: Vec, + /// Latest review, including expiry and its own provenance declaration. + pub review: Option, + /// Whether this assertion can be considered for approval. + pub eligible: bool, +} + +/// Audit result. Counts include all matches before the output limit. +#[derive(Debug, Serialize)] +pub struct Lookup { + /// Normalized query. + pub query: String, + /// Distinct matching entities, including rejected/expired alternatives. + pub total_entities: u64, + /// Total assertion groups across matching entities. + pub total_edges: u64, + /// True when the output limit omits assertions. + pub truncated: bool, + /// Bounded candidate list. + pub candidates: Vec, + /// License attribution required when displaying imported names. + pub attribution: Value, +} + +/// Complete exact-name alternatives, independent of display limits or reviews. +/// The identity binds the whole immutable registry and all matching name/edge +/// fingerprints. It is evidence for query review, not a destination approval. +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +pub struct SelectionContext { + /// SHA-256 of the complete normalized alternative context. + pub identity: String, + /// Versioned native name key. + pub normalized_query: String, + /// Matching entities, including those without an eligible website. + pub total_entities: u64, + /// All matching assertion edges, before any output limit. + pub total_edges: u64, + /// Whether the requested fingerprint belongs to a matching entity. + pub selected_assertion_present: bool, +} + +impl Registry { + /// Streams the complete alternative context for an explicitly selected edge. + /// + /// # Errors + /// Rejects malformed keys/fingerprints and corrupt registry data. + pub fn selection_context( + &self, + query: &str, + fingerprint: &str, + ) -> anyhow::Result { + use sha2::{Digest, Sha256}; + ensure!( + crate::model::valid_digest(fingerprint), + "invalid selection fingerprint" + ); + let key = name_key(query)?; + let mut hash = Sha256::new(); + hash.update(serde_json::to_vec(&( + "argand.site-selection/v1", + &self.identity, + &key, + ))?); + let mut statement = self.db.prepare( + "SELECT n.id,n.names_fingerprint,e.fingerprint FROM entities n LEFT JOIN edges e ON e.entity=n.id WHERE n.id IN(SELECT entity FROM names WHERE key=?1) ORDER BY n.id,e.fingerprint" + )?; + let mut rows = statement.query([&key])?; + let mut previous = None; + let mut total_entities = 0; + let mut total_edges = 0; + let mut present = false; + while let Some(row) = rows.next()? { + let entity: String = row.get(0)?; + let names: String = row.get(1)?; + let edge: Option = row.get(2)?; + hash.update(serde_json::to_vec(&(&entity, names, &edge))?); + if previous.as_ref() != Some(&entity) { + total_entities += 1; + previous = Some(entity); + } + if let Some(edge) = edge { + total_edges += 1; + present |= edge == fingerprint; + } + } + Ok(SelectionContext { + identity: format!("{:x}", hash.finalize()), + normalized_query: key, + total_entities, + total_edges, + selected_assertion_present: present, + }) + } + + /// Opens only a complete, externally pinned generation. + /// + /// # Errors + /// Rejects altered receipts/databases and unsupported contracts. + pub fn open(path: &Path, expected_pin: &str) -> anyhow::Result { + ensure!( + crate::model::valid_digest(expected_pin), + "provide a full trusted receipt SHA-256" + ); + ensure!( + crate::file_digest(&path.join("COMPLETE.json"))? == expected_pin, + "receipt pin mismatch" + ); + let receipt: Receipt = crate::read_json(&path.join("COMPLETE.json"))?; + ensure!( + crate::file_digest(&path.join("LICENSE_SOURCES.md"))? == receipt.licenses_sha256 + && crate::file_digest(&path.join("ATTRIBUTION.json"))? + == receipt.attribution_sha256, + "registry license or attribution digest mismatch" + ); + ensure!( + receipt.schema == "argand.site-registry/v1" + && receipt.rules == crate::store::RULE_VERSION, + "unsupported registry contract" + ); + let database = path.join("registry.sqlite"); + ensure!( + std::fs::symlink_metadata(&database)?.is_file(), + "database must be a regular file" + ); + ensure!( + crate::file_digest(&database)? == receipt.database_sha256, + "registry database digest mismatch" + ); + let db = Connection::open_with_flags(database, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?; + crate::store::configure(&db)?; + Ok(Self { + db, + identity: expected_pin.into(), + receipt, + }) + } + + /// Indexed exact-name/alias lookup, with counts independent of result limits. + /// + /// # Errors + /// Returns invalid query bounds, malformed data, or SQLite errors. + pub fn lookup(&self, query: &str, limit: u32) -> anyhow::Result { + ensure!((1..=100).contains(&limit), "lookup limit must be 1..100"); + let key = name_key(query)?; + let total_entities = self.db.query_row( + "SELECT count(DISTINCT entity) FROM names WHERE key=?1", + [&key], + |r| crate::store::unsigned(r, 0), + )?; + let total_edges = self.db.query_row( + "SELECT count(*) FROM edges WHERE entity IN(SELECT entity FROM names WHERE key=?1)", + [&key], + |r| crate::store::unsigned(r, 0), + )?; + let mut stmt=self.db.prepare("SELECT fingerprint FROM edges WHERE entity IN(SELECT entity FROM names WHERE key=?1) ORDER BY entity,property,fingerprint LIMIT ?2")?; + let ids = stmt + .query_map(params![key, limit], |r| r.get::<_, String>(0))? + .collect::, _>>()?; + let candidates = ids + .iter() + .map(|id| self.candidate(id)) + .collect::>>()?; + Ok(Lookup { + query: key, + total_entities, + total_edges, + truncated: total_edges > u64::from(limit), + candidates, + attribution: crate::release::attribution(), + }) + } + + /// Returns one assertion and its complete supporting source declarations. + /// + /// # Errors + /// Fails for absent fingerprints or corrupt projection data. + pub fn candidate(&self, fingerprint: &str) -> anyhow::Result { + let (entity,name,url,property,relation,evidence,eligible,facts):(String,String,String,String,String,String,bool,String)=self.db.query_row("SELECT e.entity,n.canonical_name,p.url,p.derived_json,e.relation,e.evidence,e.eligible,e.facts FROM edges e JOIN entities n ON n.id=e.entity JOIN properties p ON p.id=e.property WHERE e.fingerprint=?1",[fingerprint],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?,r.get(7)?)))?; + let ids: Vec = serde_json::from_str(&facts)?; + let mut provenance = Vec::new(); + for id in ids { + provenance.push(crate::evidence::fact(&self.db, &id)?); + } + let review=self.db.query_row("SELECT sequence,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country FROM reviews WHERE fingerprint=?1 ORDER BY sequence DESC LIMIT 1",[fingerprint],|r|Ok(json!({"sequence":crate::store::unsigned(r,0)?,"decision":r.get::<_,String>(1)?,"reviewer":r.get::<_,String>(2)?,"reason":r.get::<_,String>(3)?,"evidence":r.get::<_,String>(4)?,"retrieved_at":r.get::<_,String>(5)?,"expires_at":r.get::<_,String>(6)?,"role":r.get::<_,String>(7)?,"locale":r.get::<_,String>(8)?,"country":r.get::<_,String>(9)?,"source":"argand_operator_review","source_identifier":format!("{}:{}",fingerprint,crate::store::unsigned(r,0)?),"license":"CC0-1.0","license_url":"https://creativecommons.org/publicdomain/zero/1.0/","confidence":9000}))).optional()?; + Ok(Candidate { + identity_provenance: Vec::new(), + entity: crate::evidence::entity(&self.db, &entity, &name)?, + property_scopes: crate::evidence::scopes( + &serde_json::from_str(&evidence)?, + &provenance, + review.as_ref(), + )?, + entity_id: entity, + canonical_name: name, + url, + web_property: serde_json::from_str(&property)?, + relation, + confidence: provenance + .iter() + .filter_map(|p| p["confidence"].as_u64()) + .min() + .map(u16::try_from) + .transpose()? + .unwrap_or(0), + fingerprint: fingerprint.into(), + evidence: serde_json::from_str(&evidence)?, + provenance, + review, + eligible, + }) + } + + /// Chooses an approved regional property or an explicitly reviewed primary. + /// Never returns a destination for ambiguous entities or tied properties. + /// + /// # Errors + /// Returns malformed requests/data or database failures. + pub fn resolve( + &self, + query: &str, + locale: Option<&str>, + country: Option<&str>, + now: DateTime, + ) -> anyhow::Result> { + let candidates = crate::identity::candidates(self, query, now)?; + let mut best = None; + let mut score = 0; + let mut ambiguous = false; + for candidate in candidates { + if !candidate.eligible { + continue; + } + let Some(review) = candidate.review.as_ref() else { + continue; + }; + if review["decision"] != "approve" { + continue; + } + let expires = DateTime::parse_from_rfc3339( + review["expires_at"] + .as_str() + .context("invalid review expiry")?, + )?; + let starts = DateTime::parse_from_rfc3339( + review["retrieved_at"] + .as_str() + .context("invalid review timestamp")?, + )?; + if now < starts || now >= expires { + continue; + } + let region = review["country"].as_str().unwrap_or_default(); + let language = review["locale"].as_str().unwrap_or_default(); + let matches_country = + !region.is_empty() && country.is_some_and(|c| c.eq_ignore_ascii_case(region)); + let matches_locale = + !language.is_empty() && locale.is_some_and(|l| l.eq_ignore_ascii_case(language)); + let current = if review["role"] == "regional" { + // All asserted dimensions must match; a language alone cannot + // override an explicit country mismatch. + if (!region.is_empty() && !matches_country) + || (!language.is_empty() && !matches_locale) + { + continue; + } + 2 + u8::from(matches_country) + u8::from(matches_locale) + } else if review["role"] == "primary" { + 1 + } else { + continue; + }; + if current > score { + best = Some(candidate); + score = current; + ambiguous = false; + } else if current == score + && best + .as_ref() + .is_some_and(|b: &Candidate| b.url != candidate.url) + { + ambiguous = true; + } + } + Ok(if ambiguous { None } else { best }) + } +} diff --git a/crates/argand-site-registry/src/release.rs b/crates/argand-site-registry/src/release.rs new file mode 100644 index 0000000..ad15b4f --- /dev/null +++ b/crates/argand-site-registry/src/release.rs @@ -0,0 +1,237 @@ +// By Nic Weyand! +//! Provenance-bearing export and externally authenticated release activation. + +use crate::query::Registry; +use anyhow::{Context, ensure}; +use serde_json::{Value, json}; +use std::{ + fs::{self, File, OpenOptions}, + io::{BufWriter, Write}, + path::Path, + process::{Command, Stdio}, +}; + +/// Source terms shipped and authenticated with every generation. +pub const LICENSES: &str = include_str!("../LICENSE_SOURCES.md"); + +/// Source attribution envelope for CLI and JSON consumers. +#[must_use] +pub fn attribution() -> Value { + json!({"wikidata":{"license":"CC0-1.0","url":"https://www.wikidata.org/"}, + "majestic":{"license":"CC-BY-3.0","credit":"Majestic Million, Majestic","url":"https://majestic.com/reports/majestic-million","license_url":"https://creativecommons.org/licenses/by/3.0/"}, + "crux":{"license":"CC-BY-4.0","credit":"Chrome UX Report, Google","url":"https://developer.chrome.com/docs/crux/","license_url":"https://creativecommons.org/licenses/by/4.0/"}, + "curlie":{"license":"CC-BY-3.0","credit":"With content from Curlie.org - the largest human-edited directory of the web. Contribute by submitting a website or becoming an editor.","url":"https://curlie.org/","license_url":"https://creativecommons.org/licenses/by/3.0/","public_display":"Use the prescribed HTML attribution on every page using Curlie content: https://curlie.org/docs/en/license.html"}, + "psl":{"license":"MPL-2.0","url":"https://publicsuffix.org/list/","license_url":"https://mozilla.org/MPL/2.0/"}, + "changes":"Argand normalizes and combines assertions; provider endorsement is not implied."}) +} + +/// Streams all retained facts with their source declarations and attribution. +/// Description values and raw records are omitted unless explicitly requested. +/// +/// # Errors +/// Rejects existing destinations or malformed registry data. +pub fn export( + registry: &Registry, + output: &Path, + include_descriptions: bool, +) -> anyhow::Result<()> { + argand_atomic::create_durable_with(output, |file| { + export_inner(registry, file, include_descriptions).map_err(std::io::Error::other) + })?; + Ok(()) +} + +fn export_inner( + registry: &Registry, + file: &mut File, + include_descriptions: bool, +) -> anyhow::Result<()> { + let mut writer = BufWriter::new(file); + writeln!( + writer, + "{}", + json!({"schema":"argand.site-export/v1","registry":registry.identity,"attribution":attribution(),"descriptions_included":include_descriptions}) + )?; + let mut stmt=registry.db.prepare("SELECT f.id,f.subject,f.predicate,f.value,f.selector,f.confidence,s.manifest,r.native_id,f.source_id FROM facts f JOIN sources s ON s.id=f.source_id JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal WHERE s.complete=1 ORDER BY f.id")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let predicate: String = row.get(2)?; + if !include_descriptions && predicate == "description" { + continue; + } + let mut value: Value = serde_json::from_str(&row.get::<_, String>(3)?)?; + if !include_descriptions + && predicate == "category" + && let Some(object) = value.as_object_mut() + { + object.remove("description"); + } + writeln!( + writer, + "{}", + json!({"type":"assertion","id":row.get::<_,String>(0)?,"subject":row.get::<_,String>(1)?,"predicate":predicate,"value":value,"selector":row.get::<_,String>(4)?,"confidence":row.get::<_,u16>(5)?,"source":serde_json::from_str::(&row.get::<_,String>(6)?)?,"source_identifier":row.get::<_,String>(7)?,"source_snapshot_id":row.get::<_,String>(8)?,"description_redacted":!include_descriptions && predicate=="category"}) + )?; + } + writer.flush()?; + Ok(()) +} + +/// Signs a complete release using the operator's SSH signing key. +/// +/// # Errors +/// Returns missing key, existing signature, and signing process failures. +pub fn sign(generation: &Path, key: &Path, pin: &str) -> anyhow::Result<()> { + Registry::open(generation, pin)?; + ensure!( + !generation.join("COMPLETE.json.sig").exists(), + "signature already exists" + ); + let status = Command::new("ssh-keygen") + .args(["-Y", "sign", "-n", "argand-site-registry", "-f"]) + .arg(key) + .arg(generation.join("COMPLETE.json")) + .status()?; + ensure!(status.success(), "SSH signing failed"); + File::open(generation.join("COMPLETE.json.sig"))?.sync_all()?; + File::open(generation)?.sync_all()?; + Ok(()) +} + +/// Verifies a signature against an external allowed-signers trust file. +/// +/// # Errors +/// Rejects untrusted signatures, altered receipts, and corrupt databases. +pub fn verify_signed( + generation: &Path, + signers: &Path, + identity: &str, +) -> anyhow::Result { + let pin = crate::file_digest(&generation.join("COMPLETE.json"))?; + let status = Command::new("ssh-keygen") + .args(["-Y", "verify", "-n", "argand-site-registry", "-f"]) + .arg(signers) + .arg("-I") + .arg(identity) + .arg("-s") + .arg(generation.join("COMPLETE.json.sig")) + .stdin(Stdio::from(File::open(generation.join("COMPLETE.json"))?)) + .stdout(Stdio::null()) + .status()?; + ensure!(status.success(), "untrusted registry signature"); + Registry::open(generation, &pin) +} + +/// Activates a verified generation using one durable pointer. Refuses rollback +/// past distributed revocations; rebuild old inputs with the current review log. +/// +/// # Errors +/// Returns signature, rollback, lock, or filesystem failures. +pub fn activate( + generation: &Path, + current: &Path, + signers: &Path, + identity: &str, +) -> anyhow::Result<()> { + let registry = verify_signed(generation, signers, identity)?; + let parent = current + .parent() + .context("current pointer needs a parent directory")?; + fs::create_dir_all(parent)?; + let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(parent.join("activation.lock"))?; + lock.try_lock().context("another activation is running")?; + let revocation: u64 = registry.db.query_row( + "SELECT coalesce(max(sequence),0) FROM reviews WHERE decision='revoke'", + [], + |r| crate::store::unsigned(r, 0), + )?; + if current.exists() { + let previous: Value = crate::read_json(current)?; + ensure!( + revocation + >= previous["revocation_sequence"] + .as_u64() + .context("invalid current pointer")?, + "rollback would discard revocations; rebuild using the current review log" + ); + let old = Registry::open( + Path::new( + previous["generation"] + .as_str() + .context("invalid previous generation")?, + ), + previous["receipt_sha256"] + .as_str() + .context("invalid previous receipt")?, + )?; + preserve_revocations(&old, ®istry)?; + } + argand_atomic::replace_durable( + current, + &serde_json::to_vec_pretty( + &json!({"schema":"argand.site-current/v1","generation":generation.canonicalize()?,"receipt_sha256":registry.identity,"signer":identity,"revocation_sequence":revocation}), + )?, + )?; + Ok(()) +} + +fn preserve_revocations(old: &Registry, new: &Registry) -> anyhow::Result<()> { + // A sequence number alone is insufficient: a forked log could contain an + // unrelated revocation with a larger sequence. Require every exact old entry. + let mut statement = old + .db + .prepare("SELECT * FROM reviews WHERE decision='revoke' ORDER BY sequence")?; + let mut rows = statement.query([])?; + while let Some(row) = rows.next()? { + let sequence: i64 = row.get(0)?; + let expected = (1..11) + .map(|i| row.get::<_, String>(i)) + .collect::, _>>()?; + let found = new + .db + .query_row("SELECT * FROM reviews WHERE sequence=?1", [sequence], |r| { + (1..11) + .map(|i| r.get::<_, String>(i)) + .collect::, _>>() + }) + .context("rollback would discard a revocation")?; + ensure!( + found == expected, + "rollback would replace revocation history" + ); + } + Ok(()) +} + +/// Streams added/removed assertion fingerprints between two pinned generations. +/// +/// # Errors +/// Returns query or output errors. +pub fn diff(old: &Registry, new: &Registry, output: &mut dyn Write) -> anyhow::Result<()> { + for (kind, from, to) in [("removed", old, new), ("added", new, old)] { + let mut stmt = from + .db + .prepare("SELECT fingerprint,entity,property FROM edges ORDER BY fingerprint")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let fingerprint: String = row.get(0)?; + let exists: bool = to.db.query_row( + "SELECT EXISTS(SELECT 1 FROM edges WHERE fingerprint=?1)", + [&fingerprint], + |r| r.get(0), + )?; + if !exists { + writeln!( + output, + "{}", + json!({"change":kind,"fingerprint":fingerprint,"entity":row.get::<_,String>(1)?,"property":row.get::<_,String>(2)?}) + )?; + } + } + } + Ok(()) +} diff --git a/crates/argand-site-registry/src/review.rs b/crates/argand-site-registry/src/review.rs new file mode 100644 index 0000000..8480ebd --- /dev/null +++ b/crates/argand-site-registry/src/review.rs @@ -0,0 +1,116 @@ +// By Nic Weyand! +//! Append-only operator decisions bound to exact assertions and all entity names. + +use crate::query::Registry; +use anyhow::ensure; +use chrono::{DateTime, Utc}; +use rusqlite::{Connection, params}; +use serde::{Deserialize, Serialize}; + +/// Explicit review input, independent of imported source confidence. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Review { + /// Exact fingerprint printed by pinned lookup. + pub fingerprint: String, + /// `approve` or `revoke`. + pub decision: String, + /// Human/operator identity, carried into signed release. + pub reviewer: String, + /// Explanation, including how ownership and the role were checked. + pub reason: String, + /// Immutable external evidence reference or digest. + pub evidence: String, + /// Actual decision timestamp. + pub reviewed_at: DateTime, + /// Hard expiry; refresh never extends this automatically. + pub expires_at: DateTime, + /// `primary`, `regional`, or `unspecified`. + pub role: String, + /// Explicitly reviewed locale, or empty if unspecified. + pub locale: String, + /// Explicitly reviewed two-letter country code, or empty. + pub country: String, +} + +/// Appends a decision after verifying the exact generation and fingerprint. +/// Rebuild and promote to distribute it; existing immutable artifacts never mutate. +/// +/// # Errors +/// Rejects missing evidence, expired/overlong approval, and ineligible assertions. +pub fn record(db: &Connection, registry: &Registry, review: &Review) -> anyhow::Result { + validate(review)?; + let candidate = registry.candidate(&review.fingerprint)?; + if review.decision == "approve" { + ensure!( + candidate.eligible, + "deprecated/invalid assertion cannot be approved" + ); + } + // A reviewed assertion must originate in this store, not an unrelated artifact. + for provenance in &candidate.provenance { + let id = provenance["fact_id"].as_str().unwrap_or_default(); + let exists: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM facts WHERE id=?1)", + [id], + |r| r.get(0), + )?; + ensure!(exists, "review source is absent from this store"); + } + append(db, review) +} + +pub(crate) fn validate(review: &Review) -> anyhow::Result<()> { + ensure!( + matches!(review.decision.as_str(), "approve" | "revoke"), + "invalid decision" + ); + ensure!( + !review.reviewer.trim().is_empty() + && review.reviewer.len() <= 256 + && !review.reason.trim().is_empty() + && review.reason.len() <= 8192 + && !review.evidence.trim().is_empty() + && review.evidence.len() <= 8192, + "reviewer, reason and evidence are required and bounded" + ); + ensure!( + matches!(review.role.as_str(), "primary" | "regional" | "unspecified"), + "invalid property role" + ); + ensure!( + review.locale.len() <= 64 + && review + .locale + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-'), + "invalid locale" + ); + ensure!( + review.country.is_empty() + || (review.country.len() == 2 + && review.country.bytes().all(|b| b.is_ascii_uppercase())), + "country must be uppercase two-letter code" + ); + ensure!( + review.role != "regional" || !review.locale.is_empty() || !review.country.is_empty(), + "regional role needs locale or country evidence" + ); + ensure!( + review.role != "primary" || (review.locale.is_empty() && review.country.is_empty()), + "scoped destinations use the regional role; primary is the global fallback" + ); + if review.decision == "approve" { + ensure!( + review.expires_at > review.reviewed_at + && review.expires_at - review.reviewed_at <= chrono::Duration::days(90), + "approval must expire within 90 days" + ); + } + Ok(()) +} + +pub(crate) fn append(db: &Connection, review: &Review) -> anyhow::Result { + db.execute("INSERT INTO reviews(fingerprint,decision,reviewer,reason,evidence,reviewed_at,expires_at,role,locale,country) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",params![review.fingerprint,review.decision,review.reviewer,review.reason,review.evidence,review.reviewed_at.to_rfc3339(),review.expires_at.to_rfc3339(),review.role,review.locale,review.country])?; + Ok(u64::try_from(db.last_insert_rowid())?) +} diff --git a/crates/argand-site-registry/src/store.rs b/crates/argand-site-registry/src/store.rs new file mode 100644 index 0000000..d93a419 --- /dev/null +++ b/crates/argand-site-registry/src/store.rs @@ -0,0 +1,209 @@ +// By Nic Weyand! +//! Transactional source import and durable replay checkpoints. + +use crate::{ + adapters::{self, RecordSink}, + model::{Compression, Record, SourceManifest}, +}; +use anyhow::{Context, ensure}; +use rusqlite::{Connection, OptionalExtension, params}; +use std::{ + fs::File, + io::{BufReader, Read}, + path::Path, + time::Duration, +}; + +/// Adapter/normalization contract recorded in all generation identities. +pub const RULE_VERSION: &str = "argand.site-rules/v1"; + +/// Opens or migrates the local assertion store with bounded page cache. +/// +/// # Errors +/// Rejects newer schemas and SQLite/filesystem failures. +pub fn open(path: &Path) -> anyhow::Result { + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent)?; + } + let db = Connection::open(path)?; + configure(&db)?; + let version: i64 = db.query_row("PRAGMA user_version", [], |r| r.get(0))?; + match version { + 0 => { + db.execute_batch("BEGIN IMMEDIATE")?; + db.execute_batch(include_str!("../migrations/001.sql"))?; + db.execute_batch("COMMIT")?; + } + 1 => {} + _ => anyhow::bail!("unsupported registry schema {version}"), + } + let rules: String = db.query_row( + "SELECT rules FROM registry_metadata WHERE singleton=1", + [], + |r| r.get(0), + )?; + ensure!( + rules == RULE_VERSION, + "source parser rules changed; migrate explicitly or reimport pinned sources into a new store" + ); + Ok(db) +} + +pub(crate) fn configure(db: &Connection) -> anyhow::Result<()> { + db.busy_timeout(Duration::from_secs(10))?; + db.execute_batch("PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA cache_size=-8192; PRAGMA temp_store=FILE; PRAGMA synchronous=FULL;")?; + Ok(()) +} + +/// Verifies a source object before and after parsing, publishing completion last. +/// Incomplete transactions are rolled back; committed batches replay idempotently. +/// +/// # Errors +/// Returns source integrity, parser, filesystem, or database errors. +pub fn import( + db: &mut Connection, + manifest: &SourceManifest, + path: &Path, +) -> anyhow::Result { + manifest.validate()?; + verify_input(manifest, path)?; + let id = manifest.id()?; + let declaration = serde_json::to_string(manifest)?; + db.execute("INSERT OR IGNORE INTO sources(id,source,scope,retrieved_at,manifest) VALUES(?1,?2,?3,?4,?5)", params![id, manifest.source.key(),manifest.scope,manifest.retrieved_at.to_rfc3339(),declaration])?; + let (checkpoint, complete): (u64, bool) = db.query_row( + "SELECT checkpoint,complete FROM sources WHERE id=?1", + [&id], + |r| Ok((unsigned(r, 0)?, r.get(1)?)), + )?; + if complete { + return Ok(id); + } + let file = File::open(path)?; + let reader: Box = match manifest.compression { + Compression::None => Box::new(file), + Compression::Gzip => Box::new(flate2::read::MultiGzDecoder::new(file)), + Compression::Bzip2 => Box::new(bzip2::read::MultiBzDecoder::new(file)), + }; + db.execute_batch("BEGIN IMMEDIATE")?; + let result = { + let mut sink = SqlSink { + db, + source: &id, + ordinal: 0, + checkpoint, + }; + let parsed = + adapters::adapter(manifest.format).ingest(&mut BufReader::new(reader), &mut sink); + parsed.and_then(|()| { + ensure!( + sink.ordinal > 0 && sink.ordinal >= checkpoint, + "source empty or shorter than checkpoint" + ); + verify_input(manifest, path)?; + sink.db.execute( + "UPDATE sources SET checkpoint=?2,complete=1 WHERE id=?1", + params![id, i64::try_from(sink.ordinal)?], + )?; + Ok(()) + }) + }; + match result { + Ok(()) => db.execute_batch("COMMIT")?, + Err(error) => { + db.execute_batch("ROLLBACK")?; + return Err(error); + } + } + Ok(id) +} + +fn verify_input(manifest: &SourceManifest, path: &Path) -> anyhow::Result<()> { + let metadata = std::fs::symlink_metadata(path)?; + ensure!( + metadata.is_file() && metadata.len() == manifest.bytes, + "source length/type mismatch" + ); + ensure!( + crate::file_digest(path)? == manifest.sha256, + "source digest mismatch" + ); + Ok(()) +} + +struct SqlSink<'a> { + db: &'a Connection, + source: &'a str, + ordinal: u64, + checkpoint: u64, +} + +impl RecordSink for SqlSink<'_> { + fn emit(&mut self, record: Record) -> anyhow::Result<()> { + self.ordinal += 1; + if self.ordinal <= self.checkpoint { + return Ok(()); + } + ensure!( + record.native_id.len() <= 8192 && record.facts.len() <= 100_000, + "record identity/fact bound exceeded" + ); + self.db + .prepare_cached("INSERT INTO records VALUES(?1,?2,?3,?4)")? + .execute(params![ + self.source, + i64::try_from(self.ordinal)?, + record.native_id, + serde_json::to_string(&record.raw)? + ])?; + for fact in record.facts { + ensure!(fact.confidence <= 10000, "invalid confidence"); + let id = crate::digest(&serde_json::to_vec(&(self.source, self.ordinal, &fact))?); + self.db + .prepare_cached("INSERT OR IGNORE INTO facts VALUES(?1,?2,?3,?4,?5,?6,?7,?8)")? + .execute(params![ + id, + self.source, + i64::try_from(self.ordinal)?, + fact.subject, + fact.predicate, + serde_json::to_string(&fact.value)?, + fact.selector, + fact.confidence + ])?; + } + if self.ordinal.is_multiple_of(256) { + self.db.execute( + "UPDATE sources SET checkpoint=?2 WHERE id=?1", + params![self.source, i64::try_from(self.ordinal)?], + )?; + self.db.execute_batch("COMMIT; BEGIN IMMEDIATE")?; + } + Ok(()) + } +} + +/// Returns the manifest of a complete source snapshot. +/// +/// # Errors +/// Fails on missing, incomplete, or malformed declarations. +pub fn source(db: &Connection, id: &str) -> anyhow::Result { + let raw: String = db + .query_row( + "SELECT manifest FROM sources WHERE id=?1 AND complete=1", + [id], + |r| r.get(0), + ) + .optional()? + .context("source is absent or incomplete")?; + Ok(serde_json::from_str(&raw)?) +} + +pub(crate) fn unsigned(row: &rusqlite::Row<'_>, index: usize) -> rusqlite::Result { + u64::try_from(row.get::<_, i64>(index)?).map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + index, + rusqlite::types::Type::Integer, + Box::new(e), + ) + }) +} diff --git a/crates/argand-site-registry/src/update.rs b/crates/argand-site-registry/src/update.rs new file mode 100644 index 0000000..f4ce455 --- /dev/null +++ b/crates/argand-site-registry/src/update.rs @@ -0,0 +1,96 @@ +// By Nic Weyand! +//! Scheduler entry point; build candidates automatically and report failures. + +use crate::{ + download::{CachedSource, Download}, + model::SourceManifest, +}; +use anyhow::Context; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{self, OpenOptions}, + path::PathBuf, +}; + +/// Update configuration, usable from systemd or cron without a resident daemon. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Config { + /// Source cache root. + pub cache: PathBuf, + /// Mutable import/review database. + pub database: PathBuf, + /// Immutable generation parent. + pub generations: PathBuf, + /// Explicit bounded network downloads. `{date}` and `{month}` expand in snapshot. + #[serde(default)] + pub downloads: Vec, + /// Pinned local sources, including existing acquisitions. + #[serde(default)] + pub inputs: Vec, + /// Explicit billed `CrUX` jobs, empty by default. + #[serde(default)] + pub crux: Vec, +} + +/// Runs all declared imports, refusing candidate publication on any failure. +/// Complete prior generations remain available. Does not sign or activate. +/// +/// # Errors +/// Returns configuration, source, lock, import, or build errors. +pub async fn run(config: &Config) -> anyhow::Result { + fs::create_dir_all(&config.generations)?; + let lock = OpenOptions::new() // atomic-writes: allow advisory lock inode must remain stable + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(config.generations.join("update.lock"))?; + lock.try_lock().context("registry update already running")?; + let mut inputs = config.inputs.clone(); + let now = Utc::now(); + for request in &config.downloads { + let mut request = request.clone(); + request.snapshot = request + .snapshot + .replace("{date}", &now.format("%Y-%m-%d").to_string()) + .replace("{month}", &now.format("%Y-%m").to_string()); + inputs.push(crate::download::download(&config.cache, &request).await?); + } + for request in &config.crux { + let mut request = request.clone(); + if request.month == "{previous_month}" { + request.month = now + .checked_sub_months(chrono::Months::new(1)) + .context("previous calendar month unavailable")? + .format("%Y%m") + .to_string(); + } + inputs.push(crate::crux::download(&config.cache, &request).await?); + } + anyhow::ensure!(!inputs.is_empty(), "update config contains no sources"); + let mut db = crate::store::open(&config.database)?; + for input in inputs { + let manifest: SourceManifest = crate::read_json(&input.manifest)?; + crate::store::import(&mut db, &manifest, &input.input)?; + } + // Build in a unique unpublished directory, then name the complete generation + // by its receipt. Identical inputs/reviews reuse the same immutable artifact. + let pending = config + .generations + .join(format!("pending-{}", now.format("%Y%m%dT%H%M%S%.9fZ"))); + crate::build::build(&db, &pending)?; + let pin = crate::file_digest(&pending.join("COMPLETE.json"))?; + let output = config.generations.join(format!("candidate-{pin}")); + if output.exists() { + crate::query::Registry::open(&output, &pin)?; + // This directory was created by this invocation and contains only its + // successfully validated duplicate build, never an operator generation. + fs::remove_dir_all(&pending)?; + } else { + fs::rename(&pending, &output)?; + std::fs::File::open(&config.generations)?.sync_all()?; + } + Ok(output) +} diff --git a/crates/argand-site-registry/tests/cli.rs b/crates/argand-site-registry/tests/cli.rs new file mode 100644 index 0000000..21dd87d --- /dev/null +++ b/crates/argand-site-registry/tests/cli.rs @@ -0,0 +1,349 @@ +// By Nic Weyand! +//! Fresh-process proof for all source imports and the signed release lifecycle. +#[allow(dead_code)] // Shared fixture helpers also support the library contract suite. +mod common; + +use anyhow::{Context, ensure}; +use argand_site_registry::model::{Format, Source}; +use serde_json::{Value, json}; +use std::{fs, path::Path, process::Command}; + +fn run(args: &[&str]) -> anyhow::Result { + let output = Command::new(env!("CARGO_BIN_EXE_argand-site-registry")) + .args(args) + .output()?; + ensure!( + output.status.success(), + "CLI failed: {:?}\n{}", + args, + String::from_utf8_lossy(&output.stderr) + ); + Ok(serde_json::from_slice(&output.stdout)?) +} + +fn text(path: &Path) -> anyhow::Result<&str> { + path.to_str().context("non-UTF8 fixture path") +} + +#[test] +fn all_source_import_review_resolve_revoke_and_signed_rollback() -> anyhow::Result<()> { + let temporary = tempfile::tempdir()?; + let configured = std::env::var_os("ARGAND_REGISTRY_E2E_OUTPUT").map(std::path::PathBuf::from); + let root = configured.as_deref().unwrap_or(temporary.path()); + if configured.is_some() { + fs::create_dir(root)?; + } + let database = root.join("store.sqlite"); + import_sources(root, &database)?; + let candidate = root.join("candidate"); + let built = run(&[ + "build", + "--database", + text(&database)?, + "--output", + text(&candidate)?, + ])?; + let pin = built["pin"].as_str().context("missing pin")?; + let lookup = run(&[ + "lookup", + "--generation", + text(&candidate)?, + "--pin", + pin, + "--query", + "facebook", + ])?; + assert_eq!(lookup["candidates"][0]["canonical_name"], "Facebook"); + assert_eq!( + lookup["candidates"][0]["web_property"]["domain"]["registrable_domain"], + "facebook.com" + ); + assert!( + run(&[ + "resolve", + "--generation", + text(&candidate)?, + "--pin", + pin, + "--query", + "facebook" + ])?["destination"] + .is_null() + ); + let now = chrono::Utc::now() - chrono::Duration::seconds(1); + let decision = json!({"fingerprint":lookup["candidates"][0]["fingerprint"],"decision":"approve","reviewer":"synthetic fixture reviewer","reason":"E2E test only, not actual site verification","evidence":"synthetic:fixture","reviewed_at":now,"expires_at":now+chrono::Duration::days(1),"role":"primary","locale":"","country":""}); + let decision_path = root.join("review.json"); + fs::write(&decision_path, serde_json::to_vec(&decision)?)?; + run(&[ + "review", + "--database", + text(&database)?, + "--generation", + text(&candidate)?, + "--pin", + pin, + "--decision", + text(&decision_path)?, + ])?; + let approved = root.join("approved"); + review_identity(root, &database, &candidate, pin, &decision)?; + let built = run(&[ + "build", + "--database", + text(&database)?, + "--output", + text(&approved)?, + ])?; + let approved_pin = built["pin"].as_str().context("approved pin")?; + assert_eq!( + run(&[ + "resolve", + "--generation", + text(&approved)?, + "--pin", + approved_pin, + "--query", + "FB" + ])?["destination"]["url"], + "https://facebook.com/" + ); + let (revoked, revoked_pin) = release_lifecycle( + root, + &database, + &approved, + approved_pin, + decision, + &decision_path, + )?; + export_fixture(root, &revoked, &revoked_pin)?; + println!("Native fixture lifecycle passed: {}", root.display()); + Ok(()) +} + +fn import_sources(root: &Path, database: &Path) -> anyhow::Result<()> { + let sources = [ + ( + Source::Psl, + Format::PslText, + common::PSL.as_bytes().to_vec(), + ), + ( + Source::Wikidata, + Format::WikidataEntities, + serde_json::to_vec(&common::wikidata())?, + ), + ( + Source::Majestic, + Format::MajesticCsv, + common::MAJESTIC.as_bytes().to_vec(), + ), + ( + Source::Crux, + Format::CruxCsv, + common::CRUX.as_bytes().to_vec(), + ), + (Source::Curlie, Format::CurlieTarGz, common::curlie()?), + ]; + for (source, format, bytes) in sources { + let input = root.join(format!("{}.input", source.key())); + let manifest = root.join(format!("{}.json", source.key())); + fs::write(&input, &bytes)?; + fs::write( + &manifest, + serde_json::to_vec_pretty(&common::manifest(source, format, &bytes)?)?, + )?; + let args = [ + "import", + "--database", + text(database)?, + "--input", + text(&input)?, + "--manifest", + text(&manifest)?, + ]; + assert_eq!(run(&args)?, run(&args)?); + } + Ok(()) +} + +fn release_lifecycle( + root: &Path, + database: &Path, + approved: &Path, + approved_pin: &str, + mut decision: Value, + decision_path: &Path, +) -> anyhow::Result<(std::path::PathBuf, String)> { + let key = root.join("signer"); + let status = Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(&key) + .status()?; + ensure!(status.success(), "generate test key"); + let allowed = root.join("allowed_signers"); + fs::write( + &allowed, + format!("fixture {}", fs::read_to_string(key.with_extension("pub"))?), + )?; + sign_and_activate(root, approved, approved_pin)?; + decision["decision"] = json!("revoke"); + fs::write(decision_path, serde_json::to_vec(&decision)?)?; + run(&[ + "review", + "--database", + text(database)?, + "--generation", + text(approved)?, + "--pin", + approved_pin, + "--decision", + text(decision_path)?, + ])?; + let revoked = root.join("revoked"); + let built = run(&[ + "build", + "--database", + text(database)?, + "--output", + text(&revoked)?, + ])?; + let revoked_pin = built["pin"].as_str().context("revoked pin")?; + assert!( + run(&[ + "resolve", + "--generation", + text(&revoked)?, + "--pin", + revoked_pin, + "--query", + "FB" + ])?["destination"] + .is_null() + ); + sign_and_activate(root, &revoked, revoked_pin)?; + assert!( + run(&[ + "activate", + "--generation", + text(approved)?, + "--current", + text(&root.join("current.json"))?, + "--allowed-signers", + text(&root.join("allowed_signers"))?, + "--identity", + "fixture" + ]) + .is_err() + ); + Ok((revoked, revoked_pin.into())) +} + +fn sign_and_activate(root: &Path, approved: &Path, approved_pin: &str) -> anyhow::Result<()> { + let key = root.join("signer"); + let allowed = root.join("allowed_signers"); + run(&[ + "sign", + "--generation", + text(approved)?, + "--pin", + approved_pin, + "--key", + text(&key)?, + ])?; + let current = root.join("current.json"); + assert!( + run(&[ + "activate", + "--generation", + text(approved)?, + "--current", + text(¤t)?, + "--allowed-signers", + text(&allowed)?, + "--identity", + "untrusted" + ]) + .is_err() + ); + run(&[ + "activate", + "--generation", + text(approved)?, + "--current", + text(¤t)?, + "--allowed-signers", + text(&allowed)?, + "--identity", + "fixture", + ])?; + Ok(()) +} + +fn review_identity( + root: &Path, + database: &Path, + generation: &Path, + pin: &str, + template: &Value, +) -> anyhow::Result<()> { + let wiki = run(&[ + "lookup", + "--generation", + text(generation)?, + "--pin", + pin, + "--query", + "FB", + ])?; + let curlie = run(&[ + "lookup", + "--generation", + text(generation)?, + "--pin", + pin, + "--query", + "Facebook directory listing", + ])?; + let left = wiki["candidates"][0]["entity_id"] + .as_str() + .context("Wiki identity")?; + let right = curlie["candidates"][0]["entity_id"] + .as_str() + .context("Curlie identity")?; + let args = [ + "equivalence", + "--generation", + text(generation)?, + "--pin", + pin, + "--left", + left, + "--right", + right, + ]; + let preview = run(&args)?; + let mut decision = template.clone(); + decision["fingerprint"] = preview["fingerprint"].clone(); + decision["role"] = json!("unspecified"); + let path = root.join("identity-review.json"); + fs::write(&path, serde_json::to_vec(&decision)?)?; + let mut args = args.to_vec(); + args.extend(["--database", text(database)?, "--decision", text(&path)?]); + run(&args)?; + Ok(()) +} + +fn export_fixture(root: &Path, revoked: &Path, revoked_pin: &str) -> anyhow::Result<()> { + let export = root.join("registry.jsonl"); + run(&[ + "export", + "--generation", + text(revoked)?, + "--pin", + revoked_pin, + "--output", + text(&export)?, + ])?; + assert!(fs::read_to_string(export)?.contains("CC-BY-4.0")); + Ok(()) +} diff --git a/crates/argand-site-registry/tests/common/mod.rs b/crates/argand-site-registry/tests/common/mod.rs new file mode 100644 index 0000000..9ec6c54 --- /dev/null +++ b/crates/argand-site-registry/tests/common/mod.rs @@ -0,0 +1,179 @@ +// By Nic Weyand! +//! Synthetic source-shaped fixtures; these are not real ownership evidence. + +use anyhow::Context; +use argand_site_registry::{ + model::{Compression, Format, Source, SourceManifest}, + query::Registry, + store, +}; +use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; +use std::{fs, io::Write, path::Path}; + +pub const PSL: &str = "// This fixture is authored for Argand tests.\n// ===BEGIN ICANN DOMAINS===\ncom\norg\nuk\nco.uk\nde\nbe\nfr\njp\n*.kawasaki.jp\n!city.kawasaki.jp\n// ===END ICANN DOMAINS===\n// ===BEGIN PRIVATE DOMAINS===\nblogspot.com\n// ===END PRIVATE DOMAINS===\n"; +pub const MAJESTIC: &str = "GlobalRank,TldRank,Domain,TLD,RefSubNets,RefIPs,IDN_Domain,IDN_TLD,PrevGlobalRank,PrevTldRank,PrevRefSubNets,PrevRefIPs\n2,2,facebook.com,com,100,200,facebook.com,com,2,2,99,199\n3,3,atlas.example.co.uk,uk,90,180,atlas.example.co.uk,uk,3,3,90,180\n"; +pub const CRUX: &str = "origin,rank,yyyymm,country_code\nhttps://facebook.com,1000,202608,US\nhttps://atlas.example.co.uk,100000,202608,GB\n"; + +pub fn timestamp() -> anyhow::Result> { + Ok(DateTime::parse_from_rfc3339("2026-09-12T12:00:00Z")?.with_timezone(&Utc)) +} + +pub fn entity(id: &str, name: &str, aliases: &[&str], urls: &[&str]) -> Value { + let websites:Vec=urls.iter().enumerate().map(|(i,url)|json!({"id":format!("{id}${i}"),"rank":"normal","mainsnak":{"property":"P856","snaktype":"value","datavalue":{"type":"string","value":url}},"references":[{"hash":"synthetic-reference","snaks":{}}]})).collect(); + json!({"id":id,"type":"item","lastrevid":1,"labels":{"en":{"language":"en","value":name}},"aliases":{"en":aliases.iter().map(|text|json!({"language":"en","value":text})).collect::>()},"claims":{"P856":websites}}) +} + +pub fn wikidata() -> Value { + let mut atlas = entity( + "Q900001", + "Atlas Fixture", + &["Atlas", "Cafe\u{301} Atlas"], + &[ + "https://atlas.example.com/", + "https://atlas.example.co.uk/", + "https://atlas.example.de/", + ], + ); + atlas["claims"]["P856"][1]["qualifiers"] = json!({"P1001":[{"snaktype":"value","property":"P1001","datavalue":{"value":{"id":"Q145"}}}],"P407":[{"snaktype":"value","property":"P407","datavalue":{"value":{"id":"Q1860"}}}]}); + json!({"entities":{"Q355":entity("Q355","Facebook",&["FB"],&["https://facebook.com/"]),"Q900001":atlas}}) +} + +pub fn curlie() -> anyhow::Result> { + let gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + let mut archive = tar::Builder::new(gzip); + for (name, bytes) in [ + ( + "curlie-rdf/rdf-Top-c.tsv", + "https://facebook.com/\tFacebook directory listing\tSynthetic editorial description\t42\n", + ), + ( + "curlie-rdf/rdf-Top-s.tsv", + "42\tComputers/Internet\t1\tSynthetic category description\t\t\n", + ), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(u64::try_from(bytes.len())?); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, name, bytes.as_bytes())?; + } + Ok(archive.into_inner()?.finish()?) +} + +pub fn manifest(source: Source, format: Format, bytes: &[u8]) -> anyhow::Result { + let source_url = match source { + Source::Psl => "https://publicsuffix.org/list/public_suffix_list.dat", + Source::Wikidata => "https://www.wikidata.org/wiki/Special:EntityData/Q355.json", + Source::Majestic => "https://downloads.majestic.com/majestic_million.csv", + Source::Crux => "https://developer.chrome.com/docs/crux/bigquery/", + Source::Curlie => "https://curlie.org/directory-dl", + }; + Ok(SourceManifest { + schema: "argand.site-source/v1".into(), + source, + format, + compression: Compression::None, + snapshot: "synthetic-fixture-v1".into(), + scope: "fixture".into(), + source_url: source_url.into(), + license: source.license().into(), + license_url: source.license_url().into(), + retrieved_at: timestamp()?, + sha256: argand_site_registry::digest(bytes), + bytes: u64::try_from(bytes.len())?, + }) +} + +pub fn import( + db: &mut rusqlite::Connection, + root: &Path, + source: Source, + format: Format, + bytes: &[u8], +) -> anyhow::Result { + let manifest = manifest(source, format, bytes)?; + let input = root.join(format!("{}.input", manifest.sha256)); + fs::write(&input, bytes)?; + store::import(db, &manifest, &input)?; + Ok(manifest) +} + +pub fn fixture(root: &Path) -> anyhow::Result { + let mut db = store::open(&root.join("store.sqlite"))?; + import(&mut db, root, Source::Psl, Format::PslText, PSL.as_bytes())?; + import( + &mut db, + root, + Source::Wikidata, + Format::WikidataEntities, + &serde_json::to_vec(&wikidata())?, + )?; + import( + &mut db, + root, + Source::Majestic, + Format::MajesticCsv, + MAJESTIC.as_bytes(), + )?; + import( + &mut db, + root, + Source::Crux, + Format::CruxCsv, + CRUX.as_bytes(), + )?; + import( + &mut db, + root, + Source::Curlie, + Format::CurlieTarGz, + &curlie()?, + )?; + Ok(db) +} + +pub fn build(db: &rusqlite::Connection, root: &Path, name: &str) -> anyhow::Result { + let generation = root.join(name); + argand_site_registry::build::build(db, &generation)?; + Registry::open( + &generation, + &argand_site_registry::file_digest(&generation.join("COMPLETE.json"))?, + ) +} + +pub fn approve( + db: &rusqlite::Connection, + registry: &Registry, + query: &str, + url: &str, + role: &str, + country: &str, +) -> anyhow::Result<()> { + let candidate = registry + .lookup(query, 100)? + .candidates + .into_iter() + .find(|c| c.url == url) + .context("fixture destination missing")?; + let review = argand_site_registry::review::Review { + fingerprint: candidate.fingerprint, + decision: "approve".into(), + reviewer: "synthetic fixture reviewer".into(), + reason: "Test only; not a real-world ownership assertion".into(), + evidence: "synthetic:fixture-observation".into(), + reviewed_at: timestamp()?, + expires_at: timestamp()? + chrono::Duration::days(7), + role: role.into(), + locale: String::new(), + country: country.into(), + }; + argand_site_registry::review::record(db, registry, &review)?; + Ok(()) +} + +pub fn gzip(bytes: &[u8]) -> anyhow::Result> { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?) +} diff --git a/crates/argand-site-registry/tests/failures.rs b/crates/argand-site-registry/tests/failures.rs new file mode 100644 index 0000000..6849c51 --- /dev/null +++ b/crates/argand-site-registry/tests/failures.rs @@ -0,0 +1,272 @@ +// By Nic Weyand! +//! Source poisoning, repeatability and metadata regression cases. +#[allow(dead_code)] // Same source-shaped helpers as the native lifecycle suite. +mod common; +use argand_site_registry::{ + download::CachedSource, + model::{Compression, Format, Source}, + query::Registry, + store, +}; +use serde_json::json; +use std::{fs, io::Write}; + +#[test] +fn removed_entity_websites_retire_the_previous_selection() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let mut db = common::fixture(root.path())?; + let before = common::build(&db, root.path(), "before-retirement")?; + common::approve(&db, &before, "FB", "https://facebook.com/", "primary", "")?; + let bytes = br#"{"entities":{"Q355":{"id":"Q355","missing":""}}}"#; + let mut source = common::manifest(Source::Wikidata, Format::WikidataEntities, bytes)?; + source.retrieved_at += chrono::Duration::days(1); + let input = root.path().join("retirement.json"); + fs::write(&input, bytes)?; + store::import(&mut db, &source, &input)?; + let retired = common::build(&db, root.path(), "retired")?; + assert_eq!(retired.lookup("FB", 1)?.total_entities, 0); + assert!( + retired + .resolve("FB", None, None, common::timestamp()?)? + .is_none() + ); + assert_eq!(before.lookup("FB", 1)?.total_entities, 1); + Ok(()) +} + +#[test] +fn import_order_does_not_change_generation_and_psl_refresh_preserves_review() -> anyhow::Result<()> +{ + let root = tempfile::tempdir()?; + let a = common::fixture(root.path())?; + let mut b = store::open(&root.path().join("other.sqlite"))?; + for (source, format, bytes) in [ + (Source::Curlie, Format::CurlieTarGz, common::curlie()?), + ( + Source::Crux, + Format::CruxCsv, + common::CRUX.as_bytes().to_vec(), + ), + ( + Source::Majestic, + Format::MajesticCsv, + common::MAJESTIC.as_bytes().to_vec(), + ), + ( + Source::Wikidata, + Format::WikidataEntities, + serde_json::to_vec(&common::wikidata())?, + ), + ( + Source::Psl, + Format::PslText, + common::PSL.as_bytes().to_vec(), + ), + ] { + common::import(&mut b, root.path(), source, format, &bytes)?; + } + let first = common::build(&a, root.path(), "a")?; + let second = common::build(&b, root.path(), "b")?; + assert_eq!(first.identity, second.identity); + common::approve(&b, &second, "FB", "https://facebook.com/", "primary", "")?; + let input = root.path().join("psl"); + fs::write(&input, common::PSL)?; + let mut manifest = common::manifest(Source::Psl, Format::PslText, common::PSL.as_bytes())?; + manifest.retrieved_at += chrono::Duration::days(1); + store::import(&mut b, &manifest, &input)?; + let refreshed = common::build(&b, root.path(), "refresh")?; + assert!( + refreshed + .resolve("FB", None, None, common::timestamp()?)? + .is_some() + ); + Ok(()) +} + +#[test] +fn malformed_archives_and_incomplete_refresh_never_replace_sources() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let mut db = common::fixture(root.path())?; + let mut archive = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + archive.append_link(&mut header, "curlie-rdf/evil-c.tsv", "/etc/passwd")?; + let link = archive.into_inner()?.finish()?; + let mut truncated = common::curlie()?; + truncated.truncate(truncated.len() - 5); + for bytes in [&link, &truncated] { + assert!( + common::import( + &mut db, + root.path(), + Source::Curlie, + Format::CurlieTarGz, + bytes + ) + .is_err() + ); + } + let registry = common::build(&db, root.path(), "complete")?; + assert_eq!(registry.receipt.sources.len(), 5); + assert_eq!( + registry + .lookup("Facebook directory listing", 1)? + .total_entities, + 1 + ); + fs::write(root.path().join("complete/ATTRIBUTION.json"), b"{}")?; + assert!(Registry::open(&root.path().join("complete"), ®istry.identity).is_err()); + Ok(()) +} + +#[test] +fn temporal_deprecated_and_nonvalue_assertions_keep_evidence_without_admission() +-> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let mut db = store::open(&root.path().join("data.sqlite"))?; + common::import( + &mut db, + root.path(), + Source::Psl, + Format::PslText, + common::PSL.as_bytes(), + )?; + let mut entity = common::entity( + "Q100", + "Historical", + &[], + &[ + "https://old.example.com", + "https://ancient.example.com", + "https://unknown.example.com", + ], + ); + entity["claims"]["P856"][0]["qualifiers"] = + json!({"P582":[{"datavalue":{"value":{"time":"+2001-01-01T00:00:00Z"}}}]}); + entity["claims"]["P856"][1]["rank"] = json!("deprecated"); + entity["claims"]["P856"][2]["mainsnak"] = json!({"property":"P856","snaktype":"novalue"}); + let bytes = serde_json::to_vec(&json!({"entities":{"Q100":entity}}))?; + common::import( + &mut db, + root.path(), + Source::Wikidata, + Format::WikidataEntities, + &bytes, + )?; + let registry = common::build(&db, root.path(), "generation")?; + assert_eq!(registry.receipt.rejected, 1); + assert!( + registry + .lookup("Historical", 100)? + .candidates + .iter() + .all(|c| !c.eligible) + ); + assert!( + common::approve( + &db, + ®istry, + "Historical", + "https://old.example.com/", + "primary", + "" + ) + .is_err() + ); + Ok(()) +} + +#[test] +fn bzip2_multistream_and_property_scope_metadata() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let mut db = store::open(&root.path().join("data.sqlite"))?; + let data = serde_json::to_vec(&common::wikidata())?; + let mut bytes = Vec::new(); + for half in data.chunks(data.len().div_ceil(2)) { + let mut bz = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::fast()); + bz.write_all(half)?; + bytes.extend(bz.finish()?); + } + let input = root.path().join("entities.bz2"); + fs::write(&input, &bytes)?; + let mut manifest = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?; + manifest.compression = Compression::Bzip2; + store::import(&mut db, &manifest, &input)?; + common::import( + &mut db, + root.path(), + Source::Psl, + Format::PslText, + common::PSL.as_bytes(), + )?; + let registry = common::build(&db, root.path(), "generation")?; + let candidates = registry.lookup("Atlas", 10)?.candidates; + let regional = candidates + .iter() + .find(|c| c.url.contains("co.uk")) + .ok_or_else(|| anyhow::anyhow!("regional missing"))?; + assert_eq!(regional.property_scopes[0].jurisdiction_entities, ["Q145"]); + assert_eq!(regional.property_scopes[0].language_entities, ["Q1860"]); + assert_eq!(regional.property_scopes[0].country, None); + assert!( + regional + .entity + .names + .iter() + .any(|n| n["value"]["text"] == "Atlas" && n["source"]["license"] == "CC0-1.0") + ); + Ok(()) +} + +#[tokio::test] +async fn repeated_update_reuses_generation_and_failure_preserves_it() -> anyhow::Result<()> { + let root = tempfile::tempdir()?; + let mut inputs = Vec::new(); + for (source, format, bytes) in [ + ( + Source::Psl, + Format::PslText, + common::PSL.as_bytes().to_vec(), + ), + ( + Source::Wikidata, + Format::WikidataEntities, + serde_json::to_vec(&common::wikidata())?, + ), + ] { + let input = root.path().join(source.key()); + let manifest = input.with_extension("json"); + fs::write(&input, &bytes)?; + fs::write( + &manifest, + serde_json::to_vec(&common::manifest(source, format, &bytes)?)?, + )?; + inputs.push(CachedSource { input, manifest }); + } + let config = argand_site_registry::update::Config { + cache: root.path().join("cache"), + database: root.path().join("data.sqlite"), + generations: root.path().join("generations"), + downloads: vec![], + inputs, + crux: vec![], + }; + let first = argand_site_registry::update::run(&config).await?; + let second = argand_site_registry::update::run(&config).await?; + assert_eq!(first, second); + let pin = argand_site_registry::file_digest(&first.join("COMPLETE.json"))?; + fs::write(&config.inputs[1].input, "truncated")?; + assert!(argand_site_registry::update::run(&config).await.is_err()); + assert_eq!( + Registry::open(&first, &pin)? + .lookup("FB", 1)? + .total_entities, + 1 + ); + Ok(()) +} diff --git a/crates/argand-site-registry/tests/identity.rs b/crates/argand-site-registry/tests/identity.rs new file mode 100644 index 0000000..e65f7c7 --- /dev/null +++ b/crates/argand-site-registry/tests/identity.rs @@ -0,0 +1,173 @@ +// By Nic Weyand! +//! Cross-source collisions need an exact, revocable identity decision. +#[allow(dead_code)] +mod common; +use argand_site_registry::{ + identity, + model::{Format, Source}, + review::{self, Review}, + store, +}; +use serde_json::json; + +fn decision(fingerprint: String) -> anyhow::Result { + Ok(Review { + fingerprint, + decision: "approve".into(), + reviewer: "synthetic reviewer".into(), + reason: "Synthetic identity test, not actual ownership".into(), + evidence: "synthetic:identity".into(), + reviewed_at: common::timestamp()?, + expires_at: common::timestamp()? + chrono::Duration::days(2), + role: "unspecified".into(), + locale: String::new(), + country: String::new(), + }) +} + +#[test] +fn explicit_equivalence_resolves_cross_source_aliases_and_expires_or_revokes() -> anyhow::Result<()> +{ + let root = tempfile::tempdir()?; + let db = common::fixture(root.path())?; + let baseline = common::build(&db, root.path(), "baseline")?; + common::approve(&db, &baseline, "FB", "https://facebook.com/", "primary", "")?; + let wiki = &baseline.lookup("FB", 1)?.candidates[0].entity_id; + let curlie = &baseline.lookup("Facebook directory listing", 1)?.candidates[0].entity_id; + let pair = identity::propose(&baseline, wiki, curlie)?; + let mut review = decision(pair.fingerprint.clone())?; + identity::record(&db, &baseline, &pair, &review)?; + let linked = common::build(&db, root.path(), "linked")?; + let resolved = linked + .resolve( + "Facebook directory listing", + None, + None, + common::timestamp()?, + )? + .ok_or_else(|| anyhow::anyhow!("approved alias not linked"))?; + assert_eq!(resolved.entity_id, *wiki); + assert_eq!(resolved.identity_provenance.len(), 1); + assert!( + linked + .resolve("Facebook directory listing", None, None, review.expires_at)? + .is_none() + ); + review.decision = "revoke".into(); + identity::record(&db, &linked, &pair, &review)?; + let revoked = common::build(&db, root.path(), "revoked")?; + assert!( + revoked + .resolve( + "Facebook directory listing", + None, + None, + common::timestamp()? + )? + .is_none() + ); + assert!( + revoked + .resolve("FB", None, None, common::timestamp()?)? + .is_some() + ); + Ok(()) +} + +#[test] +fn name_collision_remains_ambiguous_until_review_and_changes_invalidate_link() -> anyhow::Result<()> +{ + let root = tempfile::tempdir()?; + let mut db = common::fixture(root.path())?; + let mut entity = common::entity( + "Q355", + "Facebook", + &["FB", "Facebook directory listing"], + &["https://facebook.com/"], + ); + let bytes = serde_json::to_vec(&json!({"entities":{"Q355":entity}}))?; + let mut source = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?; + source.retrieved_at += chrono::Duration::hours(1); + let input = root.path().join("changed"); + std::fs::write(&input, &bytes)?; + store::import(&mut db, &source, &input)?; + let baseline = common::build(&db, root.path(), "baseline")?; + common::approve(&db, &baseline, "FB", "https://facebook.com/", "primary", "")?; + let approved = common::build(&db, root.path(), "approved")?; + assert_eq!( + approved + .lookup("Facebook directory listing", 1)? + .total_entities, + 2 + ); + assert!( + approved + .resolve( + "Facebook directory listing", + None, + None, + common::timestamp()? + )? + .is_none() + ); + let candidates = baseline + .lookup("Facebook directory listing", 10)? + .candidates; + let pair = identity::propose( + &baseline, + &candidates[0].entity_id, + &candidates[1].entity_id, + )?; + identity::record(&db, &baseline, &pair, &decision(pair.fingerprint.clone())?)?; + let linked = common::build(&db, root.path(), "linked")?; + assert!( + linked + .resolve( + "Facebook directory listing", + None, + None, + common::timestamp()? + )? + .is_some() + ); + assert_eq!( + linked + .lookup("Facebook directory listing", 1)? + .total_entities, + 2 + ); + // Even renewing a destination review does not silently renew identity evidence. + entity["aliases"]["en"] + .as_array_mut() + .ok_or_else(|| anyhow::anyhow!("aliases"))? + .push(json!({"language":"en","value":"new unreviewed alias"})); + let bytes = serde_json::to_vec(&json!({"entities":{"Q355":entity}}))?; + source.sha256 = argand_site_registry::digest(&bytes); + source.bytes = u64::try_from(bytes.len())?; + source.retrieved_at += chrono::Duration::hours(1); + std::fs::write(&input, &bytes)?; + store::import(&mut db, &source, &input)?; + let changed = common::build(&db, root.path(), "changed-generation")?; + common::approve(&db, &changed, "FB", "https://facebook.com/", "primary", "")?; + let renewed = common::build(&db, root.path(), "renewed-destination")?; + assert!( + renewed + .resolve( + "Facebook directory listing", + None, + None, + common::timestamp()? + )? + .is_none() + ); + assert!( + renewed + .resolve("FB", None, None, common::timestamp()?)? + .is_some() + ); + let mut tampered = decision(pair.fingerprint.clone())?; + tampered.role = "primary".into(); + assert!(identity::record(&db, &baseline, &pair, &tampered).is_err()); + assert!(review::record(&db, &baseline, &tampered).is_err()); + Ok(()) +} diff --git a/crates/argand-site-registry/tests/registry.rs b/crates/argand-site-registry/tests/registry.rs new file mode 100644 index 0000000..32740a6 --- /dev/null +++ b/crates/argand-site-registry/tests/registry.rs @@ -0,0 +1,322 @@ +// By Nic Weyand! +//! Registry contracts exercised through real SQLite stores and immutable releases. +mod common; +use anyhow::{Context, ensure}; +use argand_site_registry::{ + model::{Compression, Format, Source}, + normalize::{Normalizer, name_key}, + query::Registry, + review::{self, Review}, + store, +}; +use common::{approve, build, fixture, import, timestamp}; +use serde_json::json; + +#[test] +fn normalization_psl_and_identity() -> anyhow::Result<()> { + let n = Normalizer::new(common::PSL.as_bytes(), "fixture-psl".into())?; + let p = n.url("https://WWW.Example.co.uk:443/a?b=2&a=1#fragment")?; + assert_eq!(p.url, "https://www.example.co.uk/a?b=2&a=1"); + assert_eq!(p.domain.registrable_domain, "example.co.uk"); + assert_eq!(p.domain.public_suffix, "co.uk"); + assert_eq!( + n.domain("foo.blogspot.com")?.registrable_domain, + "foo.blogspot.com" + ); + assert!(n.domain("foo.blogspot.com")?.private_suffix); + assert_eq!( + n.domain("a.city.kawasaki.jp")?.registrable_domain, + "city.kawasaki.jp" + ); + assert_eq!(n.domain("a.b.kawasaki.jp")?.public_suffix, "b.kawasaki.jp"); + assert_eq!(n.domain("BÜCHER.de.")?.hostname, "xn--bcher-kva.de"); + for url in [ + "javascript:alert(1)", + "https://user:pass@example.com/", + "https://127.1/", + "https://[::1]/", + "https://foo.local/", + "https://co.uk/", + "https://example.com\\@evil.com/", + "https://example.com/\n", + "https:/example.com/", + "https://example.com../", + ] { + assert!(n.url(url).is_err(), "accepted {url}"); + } + assert_ne!( + n.url("http://example.com/")?.id, + n.url("https://example.com/")?.id + ); + assert_ne!( + n.url("https://example.com/")?.id, + n.url("https://www.example.com/")?.id + ); + assert_eq!(name_key(" Cafe\u{301} ATLAS ")?, "café atlas"); + assert!(name_key("Face\u{202e}book").is_err()); + Ok(()) +} + +#[test] +fn aliases_deduplication_provenance_and_separate_popularity() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let mut db = fixture(dir.path())?; + let before: i64 = db.query_row("SELECT count(*) FROM facts", [], |r| r.get(0))?; + import( + &mut db, + dir.path(), + Source::Wikidata, + Format::WikidataEntities, + &serde_json::to_vec(&common::wikidata())?, + )?; + let after: i64 = db.query_row("SELECT count(*) FROM facts", [], |r| r.get(0))?; + assert_eq!(before, after); + let mut selectors = db.prepare("SELECT r.raw_json,f.selector FROM facts f JOIN records r ON r.source_id=f.source_id AND r.ordinal=f.ordinal JOIN sources s ON s.id=f.source_id WHERE s.source='wikidata' AND f.predicate='name'")?; + for item in selectors.query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? { + let (raw, selector) = item?; + assert!( + serde_json::from_str::(&raw)? + .pointer(&selector) + .is_some(), + "invalid source selector {selector}" + ); + } + let r = build(&db, dir.path(), "generation")?; + assert_eq!(r.lookup("FB", 10)?.candidates[0].canonical_name, "Facebook"); + assert_eq!(r.lookup("Café Atlas", 10)?.total_edges, 3); + let facebook = r.lookup("Facebook", 10)?; + assert_eq!(facebook.total_entities, 1); + let c = &facebook.candidates[0]; + assert_eq!(c.provenance[0]["source"]["license"], "CC0-1.0"); + assert!(c.evidence["assertion"]["statement"]["references"].is_array()); + assert_eq!(r.receipt.properties, 4); // Curlie Facebook shares the same property. + assert_eq!(r.receipt.entities, 3); // Curlie is not automatically the Wikidata entity. + assert!(r.resolve("Facebook", None, None, timestamp()?)?.is_none()); + let export = dir.path().join("export.jsonl"); + argand_site_registry::release::export(&r, &export, false)?; + let text = std::fs::read_to_string(&export)?; + assert!(!text.contains("Synthetic editorial description")); + assert!(!text.contains("Synthetic category description")); + assert!(text.contains("With content from Curlie.org")); + assert!(text.contains("referring_subnets") && text.contains("coarse_rank")); + Ok(()) +} + +#[test] +fn regional_review_expiry_and_revocation() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let db = fixture(dir.path())?; + let candidate = build(&db, dir.path(), "candidate")?; + approve( + &db, + &candidate, + "Atlas", + "https://atlas.example.com/", + "primary", + "", + )?; + approve( + &db, + &candidate, + "Atlas", + "https://atlas.example.co.uk/", + "regional", + "GB", + )?; + let approved = build(&db, dir.path(), "approved")?; + assert_eq!( + approved + .resolve("Atlas", None, Some("GB"), timestamp()?)? + .context("regional missing")? + .url, + "https://atlas.example.co.uk/" + ); + assert_eq!( + approved + .resolve("Atlas", None, Some("DE"), timestamp()?)? + .context("primary missing")? + .url, + "https://atlas.example.com/" + ); + assert!( + approved + .resolve( + "Atlas", + None, + None, + timestamp()? + chrono::Duration::days(8) + )? + .is_none() + ); + let fingerprint = approved + .lookup("Atlas", 10)? + .candidates + .into_iter() + .find(|c| c.url == "https://atlas.example.co.uk/") + .context("missing GB")? + .fingerprint; + review::record( + &db, + &approved, + &Review { + fingerprint, + decision: "revoke".into(), + reviewer: "test".into(), + reason: "test revocation".into(), + evidence: "synthetic:revocation".into(), + reviewed_at: timestamp()?, + expires_at: timestamp()?, + role: "unspecified".into(), + locale: String::new(), + country: String::new(), + }, + )?; + let revoked = build(&db, dir.path(), "revoked")?; + assert_eq!( + revoked + .resolve("Atlas", None, Some("GB"), timestamp()?)? + .context("fallback missing")? + .url, + "https://atlas.example.com/" + ); + Ok(()) +} + +#[test] +fn ambiguity_survives_limits_and_same_named_domains_do_not_merge() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let mut db = fixture(dir.path())?; + let raw = json!({"entities":{"Q900002":common::entity("Q900002","Atlas Fixture",&["Atlas"],&["https://atlas.example.fr/"])}}); + let bytes = serde_json::to_vec(&raw)?; + let mut m = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?; + m.scope = "additional".into(); + let path = dir.path().join("conflict.json"); + std::fs::write(&path, bytes)?; + store::import(&mut db, &m, &path)?; + let r = build(&db, dir.path(), "conflicts")?; + let result = r.lookup("Atlas", 1)?; + assert_eq!(result.total_entities, 2); + assert_eq!(result.total_edges, 4); + assert!(result.truncated); + assert!(r.resolve("Atlas", None, None, timestamp()?)?.is_none()); + Ok(()) +} + +#[test] +fn changed_names_and_urls_invalidate_approval_but_history_survives() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let mut db = fixture(dir.path())?; + let initial = build(&db, dir.path(), "initial")?; + approve( + &db, + &initial, + "Facebook", + "https://facebook.com/", + "primary", + "", + )?; + let mut raw = common::wikidata(); + raw["entities"]["Q355"]["aliases"]["en"] + .as_array_mut() + .context("aliases")? + .push(json!({"language":"en","value":"New alias"})); + let bytes = serde_json::to_vec(&raw)?; + let mut m = common::manifest(Source::Wikidata, Format::WikidataEntities, &bytes)?; + m.snapshot = "v2".into(); + m.retrieved_at += chrono::Duration::days(1); + let path = dir.path().join("v2.json"); + std::fs::write(&path, bytes)?; + store::import(&mut db, &m, &path)?; + let r = build(&db, dir.path(), "changed")?; + assert!( + r.resolve( + "Facebook", + None, + None, + timestamp()? + chrono::Duration::days(1) + )? + .is_none() + ); + let sources: i64 = db.query_row( + "SELECT count(*) FROM sources WHERE source='wikidata' AND complete=1", + [], + |r| r.get(0), + )?; + assert_eq!(sources, 2); + assert_eq!( + initial.lookup("Facebook", 10)?.candidates[0].entity_id, + r.lookup("Facebook", 10)?.candidates[0].entity_id + ); + Ok(()) +} + +#[test] +fn compressed_dumps_resume_and_corruption_fail_closed() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let mut db = store::open(&dir.path().join("store"))?; + let mut dump = String::from("[\n"); + for i in 0..300 { + if i > 0 { + dump.push_str(",\n"); + } + dump.push_str(&serde_json::to_string(&common::entity( + &format!("Q{}", 900_000 + i), + &format!("Fixture {i}"), + &[], + &["https://example.com/"], + ))?); + } + dump.push_str("\n]\n"); + let bytes = common::gzip(dump.as_bytes())?; + let mut m = common::manifest(Source::Wikidata, Format::WikidataDump, &bytes)?; + m.compression = Compression::Gzip; + let path = dir.path().join("dump.gz"); + std::fs::write(&path, &bytes)?; + // Interrupt the sink after its first committed batch; rerun the exact input. + db.execute_batch("CREATE TRIGGER simulated_crash BEFORE INSERT ON records WHEN NEW.ordinal=270 BEGIN SELECT RAISE(ABORT,'simulated interruption'); END;")?; + assert!(store::import(&mut db, &m, &path).is_err()); + let checkpoint: i64 = db.query_row("SELECT checkpoint FROM sources", [], |r| r.get(0))?; + assert_eq!(checkpoint, 256); + db.execute_batch("DROP TRIGGER simulated_crash")?; + store::import(&mut db, &m, &path)?; + let records: i64 = db.query_row("SELECT count(*) FROM records", [], |r| r.get(0))?; + assert_eq!(records, 300); + let mut damaged = bytes.clone(); + damaged.pop(); + std::fs::write(&path, &damaged)?; + assert!(store::import(&mut db, &m, &path).is_err()); + let mut truncated = common::manifest(Source::Wikidata, Format::WikidataDump, &damaged)?; + truncated.compression = Compression::Gzip; + assert!(store::import(&mut db, &truncated, &path).is_err()); + let complete: i64 = db.query_row( + "SELECT complete FROM sources WHERE id=?1", + [truncated.id()?], + |r| r.get(0), + )?; + assert_eq!(complete, 0); + Ok(()) +} + +#[test] +fn repeated_builds_have_identical_bytes_and_corruption_is_rejected() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let db = fixture(dir.path())?; + let first = build(&db, dir.path(), "first")?; + let second = build(&db, dir.path(), "second")?; + assert_eq!(first.identity, second.identity); + assert_eq!( + first.receipt.database_sha256, + second.receipt.database_sha256 + ); + assert!(argand_site_registry::build::build(&db, &dir.path().join("first")).is_err()); + let path = dir.path().join("second/registry.sqlite"); + std::fs::write(path, b"corrupted")?; + assert!(Registry::open(&dir.path().join("second"), &second.identity).is_err()); + ensure!( + first.lookup("Facebook", 1)?.total_entities == 1, + "old generation damaged" + ); + Ok(()) +} diff --git a/docs/CONSUMERS.md b/docs/CONSUMERS.md new file mode 100644 index 0000000..8afc168 --- /dev/null +++ b/docs/CONSUMERS.md @@ -0,0 +1,57 @@ +# Consumer and compatibility contract + +## Rust library + +Use `argand_site_registry::query::Registry::open(generation, trusted_pin)` once per +immutable generation and reuse the reader. `lookup(query, limit)` returns evidence +and complete ambiguity counts; `resolve(query, locale, country, now)` returns an +optional reviewed candidate. Check the compiled example and API docs for exact +types. `selection_context` binds the full alternative set for downstream query +review. Preserve the returned provenance, scopes, counts and attribution. + +For local integration, point a Cargo dependency at +`crates/argand-site-registry` inside an extracted standalone source tree. Once an +upstream repository is published, use its actual Git URL and a full reviewed `rev` +pin. Do not invent a crates.io version or track a mutable branch in production. +Both crates remain in this workspace; the atomic helper is a relative dependency. + +## CLI and other languages + +`lookup`, `resolve`, `verify` and the other commands emit JSON. The Python example +passes arguments directly to the native executable, preserving query text and the +entire response. A nonzero exit is an error. `destination: null` is a successful +abstention, not a request to pick the first lookup candidate. Render source names +as untrusted text and satisfy their source-specific attribution requirements. + +Use a bounded process or service wrapper appropriate to your workload. The Python +example has a 60-second timeout and invokes a fresh reader per request; for repeated +low-latency queries, use the reusable Rust reader. No hosted API or Python package +registry publication is claimed by this repository. + +## SQLite, JSONL and license scope + +Distribute `registry.sqlite`, `COMPLETE.json`, `LICENSE_SOURCES.md` and +`ATTRIBUTION.json` together, plus the publisher signature when applicable. The +database contains audit records and source descriptions. Default JSONL export +omits descriptions and includes fact provenance plus an attribution envelope. +It is an assertion export, not a self-contained signed list of admitted routes. +Raw SQL inspection is useful for audit; it does not implement resolution policy. + +Code version 0.1.0 is an initial interface. Schema/rule contracts are versioned +independently in receipts. Unsupported contracts fail closed. Pin source releases, +compile consumers and replay fixed fixtures before upgrades. Preserve import and +review history; never mutate complete generations to migrate them. + +## Argand transition + +UPSTREAM.json pins the exact Argand source baseline, including the beta agent's +selection-context API. The first standalone extraction preserves runtime Rust and +migration bytes. Argand currently consumes its embedded workspace crate; this +package does not silently redirect that dependency. + +At cutover, coordinate with the Argand source/build owner, compare both trees with +the recorded baseline, carry any subsequent fixes forward, replace the embedded +dependency with a reviewed standalone revision, and run Argand's complete engine +and navigation-compiler gates. Preserve existing registry receipts and public +navigation admission. After cutover, develop the library upstream and update +Argand through explicit pinned dependency changes, avoiding permanent dual copies. diff --git a/docs/INDEX.md b/docs/INDEX.md new file mode 100644 index 0000000..0a3571e --- /dev/null +++ b/docs/INDEX.md @@ -0,0 +1,13 @@ +# Documentation + +- [README](../README.md): build, examples and scope. +- [Operator guide](../crates/argand-site-registry/README.md): all source commands, + schema, reviews, regional resolution, releases and update configuration. +- [Source licenses](../crates/argand-site-registry/LICENSE_SOURCES.md): exact terms and attribution. +- [Consumers](CONSUMERS.md): Rust, Python/CLI, data distribution and Argand transition. +- [Trust](TRUST.md): enforced checks and publisher/consumer responsibilities. +- [Releasing](RELEASING.md): CI, source signing and archive verification. +- [Contributing](../CONTRIBUTING.md), [governance](../GOVERNANCE.md), + [security](../SECURITY.md): proposals, decisions and incidents. +- [Extraction design](superpowers/specs/2026-09-12-standalone-design.md) and + [implementation plan](superpowers/plans/2026-09-12-standalone.md). diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..dbe2e17 --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,92 @@ +# Source and dataset releases + +## Build environment and CI + +The two-crate workspace requires the normal Rust/C build toolchain, OpenSSH and +Python 3.11+ for checks and packaging. The initial extraction declares Rust 1.97 +and was tested on Linux with Rust 1.98.1. Install rustfmt and Clippy alongside the +compiler. TLS dependencies may require CMake and Perl. Keep Cargo.lock tracked. +Run `cargo fetch --locked` once, then `bash scripts/check.sh` offline. Cargo's +[workspace inheritance](https://doc.rust-lang.org/cargo/reference/workspaces.html) +and [locked/offline options](https://doc.rust-lang.org/cargo/commands/cargo-test.html) +define the build behavior. A lockfile pins dependencies, not the host compiler. + +The Forgejo workflow uses the documented [workflow and context syntax](https://forgejo.org/docs/latest/user/actions/reference/). +Register `site-registry-isolated` only on a disposable, repository-scoped runner +with the above tools, two build jobs and at least 4 GiB memory. Use a pinned, +reviewed runner image. Do not mount production directories, share signing keys, +or use Argand's host runners. Follow Forgejo's [runner security guidance](https://forgejo.org/docs/latest/user/actions/security/). +The label is a deployment requirement, not a provisioned runner supplied by this +repository. The workflow runs on trusted main pushes or manual dispatch, fetches +the exact public commit without credentials, runs offline gates, compares two +source archives and reruns acceptance on extracted source. It neither publishes +artifacts nor signs them. Review outside contributions before allowing them to +execute on infrastructure. No hosted CI pass is claimed by a local test run. + +## Deterministic source archive + +From a reviewed, clean committed tree, choose a new output directory outside the +checkout (its parent must exist): + +```bash +python3 scripts/source_release.py create --output /data/releases/site-registry-source +``` + +The command emits the full receipt `pin` and creates `source.tar.gz` plus +`RELEASE.json`. The archive contains only allowed tracked source files from the +exact commit, including both crates, lockfile and license texts. Tar metadata and +gzip timestamps are fixed; identical source commits and Python/zlib packaging +versions produce identical bytes. The receipt records those packaging versions. +Links, submodules, datasets, private-key extensions and unsafe paths are refused. +The complete source tree is limited to 2,000 files and 32 MiB; review these limits +before expanding them. The allowlist is not a secret scanner: review source contents +for embedded credentials and unrelated material before signing or publication. + +The receipt records the Git commit/tree and each path's hash, size and mode. +It is written last and fsynced with its directory. Existing output is never +overwritten. An interrupted directory without a valid receipt is incomplete; +inspect it and choose a fresh output path. The verifier bounds decompression, +rejects unexpected/missing/duplicate files, and checks every member before any +extraction. It accepts an external receipt pin; a hash found in the same download +is insufficient authentication. + +## Signing and consuming source + +After all checks and source review, sign with an operator-controlled SSH key: + +```bash +ssh-keygen -Y sign -n argand-site-registry-source -f /secure/source-signing-key \ + /data/releases/site-registry-source/RELEASE.json +``` + +Distribute the archive, receipt and `RELEASE.json.sig` together. Consumers obtain +the accepted identity and allowed-signers file through an independent trusted +channel, verify the signature, then compute the now-authenticated receipt pin: + +```bash +ssh-keygen -Y verify -n argand-site-registry-source \ + -f /secure/source-allowed-signers -I registry-source-publisher \ + -s /data/releases/site-registry-source/RELEASE.json.sig \ + < /data/releases/site-registry-source/RELEASE.json +sha256sum /data/releases/site-registry-source/RELEASE.json +python3 scripts/source_release.py verify \ + --release /data/releases/site-registry-source --pin "$REGISTRY_TRUSTED_PIN" +``` + +Set `REGISTRY_TRUSTED_PIN` to that authenticated receipt hash. Use a verifier you +already trust, not one extracted from an unchecked archive. After verification, +extract into an empty directory and run `cargo fetch --locked` and +`bash scripts/check.sh`. Preserve the receipt with the installation. A source +signature authenticates reviewed source bytes, not an arbitrary executable built +elsewhere. Binary distribution needs its own artifact hash and build-environment +receipt; the initial release tool packages source only. + +## Dataset publication + +Source releases contain no provider datasets or real approvals. Dataset publishers +follow the operator guide: import, inspect, review, build, diff, sign and activate. +The dataset namespace `argand-site-registry` is distinct from the source namespace +above. Keep `LICENSE_SOURCES.md` and `ATTRIBUTION.json` with the database and receipt. +The weekly update example creates candidates. It never approves, renews, signs or +activates them. Record actual acquisition cost and dataset size separately from +small-fixture test results. Follow SECURITY.md for incidents and revocation delivery. diff --git a/docs/TRUST.md b/docs/TRUST.md new file mode 100644 index 0000000..28fef8f --- /dev/null +++ b/docs/TRUST.md @@ -0,0 +1,63 @@ +# Trust and evidence policy + +## What the implementation enforces + +Inputs use reviewed source adapters and explicit manifests. URL normalization, +PSL parsing and stable identities are deterministic. Fact provenance, conflicting +evidence and source-specific popularity remain separate. Names and similar +hostnames never silently join entities. Invalid URLs and malformed imports fail +validation; a partial import does not replace a complete source selection. + +Destination and identity decisions bind exact evidence fingerprints, including +names, assertions and normalization context. Reviews expire within 90 days. +Changed evidence invalidates earlier approvals. Resolution abstains on ambiguity, +ties, missing approval or ineligible claims; regional scopes must explicitly +match. The local writer owns the append-only review log. + +Generations bind the database, license document and attribution to a completion +receipt. Consumers provide a trusted hash or verify an external publisher key. +Activation checks signatures and refuses rollback that loses distributed +revocations. Updates build candidates and cannot approve, sign or activate them. + +## What a publisher must establish + +The reviewer name and evidence locator in a decision are operator assertions. +The CLI validates their structure and evidence binding; it does not independently +authenticate the reviewer, retrieve their evidence or prove website ownership. +Protect the writer database and signing key with separate operating permissions. +Restrict who can author decisions and require human review before release signing. + +Publish dated evidence supporting the exact entity, URL, relationship, role and +country/locale. Prefer independently corroborated primary evidence with immutable +capture identifiers. Record contrary evidence and uncertainty. TLS, DNS control, +registrable-domain spelling, redirects, `sameAs` or popularity alone cannot +establish every identity or role claim. Future crawler observations remain inputs +to review. Confidence values are assertion scores, not calibrated probabilities. + +Choose expiry based on volatility, within the enforced maximum. Do not renew +blindly on a timer. Expired approval should lead to abstention until evidence is +reviewed. Disclose editorial conflicts and use an independent reviewer for a +disputed claim when possible. The initial implementation is a local single-writer +tool; it does not provide authenticated reviewer accounts or an enforced quorum. + +## What consumers must preserve + +Authenticate a release before opening it. Keep the full receipt pin and required +attribution with caches and exports. Use `resolve` for reviewed destinations, keep +null as abstention, and enforce application-specific malware/content/navigation +policy separately. A verified signature authenticates the publisher, not the truth +of every assertion. An official website may later be compromised. + +Deliver revocations to every active consumer and derived catalogue, and bound cache +lifetimes. The resolver checks review expiry at query time; a detached cached URL +does not recheck itself. Preserve the current review history during rollback. +Copying SQLite rows or the JSONL export into a second resolver can bypass these +checks; use the native API or CLI for admission decisions. + +## Community changes + +Treat public submissions as untrusted evidence. Do not execute submitted content +or copy review decisions into production automatically. Source allowlist and +attribution changes need both implementation tests and documented rights review. +Reject unknown sources until those checks are complete. Every publisher may apply +stricter admission rules, and must state its actual review and incident procedures. diff --git a/docs/superpowers/plans/2026-09-12-standalone.md b/docs/superpowers/plans/2026-09-12-standalone.md new file mode 100644 index 0000000..42fd6d5 --- /dev/null +++ b/docs/superpowers/plans/2026-09-12-standalone.md @@ -0,0 +1,37 @@ +# Standalone Site Registry Implementation Plan + +> **For agentic workers:** Use superpowers:executing-plans inline. No subagents or Argand worktrees. + +**Goal:** Produce a reusable standalone repository and verifiable source release. + +**Architecture:** Retain the registry and atomic helper as two workspace crates. +Preserve runtime bytes and wire packaging, consumer examples and release gates +around their existing APIs. Use an isolated build directory. + +**Tech Stack:** Rust, SQLite, Python 3.11 standard library tooling, OpenSSH, Forgejo Actions. + +### Task 1: Extract the verified baseline +- [x] Verify signed Argand commit 47911062b00d87f215ba61c41965faf8a7f4b7f7. +- [x] Export only `engine/crates/argand-site-registry` and `engine/crates/argand-atomic`. +- [x] Write `UPSTREAM.json` with original path/blob/hash evidence and a two-crate `Cargo.toml`. +- [ ] Prune the inherited lock with `cargo metadata --offline --format-version 1`; retain exact dependency versions. + +### Task 2: Make standalone use and trust policy concrete +- [ ] Add root README, LICENSE, source-license entrypoint, CONTRIBUTING, SECURITY and governance docs. +- [ ] Add Rust `examples/lookup.rs` and Python `examples/lookup.py` consumers of existing query contracts. +- [ ] Run both consumers on the native fixture and compare full lookup outputs, including attribution. + +### Task 3: Package and verify source releases +- [ ] Add deterministic source archive and verification commands in `scripts/source_release.py`. +- [ ] Test determinism, dirty-tree refusal, no-clobber, tampering and unsafe archive members in `tests/test_source_release.py`. +- [ ] Add `scripts/check.sh` and `.forgejo/workflows/ci.yml` using an isolated runner without release secrets. +- [ ] Document explicit release signing, downstream pins and incident response in `docs/RELEASING.md` and `docs/TRUST.md`. + +### Task 4: Validate and land the bounded task +- [ ] Run `cargo fmt --all -- --check`, `cargo check --workspace --all-targets --locked --offline`, + `cargo clippy --workspace --all-targets --locked --offline -- -D warnings`, + `cargo test --workspace --locked --offline` and strict `cargo doc`. +- [ ] Build the native executable and retain the all-five-source CLI fixture outside the repo. +- [ ] Rebuild/run tests from the release archive outside the original workspace. +- [ ] Review every initial tracked file and dependency change, sign the local commit and release receipt. +- [ ] Record exact validation, publication and Argand cutover state; release any coordination window. diff --git a/docs/superpowers/specs/2026-09-12-standalone-design.md b/docs/superpowers/specs/2026-09-12-standalone-design.md new file mode 100644 index 0000000..add2b77 --- /dev/null +++ b/docs/superpowers/specs/2026-09-12-standalone-design.md @@ -0,0 +1,37 @@ +# Standalone Site Registry design + +The user approved continuing standalone packaging, release CI and public governance +on 2026-09-12. Preserve the existing AGPL-3.0-or-later license and all data terms. + +Create a separate two-crate workspace containing the current registry and atomic +file helper. Preserve Rust module boundaries and the existing source adapters, +SQLite schema, receipt contracts, explicit reviews and regional resolution. +Record the exact signed Argand baseline and original file hashes in UPSTREAM.json. +Do not copy Argand history, deployment configuration, datasets or signing keys. +The existing Argand checkout and shared build caches remain owned by the beta agent. + +A separate package is preferable to a permanent second implementation or a new +hosted service: it reuses the existing contracts and runs locally. Extracting only +the CLI would lose the reusable Rust API. Preserve both CLI and library, and prove +non-Rust consumption through the JSON CLI without creating a second resolver. + +Ship a focused Cargo.lock, complete code license, beginner quickstart, native Rust +and Python examples, contribution requirements, trust policy and incident/revocation +procedure. Source releases come from an exact committed tree, with deterministic +archives and hash receipts; signing is an explicit local release action. CI checks +formatting, all targets, strict lints, tests, documentation and a native all-source +fixture, and exercises the archive outside its Git checkout. No CI signing key, +provider credentials, production runner or automatic dataset promotion is included. + +Keep Argand's current consumer operational. Until an independently hosted release +is accepted into Argand, its embedded copy remains the active consumer; document +the cutover and verify baseline parity. The initial standalone package has no +runtime behavior changes. Each dataset publisher chooses its own external trust +roots; review policy is public and cannot be replaced by popularity or hostname +similarity. An authenticated release is not proof of source accuracy or malware safety. + +Acceptance: preserve upstream Rust/schema bytes; build with only this workspace; +run the complete two-crate offline gates plus fresh CLI and Rust/Python examples; +produce the same source archive twice; reject modified releases, unsafe paths and +release overwrites; inspect the initial Git inventory for private/unrelated content. +Remote publication and runner activation are separate from local package readiness. diff --git a/examples/lookup.py b/examples/lookup.py new file mode 100644 index 0000000..791a8d8 --- /dev/null +++ b/examples/lookup.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Use the native registry from Python, preserving its attribution and provenance.""" + +import argparse +import json +import subprocess + + +def lookup(binary, generation, pin, query): + """The native reader authenticates the generation using the caller's trusted pin.""" + result = subprocess.run( + [binary, "lookup", "--generation", generation, "--pin", pin, + "--query", query, "--limit", "100"], + check=True, capture_output=True, text=True, timeout=60, + ) + return json.loads(result.stdout) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", default="argand-site-registry") + parser.add_argument("--generation", required=True) + parser.add_argument("--pin", required=True) + parser.add_argument("--query", required=True) + args = parser.parse_args() + print(json.dumps(lookup(args.binary, args.generation, args.pin, args.query), + ensure_ascii=False, sort_keys=True)) diff --git a/scripts/check.sh b/scripts/check.sh new file mode 100644 index 0000000..348bbad --- /dev/null +++ b/scripts/check.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# By Nic Weyand! Run from a checkout or an extracted source release. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-2}" +export CARGO_TERM_COLOR=never +registry_check_root="${ARGAND_REGISTRY_CHECK_OUTPUT:-$(mktemp -d -t site-registry-check.XXXXXXXX)}" +mkdir -p "$registry_check_root" +registry_check_root="$(cd "$registry_check_root" && pwd)" +if [[ -e "$registry_check_root/fixture" ]]; then + echo 'Choose a new check output directory; fixture evidence is never overwritten.' >&2 + exit 1 +fi + +cargo fmt --all -- --check +cargo check --workspace --all-targets --locked --offline +cargo clippy --workspace --all-targets --locked --offline -- -D warnings +ARGAND_REGISTRY_E2E_OUTPUT="$registry_check_root/fixture" \ + cargo test --workspace --locked --offline +RUSTDOCFLAGS="${RUSTDOCFLAGS:-} -D warnings" \ + cargo doc --workspace --no-deps --locked --offline +python3 -m unittest discover -s tests -v +cargo build --workspace --bins --examples --locked --offline +cargo metadata --no-deps --format-version 1 --locked --offline > "$registry_check_root/metadata.json" +registry_target="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["target_directory"])' "$registry_check_root/metadata.json")" +python3 scripts/check_consumers.py --binary "$registry_target/debug/argand-site-registry" \ + --rust-example "$registry_target/debug/examples/lookup" --fixture "$registry_check_root/fixture" +echo "All checks passed. Synthetic fixture evidence: $registry_check_root" diff --git a/scripts/check_consumers.py b/scripts/check_consumers.py new file mode 100644 index 0000000..9938bc0 --- /dev/null +++ b/scripts/check_consumers.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Compare shipped native CLI, Rust library and Python example on authored fixtures.""" + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path + + +def run(command): + result = subprocess.run(command, check=True, capture_output=True, text=True, timeout=60) + return json.loads(result.stdout) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--rust-example", type=Path, required=True) + parser.add_argument("--fixture", type=Path, required=True) + args = parser.parse_args() + example = Path(__file__).resolve().parents[1] / "examples" / "lookup.py" + generation = args.fixture / "candidate" + # This local test generated the fixture itself. Production pins come from + # an authenticated publisher, as explained in the consumer documentation. + pin = hashlib.sha256((generation / "COMPLETE.json").read_bytes()).hexdigest() + for query in ("FB", "Atlas", "Café Atlas", "$(echo unsafe); `id`"): + shared = ["--generation", str(generation), "--pin", pin, "--query", query] + cli = run([str(args.binary), "lookup", *shared, "--limit", "100"]) + rust = run([str(args.rust_example), *shared]) + python = run([sys.executable, str(example), "--binary", str(args.binary), *shared]) + if cli != rust or cli != python: + raise ValueError(f"consumer output differs for {query!r}") + if not cli["attribution"]: + raise ValueError("consumer lost attribution") + if query == "FB": + candidate = cli["candidates"][0] + if candidate["canonical_name"] != "Facebook" or not candidate["provenance"]: + raise ValueError("name/provenance contract mismatch") + if candidate["web_property"]["domain"]["registrable_domain"] != "facebook.com": + raise ValueError("domain contract mismatch") + if query == "Atlas": + urls = {candidate["url"] for candidate in cli["candidates"]} + if urls != {"https://atlas.example.com/", "https://atlas.example.co.uk/", + "https://atlas.example.de/"}: + raise ValueError("regional properties were lost") + bad = ["--generation", str(generation), "--pin", "0" * 64, "--query", "FB"] + for command in ([str(args.binary), "lookup", *bad], [str(args.rust_example), *bad], + [sys.executable, str(example), "--binary", str(args.binary), *bad]): + result = subprocess.run(command, capture_output=True, timeout=60) + if result.returncode == 0: + raise ValueError("consumer accepted an untrusted generation") + print("Native CLI, Rust and Python consumer parity passed, including trust failures.") + + +if __name__ == "__main__": + main() diff --git a/scripts/source_release.py b/scripts/source_release.py new file mode 100644 index 0000000..2d7ea3f --- /dev/null +++ b/scripts/source_release.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Create deterministic source releases; verify them against an external receipt pin.""" + +import argparse +import gzip +import hashlib +import io +import json +import os +import platform +import re +import subprocess +import tarfile +import tomllib +import zlib +from pathlib import Path, PurePosixPath + +ARCHIVE = "source.tar.gz" +RECEIPT = "RELEASE.json" +SCHEMA = "argand.site-source-release/v1" +MAX_BYTES = 32 * 1024 * 1024 +MAX_FILES = 2000 +MAX_TAR_BYTES = MAX_BYTES + MAX_FILES * 1024 + 10240 + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def read_bounded(path, maximum): + if path.is_symlink() or not path.is_file(): + raise ValueError(f"expected a regular file: {path.name}") + with path.open("rb") as stream: + data = stream.read(maximum + 1) + if len(data) > maximum: + raise ValueError("release exceeds size limit") + return data + + +def safe_path(name): + path = PurePosixPath(name) + if not name or path.is_absolute() or str(path) != name: + raise ValueError("noncanonical archive path") + if any(part in ("..", ".git", "target", "build", "dist", "data", "cache", + "__pycache__") for part in path.parts): + raise ValueError("excluded archive path") + if any(ord(char) < 32 for char in name) or "\\" in name: + raise ValueError("unsafe archive path") + allowed = {".rs", ".toml", ".md", ".py", ".sh", ".sql", ".yml", ".yaml", + ".service", ".timer"} + if path.suffix not in allowed and path.name != ".gitignore" and name not in ( + "LICENSE", "Cargo.lock", "UPSTREAM.json", ".gitignore"): + raise ValueError(f"file is outside the source release allowlist: {name}") + + +def git(root, *args): + return subprocess.check_output(["git", "-C", str(root), *args]) + + +def write_new(path, data): + with path.open("xb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + + +def sync_directory(path): + descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def create(root, output): + root = root.resolve() + if git(root, "rev-parse", "--show-toplevel").decode().strip() != str(root): + raise ValueError("run from the source repository root") + if git(root, "status", "--porcelain", "--untracked-files=all"): + raise ValueError("source release requires a clean committed tree") + commit = git(root, "rev-parse", "HEAD").decode().strip() + tree = git(root, "rev-parse", f"{commit}^{{tree}}").decode().strip() + manifest = git(root, "show", f"{commit}:Cargo.toml") + version = tomllib.loads(manifest.decode())["workspace"]["package"]["version"] + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?", version): + raise ValueError("unsupported release version") + prefix = f"argand-site-registry-{version}/" + files = [] + buffer = io.BytesIO() + total = 0 + with gzip.GzipFile(fileobj=buffer, mode="wb", filename="", mtime=0, compresslevel=9) as compressed: + with tarfile.open(fileobj=compressed, mode="w", format=tarfile.USTAR_FORMAT) as archive: + for entry in git(root, "ls-tree", "-rz", commit).split(b"\0"): + if not entry: + continue + header, raw_path = entry.split(b"\t", 1) + mode, kind, blob = header.decode().split() + name = raw_path.decode() + safe_path(name) + if kind != "blob" or mode not in ("100644", "100755"): + raise ValueError("source releases cannot contain links or submodules") + size = int(git(root, "cat-file", "-s", blob)) + total += size + if total > MAX_BYTES or len(files) >= MAX_FILES: + raise ValueError("source tree exceeds release limits") + data = git(root, "cat-file", "blob", blob) + info = tarfile.TarInfo(prefix + name) + info.size, info.mode = size, int(mode[-3:], 8) + archive.addfile(info, io.BytesIO(data)) + files.append({"path": name, "bytes": size, "mode": info.mode, + "sha256": digest(data)}) + required = {"LICENSE", "Cargo.lock", "Cargo.toml", "UPSTREAM.json", + "crates/argand-site-registry/LICENSE_SOURCES.md"} + if not required.issubset({item["path"] for item in files}): + raise ValueError("source release is missing required license or build inputs") + payload = buffer.getvalue() + receipt = {"schema": SCHEMA, "commit": commit, "tree": tree, "version": version, + "archive": ARCHIVE, "prefix": prefix, "bytes": len(payload), + "sha256": digest(payload), "files": files, + "packager": {"python": platform.python_version(), + "zlib": zlib.ZLIB_RUNTIME_VERSION}} + encoded = (json.dumps(receipt, indent=2, sort_keys=True) + "\n").encode() + output.mkdir(mode=0o700, parents=False) # Never overwrite an earlier release. + write_new(output / ARCHIVE, payload) + write_new(output / RECEIPT, encoded) # Completion receipt is written last. + sync_directory(output) + sync_directory(output.parent) + return digest(encoded) + + +def verify(output, pin): + if not re.fullmatch(r"[0-9a-f]{64}", pin): + raise ValueError("supply a full externally trusted receipt SHA-256") + encoded = read_bounded(output / RECEIPT, 1024 * 1024) + if digest(encoded) != pin: + raise ValueError("source receipt pin mismatch") + receipt = json.loads(encoded, object_pairs_hook=unique_object) + if receipt["schema"] != SCHEMA or receipt["archive"] != ARCHIVE: + raise ValueError("unsupported source release") + if not re.fullmatch(r"argand-site-registry-[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?/", + receipt["prefix"]): + raise ValueError("unsafe archive prefix") + payload = read_bounded(output / ARCHIVE, MAX_BYTES) + if len(payload) != receipt["bytes"] or digest(payload) != receipt["sha256"]: + raise ValueError("source archive digest mismatch") + files = receipt["files"] + expected = {item["path"]: item for item in files} + if len(expected) != len(files) or not 1 <= len(files) <= MAX_FILES: + raise ValueError("duplicate or excessive source entries") + for name in expected: + safe_path(name) + seen, total = set(), 0 + # Bound expansion before tarfile parses potentially large extended headers. + with gzip.GzipFile(fileobj=io.BytesIO(payload)) as compressed: + expanded = compressed.read(MAX_TAR_BYTES + 1) + if len(expanded) > MAX_TAR_BYTES: + raise ValueError("expanded archive exceeds limit") + with tarfile.open(fileobj=io.BytesIO(expanded), mode="r:") as archive: + for member in archive: + if not member.isfile() or not member.name.startswith(receipt["prefix"]): + raise ValueError("unsafe archive member") + name = member.name.removeprefix(receipt["prefix"]) + safe_path(name) + if name in seen or name not in expected: + raise ValueError("duplicate or unexpected archive member") + item = expected[name] + total += member.size + if total > MAX_BYTES or member.size != item["bytes"]: + raise ValueError("archive size mismatch") + if member.mode not in (0o644, 0o755) or member.mode != item["mode"]: + raise ValueError("archive mode mismatch") + stream = archive.extractfile(member) + if stream is None or digest(stream.read(MAX_BYTES + 1)) != item["sha256"]: + raise ValueError("source file digest mismatch") + seen.add(name) + if seen != expected.keys(): + raise ValueError("archive omits source files") + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + make = commands.add_parser("create") + make.add_argument("--output", type=Path, required=True) + check = commands.add_parser("verify") + check.add_argument("--release", type=Path, required=True) + check.add_argument("--pin", required=True) + args = parser.parse_args() + try: + if args.command == "create": + result = {"release": str(args.output), "pin": create(Path.cwd(), args.output)} + else: + result = verify(args.release, args.pin) + print(json.dumps(result, sort_keys=True)) + except (OSError, ValueError, KeyError, TypeError, tarfile.TarError, + subprocess.CalledProcessError) as error: + parser.exit(1, f"source release refused: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/tests/test_source_release.py b/tests/test_source_release.py new file mode 100644 index 0000000..9ca9e3b --- /dev/null +++ b/tests/test_source_release.py @@ -0,0 +1,139 @@ +"""Exercise the shipped source packager against actual Git trees and corrupt archives.""" + +import importlib.util +import io +import json +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path + +MODULE = Path(__file__).resolve().parents[1] / "scripts" / "source_release.py" +SPEC = importlib.util.spec_from_file_location("source_release", MODULE) +release = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(release) + + +class SourceReleaseTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.keys = tempfile.TemporaryDirectory() + cls.key = Path(cls.keys.name) / "fixture" + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(cls.key)], + check=True) + + @classmethod + def tearDownClass(cls): + cls.keys.cleanup() + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) / "source with spaces" + self.root.mkdir() + self.git("init", "-q", "-b", "main") + self.git("config", "user.name", "Synthetic test") + self.git("config", "user.email", "test@example.invalid") + self.git("config", "core.excludesFile", "/dev/null") + self.git("config", "gpg.format", "ssh") + self.git("config", "user.signingkey", str(self.key)) + self.git("config", "commit.gpgsign", "true") + files = { + "Cargo.toml": '[workspace.package]\nversion="0.1.0"\n', + "Cargo.lock": "# Synthetic lock\n", + "LICENSE": "Synthetic code license fixture\n", + "UPSTREAM.json": "{}\n", + "crates/argand-site-registry/LICENSE_SOURCES.md": "Synthetic source terms\n", + "src/lib.rs": "// Synthetic Rust source\n", + "src/.gitignore": "*.temporary\n", + } + for name, text in files.items(): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + self.commit() + self.output = self.root.parent / "release" + + def git(self, *args): + return subprocess.check_output(["git", "-C", str(self.root), *args]) + + def commit(self): + self.git("add", "--all") + self.git("commit", "-q", "-m", "synthetic fixture") + + def test_reproducible_archive_verifies_and_keeps_licenses(self): + pin = release.create(self.root, self.output) + second = self.root.parent / "second" + self.assertEqual(pin, release.create(self.root, second)) + self.assertEqual((self.output / release.ARCHIVE).read_bytes(), + (second / release.ARCHIVE).read_bytes()) + receipt = release.verify(self.output, pin) + self.assertIn("LICENSE", [item["path"] for item in receipt["files"]]) + + def test_dirty_and_untracked_source_refused(self): + (self.root / "new.rs").write_text("// Not committed\n") + with self.assertRaisesRegex(ValueError, "clean committed"): + release.create(self.root, self.output) + self.assertFalse(self.output.exists()) + + def test_output_is_never_replaced(self): + pin = release.create(self.root, self.output) + with self.assertRaises(FileExistsError): + release.create(self.root, self.output) + release.verify(self.output, pin) + + def test_tampering_and_wrong_trust_pin_refused(self): + pin = release.create(self.root, self.output) + with self.assertRaisesRegex(ValueError, "pin mismatch"): + release.verify(self.output, "0" * 64) + archive = self.output / release.ARCHIVE + archive.write_bytes(archive.read_bytes() + b"tampered") + with self.assertRaisesRegex(ValueError, "digest mismatch"): + release.verify(self.output, pin) + + def test_link_and_dataset_excluded_before_publication(self): + (self.root / "link.rs").symlink_to("src/lib.rs") + self.commit() + with self.assertRaisesRegex(ValueError, "links or submodules"): + release.create(self.root, self.output) + (self.root / "link.rs").unlink() + (self.root / "private.sqlite").write_bytes(b"not a dataset release") + self.commit() + with self.assertRaisesRegex(ValueError, "allowlist"): + release.create(self.root, self.output) + + def test_unsafe_member_refused_even_with_matching_archive_digest(self): + release.create(self.root, self.output) + receipt_path = self.output / release.RECEIPT + receipt = json.loads(receipt_path.read_text()) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + member = tarfile.TarInfo(receipt["prefix"] + "../../escape.rs") + member.size = 4 + archive.addfile(member, io.BytesIO(b"oops")) + data = buffer.getvalue() + (self.output / release.ARCHIVE).write_bytes(data) + receipt.update(sha256=release.digest(data), bytes=len(data)) + encoded = json.dumps(receipt).encode() + receipt_path.write_bytes(encoded) + with self.assertRaisesRegex(ValueError, "excluded archive path"): + release.verify(self.output, release.digest(encoded)) + + def test_duplicate_receipt_keys_refused(self): + release.create(self.root, self.output) + receipt_path = self.output / release.RECEIPT + encoded = receipt_path.read_bytes().replace(b'"schema":', b'"schema":"duplicate","schema":', 1) + receipt_path.write_bytes(encoded) + with self.assertRaisesRegex(ValueError, "duplicate JSON key"): + release.verify(self.output, release.digest(encoded)) + + def test_special_source_paths_refused(self): + for path in ("/tmp/a.rs", "a/../b.rs", "a//b.rs", "a\\b.rs", "data/a.md", + ".git/config.toml", "secret.key", "x\na.rs"): + with self.subTest(path=path), self.assertRaises(ValueError): + release.safe_path(path) + + +if __name__ == "__main__": + unittest.main()