AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(hub): xerj hub sync and sync --install-timer (plan §8)
xerj-org3bfc6fd3cf09
diff --git a/engine/crates/xerj-hub/src/catalogue.rs b/engine/crates/xerj-hub/src/catalogue.rsindex 26de98dce..59c289cfb 100644--- a/engine/crates/xerj-hub/src/catalogue.rs+++ b/engine/crates/xerj-hub/src/catalogue.rs@@ -364,6 +364,25 @@ pub fn load_catalogue(hub_url: &str, cache_dir: &Path, offline: bool) -> Result<}}+/// Ask the hub for the catalogue NOW, ignoring the 24 h freshness window that+/// [`load_catalogue`] honours. `xerj hub sync` calls this: a sync that trusted a+/// young cache would never see a pin that moved this morning.+///+/// The request is still conditional (`If-None-Match` from `catalogue.etag`, so an+/// unchanged catalogue costs a 304) and the detached signature is still verified+/// BEFORE the fetched bytes replace the cache. Unlike [`load_catalogue`] it does+/// not fall back to the cached copy on failure: the caller asked "is there+/// something newer?", and answering from a stale copy would be a wrong "no". On+/// any network, signature or parse failure the error is returned and the cache+/// is left exactly as it was.+pub fn refresh_catalogue(hub_url: &str, cache_dir: &Path) -> Result<Catalogue> {+ fetch_and_store(+ hub_url,+ &cache_dir.join(CACHE_FILE),+ &cache_dir.join(ETAG_FILE),+ )+}+/// Read and validate the cached catalogue, or `None` if it is missing,/// unreadable, unparsable or a format version this client does not speak.fn read_cached(cache_file: &Path) -> Option<Catalogue> {@@ -887,6 +906,71 @@ mod tests {assert_eq!(on_disk, good_bytes);}+ #[test]+ fn refresh_ignores_a_young_cache_and_fetches_the_newer_catalogue() {+ let _guard = ENV_LOCK.lock().unwrap();+ let fx = tempfile::tempdir().unwrap();+ let inputs = two_inputs(fx.path());++ // A YOUNG cache holding yesterday's catalogue: `load_catalogue` trusts it.+ let (old_seed, _old_public) = keypair();+ let (old_cat, old_sig) =+ build_catalogue(&inputs, "2026-10-13T00:00:00Z", &old_seed).unwrap();+ let old_bytes = canonical_json(&old_cat).unwrap();+ let cache = tempfile::tempdir().unwrap();+ seed_cache(cache.path(), &old_bytes, &old_sig);++ // The hub now serves a newer catalogue, signed by the key the client trusts.+ let (new_seed, new_public) = keypair();+ let (new_cat, new_sig) =+ build_catalogue(&inputs, "2026-10-14T00:00:00Z", &new_seed).unwrap();+ let new_bytes = canonical_json(&new_cat).unwrap();+ let (url, handle) = spawn_hub(new_bytes.clone(), new_sig);++ std::env::set_var("XERJ_HUB_PUBKEY", &new_public);+ let trusted = load_catalogue(&url, cache.path(), false).expect("young cache is served");+ assert_eq!(+ trusted.generated_at, "2026-10-13T00:00:00Z",+ "load_catalogue trusts a young cache without asking the hub"+ );+ let refreshed = refresh_catalogue(&url, cache.path()).expect("forced refresh");+ std::env::remove_var("XERJ_HUB_PUBKEY");+ handle.join().unwrap();++ assert_eq!(+ refreshed.generated_at, "2026-10-14T00:00:00Z",+ "refresh_catalogue asked the hub despite the young cache"+ );+ let on_disk = fs::read(cache.path().join(CACHE_FILE)).unwrap();+ assert_eq!(on_disk, new_bytes, "the verified bytes replaced the cache");+ }++ #[test]+ fn refresh_reports_a_tampered_catalogue_instead_of_serving_the_cache() {+ let _guard = ENV_LOCK.lock().unwrap();+ let fx = tempfile::tempdir().unwrap();+ let (good_bytes, good_sig, public) = fixture_bytes(fx.path());+ let cache = tempfile::tempdir().unwrap();+ seed_cache(cache.path(), &good_bytes, &good_sig);++ let mut tampered = good_bytes.clone();+ tampered[good_bytes.len() / 2] ^= 0x01;+ let (url, handle) = spawn_hub(tampered, good_sig.clone());++ std::env::set_var("XERJ_HUB_PUBKEY", &public);+ let err = refresh_catalogue(&url, cache.path())+ .expect_err("a refresh that cannot be verified is an error, not a stale answer");+ std::env::remove_var("XERJ_HUB_PUBKEY");+ handle.join().unwrap();++ assert!(+ format!("{err:#}").contains("signature"),+ "the error names the signature check: {err:#}"+ );+ let on_disk = fs::read(cache.path().join(CACHE_FILE)).unwrap();+ assert_eq!(on_disk, good_bytes, "the cache is untouched");+ }+#[test]fn load_without_a_key_skips_verification_rather_than_failing() {let _guard = ENV_LOCK.lock().unwrap();diff --git a/engine/crates/xerj-hub/src/cli.rs b/engine/crates/xerj-hub/src/cli.rsindex 6f4d5022f..ba925fc12 100644--- a/engine/crates/xerj-hub/src/cli.rs+++ b/engine/crates/xerj-hub/src/cli.rs@@ -42,6 +42,9 @@ const HUB_USAGE: &str = "usage: xerj hub <command>build-pack <slug> --registry <dir> --out <dir> --url <node> --sign-key <seed-file>Build a signed, pre-indexed pack from a registry manifest (CI).+ sync [--url <node>] | sync --install-timer+ Refresh the catalogue and bring every cached pack up to its pin;+ --install-timer schedules that every 6 hours (see `xerj hub sync -h`).pull <slug> [--quiet] [--url <node>]Download, verify and restore a pack, then `xerj code <slug>` works.@@ -57,6 +60,7 @@ pub fn run_hub_cli(args: &[String]) -> i32 {let rest = &args[1..];match sub.as_str() {"build-pack" => run_build_pack(rest),+ "sync" => crate::sync::run_sync(rest),"pull" => run_pull(rest),"-h" | "--help" | "help" => {println!("{HUB_USAGE}");diff --git a/engine/crates/xerj-hub/src/lib.rs b/engine/crates/xerj-hub/src/lib.rsindex e46f16250..f6175a75c 100644--- a/engine/crates/xerj-hub/src/lib.rs+++ b/engine/crates/xerj-hub/src/lib.rs@@ -21,6 +21,7 @@ pub mod node;pub mod pack;pub mod router;pub mod sketch;+pub mod sync;pub use xerj_common::hub::{render, Answer, AnswerMode, AnsweredFrom, Catalogue, CatalogueEntry, DefRecord, Hit,diff --git a/engine/crates/xerj-hub/src/lockfile.rs b/engine/crates/xerj-hub/src/lockfile.rsindex f8be0cd3f..d07f386ff 100644--- a/engine/crates/xerj-hub/src/lockfile.rs+++ b/engine/crates/xerj-hub/src/lockfile.rs@@ -14,9 +14,9 @@//! - the `xerj hub status` data model + plain-text renderer//! ([`gather_status`] / [`render_status`]).//!-//! `xerj hub sync` is **DEFERRED (step 8b)**: it needs the step-2 pull/restore-//! path another agent is building, so [`sync`] returns an explanatory error-//! rather than faking a no-op. See its doc comment.+//! `xerj hub sync` is not here: it drives the pull path, so it lives next to the+//! other subcommands that do (`crate::sync`). This module only owns the on-disk+//! layout names it shares (`packs/`, `pack.json`, `last_used`, `last_sync`).use std::fs;use std::path::{Path, PathBuf};@@ -238,23 +238,34 @@ fn question_asks_for_current(question: &str) -> bool {// xerj hub status// ----------------------------------------------------------------------------const PACKS_DIR: &str = "packs";-const PACK_MANIFEST_FILE: &str = "pack.json";-const LAST_USED_FILE: &str = "last_used";+pub(crate) const PACKS_DIR: &str = "packs";+pub(crate) const PACK_MANIFEST_FILE: &str = "pack.json";+pub(crate) const LAST_USED_FILE: &str = "last_used";const NODE_PID_FILE: &str = "node.pid";const NODE_PORT_FILE: &str = "node.port";const CONFIG_FILE: &str = "config.toml";-const LAST_SYNC_FILE: &str = "last_sync";+pub(crate) const LAST_SYNC_FILE: &str = "last_sync";const CATALOGUE_FILE: &str = "catalogue.json";/// The default value reported for the `hosted` setting when `config.toml` is/// absent or does not set it.const DEFAULT_HOSTED: &str = "auto";-/// The default hub root, `~/.xerj/hub`. `status` takes the root as a parameter-/// (so tests inject a temp dir); callers that want the real location pass this.+/// The default hub root: `XERJ_HUB_HOME` when set (the override every `xerj hub`+/// subcommand honours), else `~/.xerj/hub`. `status` takes the root as a+/// parameter (so tests inject a temp dir); callers that want the real location+/// pass this.pub fn default_hub_root() -> PathBuf {- let home = std::env::var_os("HOME")+ hub_root_from(&|key| std::env::var_os(key))+}++/// [`default_hub_root`] over an injectable environment, so the rule is testable+/// without mutating the process-global one.+pub(crate) fn hub_root_from(env: &dyn Fn(&str) -> Option<std::ffi::OsString>) -> PathBuf {+ if let Some(h) = env("XERJ_HUB_HOME").filter(|h| !h.is_empty()) {+ return PathBuf::from(h);+ }+ let home = env("HOME").map(PathBuf::from).unwrap_or_else(|| PathBuf::from("."));home.join(".xerj").join("hub")@@ -443,26 +454,6 @@ pub fn render_status(status: &HubStatus) -> String {lines.join("\n")}-// ----------------------------------------------------------------------------// xerj hub sync — DEFERRED (step 8b): needs step-2 pull-// -----------------------------------------------------------------------------/// `xerj hub sync` — **DEFERRED (step 8b): needs step-2 pull**.-///-/// The full sync (refresh the catalogue; for every cached pack whose catalogue-/// pin is newer, pull the new pin and remove the old dir after the new one is-/// restored; verify signatures; prune to the disk budget; write-/// `~/.xerj/hub/last_sync`) all hinges on the step-2 pull/restore path another-/// agent is building. Rather than fake a no-op sync, this returns an error that-/// names the dependency, so a caller cannot mistake "nothing happened" for a-/// successful refresh. Wire the real body here once `pack::pull` lands.-pub fn sync(_hub_root: &Path) -> Result<()> {- anyhow::bail!(- "xerj hub sync is not implemented yet: it depends on the step-2 pull/restore path \- (deferred to step 8b)"- )-}-// ---------------------------------------------------------------------------// small helpers// ---------------------------------------------------------------------------@@ -480,7 +471,7 @@ fn read_trimmed(path: &Path) -> Option<String> {/// Total size in bytes of a directory tree (files only; symlinks skipped so a/// cycle cannot hang the walk).-fn dir_size(dir: &Path) -> u64 {+pub(crate) fn dir_size(dir: &Path) -> u64 {let mut total = 0;let Ok(entries) = fs::read_dir(dir) else {return 0;@@ -539,7 +530,7 @@ fn days_ago(age: Option<i64>) -> String {}/// Human-readable byte size, e.g. `0 B`, `512 B`, `2.0 MB`.-fn human_bytes(bytes: u64) -> String {+pub(crate) fn human_bytes(bytes: u64) -> String {const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"];let mut v = bytes as f64;let mut unit = 0;@@ -868,10 +859,33 @@ mod tests {}#[test]- fn sync_is_deferred() {- let tmp = tempdir().unwrap();- let err = sync(tmp.path()).unwrap_err().to_string();- assert!(err.contains("step-2"), "err was: {err}");+ fn hub_root_honours_the_override_then_home() {+ use std::ffi::OsString;+ let env = |pairs: &'static [(&'static str, &'static str)]| {+ move |k: &str| -> Option<OsString> {+ pairs+ .iter()+ .find(|(name, _)| *name == k)+ .map(|(_, v)| OsString::from(*v))+ }+ };+ // XERJ_HUB_HOME wins — the same override `xerj hub pull` honours, so a+ // sync and the pulls it drives always look at one cache.+ assert_eq!(+ hub_root_from(&env(&[("XERJ_HUB_HOME", "/srv/hub"), ("HOME", "/home/u")])),+ PathBuf::from("/srv/hub")+ );+ // An EMPTY override is not an override.+ assert_eq!(+ hub_root_from(&env(&[("XERJ_HUB_HOME", ""), ("HOME", "/home/u")])),+ PathBuf::from("/home/u/.xerj/hub")+ );+ assert_eq!(+ hub_root_from(&env(&[("HOME", "/home/u")])),+ PathBuf::from("/home/u/.xerj/hub")+ );+ // No HOME at all degrades to a relative path, never a panic.+ assert_eq!(hub_root_from(&env(&[])), PathBuf::from("./.xerj/hub"));}// ---- small helpers ---------------------------------------------------diff --git a/engine/crates/xerj-hub/src/sync.rs b/engine/crates/xerj-hub/src/sync.rsnew file mode 100644index 000000000..fbfa36090--- /dev/null+++ b/engine/crates/xerj-hub/src/sync.rs@@ -0,0 +1,1195 @@+//! Corpus Hub v2 — `xerj hub sync` and `xerj hub sync --install-timer` (plan §8).+//!+//! `sync` keeps the pack cache (`~/.xerj/hub/packs/<slug>/<pin8>/`) in step with+//! the catalogue, in this order:+//!+//! 1. **Refresh the catalogue** ([`crate::catalogue::refresh_catalogue`]): a+//! conditional GET that ignores the 24 h cache window, with the detached+//! signature verified before the cached copy is replaced. If the hub cannot be+//! reached or the signature does not verify, sync stops and changes nothing.+//! 2. **Update every cached pack whose catalogue pin moved.** The new pin is+//! pulled by `xerj hub pull <slug>` — the existing pull path, run as a child+//! process of this same binary — so the archive's sha256 and its ed25519+//! signature are verified before a byte is extracted, exactly as for a first+//! pull. The OLD pin directory is removed only after the pull exited 0 and+//! the new directory is complete and recorded in the state ledger; a failed+//! pull (network, verification, restore) leaves the old pin in place and+//! serving.+//! 3. **Prune to the disk budget**, least recently used first. Packs updated in+//! this run are not candidates (they were just fetched), and an update keeps+//! the old pin's `last_used` recency: refreshing a pack is not using it.+//! 4. **Write `last_sync`** (only when nothing failed), the file `xerj hub status`+//! reads for "last sync".+//!+//! Only extracted pack directories are removed. The restored indices live in the+//! node's own data directory and are not touched here — deleting indices from a+//! node is not sync's business (the same rule the pull-time eviction follows).+//!+//! The pure parts — [`scan_cache`], [`plan_slug`], [`prune_to_budget`], the unit+//! renderers — take plain arguments, and the parts that touch the outside world+//! (pulling, reading the ledger) sit behind [`SyncOps`], so the tests drive the+//! whole flow with throwaway directories and no network, and `tests/sync_e2e.rs`+//! drives it against real nodes and the real pull.++use std::collections::{BTreeMap, BTreeSet};+use std::ffi::OsString;+use std::fs;+use std::path::{Component, Path, PathBuf};+use std::process::{Command, Stdio};+use std::time::{Duration, SystemTime, UNIX_EPOCH};++use anyhow::{anyhow, bail, Context, Result};+use chrono::NaiveDate;+use xerj_common::hub::{+ Catalogue, CatalogueEntry, LicenceClass, PackManifest, AUTO_PULL_MAX_BYTES, DEFAULT_HUB_URL,+ DISK_BUDGET_BYTES,+};+use xerj_common::xccode::state;++use crate::catalogue::refresh_catalogue;+use crate::lockfile::{+ default_hub_root, dir_size, hub_root_from, human_bytes, LAST_SYNC_FILE, LAST_USED_FILE,+ PACKS_DIR, PACK_MANIFEST_FILE,+};++/// Exit codes of `xerj hub sync`. 3 is "a pack failed verification", kept apart+/// from 1 ("the machine or network got in the way") for the same reason+/// `xerj hub pull` keeps them apart.+const EXIT_OK: i32 = 0;+const EXIT_INFRA: i32 = 1;+const EXIT_USAGE: i32 = 2;+const EXIT_VERIFY: i32 = 3;++/// `xerj hub sync -h`. A function, not a constant, so the budget it names is the+/// one the code enforces.+fn sync_usage() -> String {+ format!(+ "usage: xerj hub sync [--url <node>]+ xerj hub sync --install-timer++Refresh the catalogue, then bring every cached pack up to the catalogue's pin:+the new pin is pulled by `xerj hub pull` (sha256 and signature are verified+before anything is extracted), the old pin's directory is removed only after the+new one is restored, and the cache is pruned to the {budget} disk budget, least+recently used first. `~/.xerj/hub/last_sync` is written when nothing failed.++ --url <node> the node the packs are restored into (passed to `hub pull`)+ --install-timer write a systemd user timer (Linux) or a launchd agent+ (macOS) that runs `xerj hub sync` every 6 hours, and print+ its path. Nothing is enabled, loaded or started: the+ comment at the top of the written file says how.++exit codes: 0 synced (or nothing to do); 1 the catalogue could not be refreshed+or a pack could not be updated (its old pin is kept); 2 usage error; 3 a pack+failed verification (its old pin is kept)",+ budget = human_bytes(DISK_BUDGET_BYTES)+ )+}++// ── the cache on disk ────────────────────────────────────────────────────────++/// One extracted pack: `<hub>/packs/<slug>/<pin8>/`.+#[derive(Debug, Clone)]+pub struct CachedPin {+ pub slug: String,+ /// The directory name: the first 8 hex characters of the pack's `pin_sha`.+ pub pin8: String,+ pub dir: PathBuf,+ /// `pin_date` from the pack's own `pack.json`, when it parses.+ pub pin_date: Option<String>,+ /// `pack.json` parses and every file `SHA256SUMS` names is on disk. A+ /// directory that fails this is a half-extracted leftover, not a pack.+ pub complete: bool,+ /// Recency for LRU: the mtime of the pack's `last_used` file, else of the+ /// directory itself (the same rule the pull-time eviction uses).+ pub last_used: SystemTime,+}++/// `^[0-9a-f]{8}$` — what a pin directory is called. Anything else under a slug+/// directory (`<pin8>.tmp`, `<pin8>.lock`, stray files) is not a pack.+fn is_pin8(name: &str) -> bool {+ name.len() == 8 && name.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))+}++/// First 8 hex characters of a commit, lower-cased — the `pin8` in a pack name.+/// `None` for anything that is not at least 8 hex characters.+fn pin8_of(pin_sha: &str) -> Option<String> {+ let s = pin_sha.trim();+ (s.len() >= 8 && s.bytes().all(|b| b.is_ascii_hexdigit())).then(|| s[..8].to_ascii_lowercase())+}++/// Every pin directory under `<hub>/packs`, sorted by slug then pin. Cheap: no+/// directory is walked for its size (only the files each `SHA256SUMS` names are+/// stat'ed). Symlinks are never followed.+pub fn scan_cache(hub_home: &Path) -> Vec<CachedPin> {+ let mut out = Vec::new();+ let Ok(slugs) = fs::read_dir(hub_home.join(PACKS_DIR)) else {+ return out;+ };+ for slug_ent in slugs.flatten() {+ if !slug_ent.file_type().is_ok_and(|t| t.is_dir()) {+ continue;+ }+ let slug = slug_ent.file_name().to_string_lossy().into_owned();+ let Ok(pins) = fs::read_dir(slug_ent.path()) else {+ continue;+ };+ for pin_ent in pins.flatten() {+ let pin8 = pin_ent.file_name().to_string_lossy().into_owned();+ if !pin_ent.file_type().is_ok_and(|t| t.is_dir()) || !is_pin8(&pin8) {+ continue;+ }+ let dir = pin_ent.path();+ let mtime = |p: &Path| fs::metadata(p).and_then(|m| m.modified()).ok();+ let manifest = read_pack_manifest(&dir);+ out.push(CachedPin {+ slug: slug.clone(),+ complete: manifest.is_some() && every_listed_file_exists(&dir),+ pin_date: manifest.map(|m| m.pin_date),+ last_used: mtime(&dir.join(LAST_USED_FILE))+ .or_else(|| mtime(&dir))+ .unwrap_or(UNIX_EPOCH),+ pin8,+ dir,+ });+ }+ }+ out.sort_by(|a, b| a.slug.cmp(&b.slug).then_with(|| a.pin8.cmp(&b.pin8)));+ out+}++fn read_pack_manifest(dir: &Path) -> Option<PackManifest> {+ let raw = fs::read(dir.join(PACK_MANIFEST_FILE)).ok()?;+ serde_json::from_slice(&raw).ok()+}++/// Every path `SHA256SUMS` lists exists under `dir`. The extractor writes+/// `pack.json` before the snapshot files, so a parsing `pack.json` alone does not+/// prove the extraction finished; the listing does. Existence only — re-hashing+/// hundreds of MB every six hours would cost more than it protects. A path that+/// is absolute or climbs out of `dir` makes the listing, and so the pack, invalid.+fn every_listed_file_exists(dir: &Path) -> bool {+ let Ok(sums) = fs::read_to_string(dir.join("SHA256SUMS")) else {+ return false;+ };+ sums.lines().filter(|l| !l.trim().is_empty()).all(|line| {+ let Some((_, rel)) = line.split_once(" ") else {+ return false;+ };+ let rel = Path::new(rel);+ rel.components().all(|c| matches!(c, Component::Normal(_))) && dir.join(rel).is_file()+ })+}++// ── what to do about one slug ────────────────────────────────────────────────++/// What the state ledger (`<code root>/state/<slug>.json`) says about a slug.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum Ledger {+ /// No readable ledger: the pack was never fully restored, or the ledger was+ /// deleted.+ Missing,+ /// The slug is served from a LOCAL index (`xerj corpus index`), not a hub+ /// pack. Sync does not flip that choice back behind the user's back.+ Local,+ /// A hub pack was restored; `pin8` is the first 8 hex of its `pin_sha`.+ Hubpack { pin8: Option<String> },+}++/// What [`run_sync_with`] decided for one slug.+#[derive(Debug, Clone)]+pub enum Verdict {+ /// The catalogue pin is cached, complete and restored. `superseded` are+ /// older pin directories still lying around, now safe to remove.+ Current { superseded: Vec<CachedPin> },+ /// Pull `target`; remove `old` afterwards, and only then.+ Pull {+ target: String,+ old: Vec<CachedPin>,+ why: PullWhy,+ },+ /// Leave the slug alone and say why.+ Keep(KeepWhy),+}++/// Why a pull is needed.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum PullWhy {+ /// The catalogue's pin is not cached.+ NewPin,+ /// The pin directory is there but the ledger does not record it as restored+ /// (an interrupted pull, a deleted ledger).+ NotRestored,+ /// The pin directory is there but incomplete.+ Damaged,+}++/// Why a slug was left alone.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum KeepWhy {+ NotInCatalogue,+ /// The catalogue's `pin_date` is older than the newest cached pin's: a+ /// replayed or rolled-back catalogue must not downgrade a cache.+ CatalogueOlder {+ catalogue: String,+ cached: String,+ },+ /// The catalogue now classes the corpus `excluded`; it is never pulled.+ Excluded,+ /// The new pack is over `AUTO_PULL_MAX_BYTES`; sync never auto-pulls those.+ OverCap {+ bytes: u64,+ },+ /// The catalogue's `pin_sha` is not a hex commit.+ BadPin(String),+ /// Served from a local index ([`Ledger::Local`]).+ LocalIndex,+}++impl KeepWhy {+ fn describe(&self) -> String {+ match self {+ KeepWhy::NotInCatalogue => "not in the catalogue".into(),+ KeepWhy::CatalogueOlder { catalogue, cached } => {+ format!("the catalogue's pin ({catalogue}) is older than the cached one ({cached})")+ }+ KeepWhy::Excluded => {+ "the catalogue now classes it excluded, so no new pin is pulled".into()+ }+ KeepWhy::OverCap { bytes } => format!(+ "its new pack is {} bytes, over the {}-byte auto-pull cap",+ bytes, AUTO_PULL_MAX_BYTES+ ),+ KeepWhy::BadPin(p) => format!("the catalogue's pin '{p}' is not a hex commit"),+ KeepWhy::LocalIndex => {+ "it is served from a local index (`xerj corpus index`), not a hub pack".into()+ }+ }+ }+}++fn parse_day(s: &str) -> Option<NaiveDate> {+ NaiveDate::parse_from_str(s.trim(), "%Y-%m-%d").ok()+}++/// Decide what to do about one slug. Pure: `entry` is the catalogue's entry for+/// the slug (if any), `cached` its pin directories on disk (never empty), and+/// `ledger` what the state ledger records.+pub fn plan_slug(entry: Option<&CatalogueEntry>, cached: &[CachedPin], ledger: &Ledger) -> Verdict {+ let Some(entry) = entry else {+ return Verdict::Keep(KeepWhy::NotInCatalogue);+ };+ let Some(target) = pin8_of(&entry.pin_sha) else {+ return Verdict::Keep(KeepWhy::BadPin(entry.pin_sha.clone()));+ };+ if *ledger == Ledger::Local {+ return Verdict::Keep(KeepWhy::LocalIndex);+ }++ let others: Vec<CachedPin> = cached+ .iter()+ .filter(|c| c.pin8 != target)+ .cloned()+ .collect();+ let have = cached.iter().find(|c| c.pin8 == target);+ let restored = matches!(ledger, Ledger::Hubpack { pin8: Some(p) } if *p == target);++ if let Some(have) = have {+ if have.complete && restored {+ return Verdict::Current { superseded: others };+ }+ }++ // A pull is needed. Refuse what sync must never auto-pull, and refuse to go+ // BACKWARDS in time, before spending a download on it.+ if entry.licence_class == LicenceClass::Excluded {+ return Verdict::Keep(KeepWhy::Excluded);+ }+ if let (Some(cat_day), Some(cached_day)) = (+ parse_day(&entry.pin_date),+ others+ .iter()+ .filter_map(|c| c.pin_date.as_deref().and_then(parse_day))+ .max(),+ ) {+ if cat_day < cached_day {+ return Verdict::Keep(KeepWhy::CatalogueOlder {+ catalogue: entry.pin_date.clone(),+ cached: cached_day.format("%Y-%m-%d").to_string(),+ });+ }+ }+ if entry.pack_bytes > AUTO_PULL_MAX_BYTES {+ return Verdict::Keep(KeepWhy::OverCap {+ bytes: entry.pack_bytes,+ });+ }++ let why = match have {+ None => PullWhy::NewPin,+ Some(h) if !h.complete => PullWhy::Damaged,+ Some(_) => PullWhy::NotRestored,+ };+ Verdict::Pull {+ target,+ old: others,+ why,+ }+}++// ── the outside world ────────────────────────────────────────────────────────++/// What a pull attempt came to.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum PullOutcome {+ Done,+ /// The pull failed with `code` (the exit code of `xerj hub pull`: 3 means+ /// the pack failed verification, anything else is the machine or network).+ Failed {+ code: i32,+ message: String,+ },+}++/// The two things sync needs from the outside world. The production+/// implementation is [`CliOps`]; tests substitute their own.+pub trait SyncOps {+ /// Pull `slug`'s catalogue pin through the verified pull path.+ fn pull(&self, slug: &str) -> PullOutcome;+ /// What the state ledger records for `slug`.+ fn ledger(&self, slug: &str) -> Ledger;+}++/// Runs `xerj hub pull <slug> --quiet [--url <node>]` as a child process of this+/// same binary and reads the ledger the pull writes.+///+/// A child process, not a function call, so sync gets whatever the pull+/// subcommand does — node selection, locks, the size cap, signature+/// verification — without re-implementing any of it, and so a change to the+/// pull path cannot silently diverge from what sync runs.+pub struct CliOps {+ program: PathBuf,+ /// Arguments placed before `hub pull …` (empty for the real binary).+ pre_args: Vec<OsString>,+ url: Option<String>,+ code_root: PathBuf,+}++impl CliOps {+ /// The running binary, with the ledger at `XERJ_CODE_HOME` (else+ /// `~/.xerj-code`) — the same place `hub pull` writes it.+ pub fn current_exe(url: Option<String>) -> Result<CliOps> {+ let program = std::env::current_exe().context("cannot locate the running xerj binary")?;+ Ok(CliOps {+ program,+ pre_args: Vec::new(),+ url,+ code_root: xerj_autoindex::xc::code_root(),+ })+ }++ /// Any program that answers `hub pull` — the unit tests point this at a+ /// shell script.+ pub fn with_program(+ program: impl Into<PathBuf>,+ pre_args: Vec<OsString>,+ url: Option<String>,+ code_root: PathBuf,+ ) -> CliOps {+ CliOps {+ program: program.into(),+ pre_args,+ url,+ code_root,+ }+ }+}++impl SyncOps for CliOps {+ fn pull(&self, slug: &str) -> PullOutcome {+ let mut cmd = Command::new(&self.program);+ cmd.args(&self.pre_args)+ .args(["hub", "pull"])+ .arg(slug)+ .arg("--quiet");+ if let Some(url) = &self.url {+ cmd.arg("--url").arg(url);+ }+ cmd.stdin(Stdio::null());+ match cmd.output() {+ Ok(out) if out.status.success() => PullOutcome::Done,+ Ok(out) => PullOutcome::Failed {+ // No code means the child was killed by a signal.+ code: out.status.code().unwrap_or(EXIT_INFRA),+ message: one_line(&String::from_utf8_lossy(&out.stderr), 400),+ },+ Err(e) => PullOutcome::Failed {+ code: EXIT_INFRA,+ message: format!("cannot run {}: {e}", self.program.display()),+ },+ }+ }++ fn ledger(&self, slug: &str) -> Ledger {+ match state::load_state(&self.code_root, slug) {+ Err(_) => Ledger::Missing,+ Ok(st) if !st.is_hubpack() => Ledger::Local,+ Ok(st) => Ledger::Hubpack {+ pin8: st.pin_sha.as_deref().and_then(pin8_of),+ },+ }+ }+}++/// Collapse whitespace (including newlines) to single spaces and cap the length,+/// so a child's stderr becomes one readable line.+fn one_line(s: &str, max: usize) -> String {+ let joined = s.split_whitespace().collect::<Vec<_>>().join(" ");+ if joined.chars().count() <= max {+ joined+ } else {+ let mut cut: String = joined.chars().take(max).collect();+ cut.push('…');+ cut+ }+}++// ── the sync itself ──────────────────────────────────────────────────────────++/// Where a progress line goes: stdout (`Info`) or stderr (`Warn`).+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Level {+ Info,+ Warn,+}++/// One pack that could not be updated.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct SyncFailure {+ pub slug: String,+ pub code: i32,+ pub message: String,+}++/// A pack directory the budget pass removed.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct Pruned {+ pub slug: String,+ pub pin8: String,+ pub bytes: u64,+}++/// The outcome of one sync, slug by slug.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct SyncReport {+ pub updated: Vec<String>,+ pub current: Vec<String>,+ pub kept: Vec<String>,+ pub failed: Vec<SyncFailure>,+ pub pruned: Vec<Pruned>,+}++impl SyncReport {+ /// 3 when any pack failed verification, else 1 when any pack failed, else 0.+ pub fn exit_code(&self) -> i32 {+ if self.failed.iter().any(|f| f.code == EXIT_VERIFY) {+ EXIT_VERIFY+ } else if !self.failed.is_empty() {+ EXIT_INFRA+ } else {+ EXIT_OK+ }+ }+}++fn pin_list(pins: &[CachedPin]) -> String {+ pins.iter()+ .map(|p| p.pin8.as_str())+ .collect::<Vec<_>>()+ .join(",")+}++/// Sync `hub_home` against `cat`: update, then prune to `budget` bytes. Progress+/// goes to `emit` line by line as it happens, so a long sync is never silent.+pub fn run_sync_with(+ hub_home: &Path,+ cat: &Catalogue,+ ops: &dyn SyncOps,+ budget: u64,+ emit: &mut dyn FnMut(Level, &str),+) -> SyncReport {+ let mut report = SyncReport::default();+ let mut by_slug: BTreeMap<String, Vec<CachedPin>> = BTreeMap::new();+ for pin in scan_cache(hub_home) {+ by_slug.entry(pin.slug.clone()).or_default().push(pin);+ }+ if by_slug.is_empty() {+ emit(Level::Info, "hub sync: no cached packs to update");+ }++ // Slugs this run fetched: not candidates for the budget pass below.+ let mut fresh: BTreeSet<String> = BTreeSet::new();++ for (slug, pins) in &by_slug {+ let entry = cat.corpora.iter().find(|c| &c.slug == slug);+ match plan_slug(entry, pins, &ops.ledger(slug)) {+ Verdict::Current { superseded } => {+ let target = entry.and_then(|e| pin8_of(&e.pin_sha)).unwrap_or_default();+ if superseded.is_empty() {+ emit(Level::Info, &format!("hub sync: {slug} {target}: current"));+ } else if remove_pins(&superseded).len() == superseded.len() {+ emit(+ Level::Info,+ &format!(+ "hub sync: {slug} {target}: current; removed superseded pin {}",+ pin_list(&superseded)+ ),+ );+ } else {+ emit(+ Level::Warn,+ &format!(+ "hub sync: {slug} {target}: current; could not remove superseded pin {}",+ pin_list(&superseded)+ ),+ );+ }+ report.current.push(slug.clone());+ }+ Verdict::Keep(why) => {+ emit(+ Level::Info,+ &format!(+ "hub sync: {slug} {}: kept ({})",+ pin_list(pins),+ why.describe()+ ),+ );+ report.kept.push(slug.clone());+ }+ Verdict::Pull { target, old, why } => {+ let from = if old.is_empty() {+ String::new()+ } else {+ format!("{} -> ", pin_list(&old))+ };+ match pull_and_replace(hub_home, slug, &target, &old, ops) {+ Ok(()) => {+ let what = match why {+ PullWhy::NewPin => "updated",+ PullWhy::NotRestored => {+ "re-pulled (the pin was cached but not restored)"+ }+ PullWhy::Damaged => "re-pulled (the cached pin was incomplete)",+ };+ let tail = if old.is_empty() {+ String::new()+ } else {+ "; old pin removed".to_string()+ };+ emit(+ Level::Info,+ &format!("hub sync: {slug} {from}{target}: {what}{tail}"),+ );+ fresh.insert(slug.clone());+ report.updated.push(slug.clone());+ }+ Err((code, message)) => {+ let kept = if old.is_empty() {+ String::new()+ } else {+ format!("; old pin {} kept", pin_list(&old))+ };+ emit(+ Level::Warn,+ &format!(+ "hub sync: {slug} {from}{target}: FAILED (exit {code}: {message}){kept}"+ ),+ );+ report.failed.push(SyncFailure {+ slug: slug.clone(),+ code,+ message,+ });+ }+ }+ }+ }+ }++ let (pruned, total) = prune_to_budget(hub_home, budget, &fresh);+ for p in &pruned {+ emit(+ Level::Info,+ &format!(+ "hub sync: pruned {} {} ({}, least recently used) to stay under the {} disk budget",+ p.slug,+ p.pin8,+ human_bytes(p.bytes),+ human_bytes(budget)+ ),+ );+ }+ if total > budget {+ emit(+ Level::Warn,+ &format!(+ "hub sync: the cache is {} over the {} disk budget and nothing more can be \+ pruned (what is left was updated in this run, or could not be removed)",+ human_bytes(total - budget),+ human_bytes(budget)+ ),+ );+ }+ report.pruned = pruned;++ emit(+ Level::Info,+ &format!(+ "hub sync: {} updated, {} current, {} kept, {} failed, {} pruned",+ report.updated.len(),+ report.current.len(),+ report.kept.len(),+ report.failed.len(),+ report.pruned.len()+ ),+ );+ report+}++/// Pull `slug`'s catalogue pin and, only if that left the new pin complete and+/// recorded as restored, remove the `old` pin directories. On any failure the old+/// pins are untouched. The error is `(exit code, message)`.+fn pull_and_replace(+ hub_home: &Path,+ slug: &str,+ target: &str,+ old: &[CachedPin],+ ops: &dyn SyncOps,+) -> std::result::Result<(), (i32, String)> {+ if let PullOutcome::Failed { code, message } = ops.pull(slug) {+ return Err((code, message));+ }++ // `hub pull` exited 0. Trust, but check what "restored" is supposed to mean+ // before the only irreversible step: the new pin's directory is complete,+ // and the ledger — written only after the restore — names it.+ let landed = scan_cache(hub_home)+ .into_iter()+ .find(|c| c.slug == slug && c.pin8 == target);+ if !landed.as_ref().is_some_and(|c| c.complete) {+ return Err((+ EXIT_INFRA,+ format!("`hub pull` reported success but {slug}/{target} is missing or incomplete"),+ ));+ }+ match ops.ledger(slug) {+ Ledger::Hubpack { pin8: Some(p) } if p == target => {}+ other => {+ return Err((+ EXIT_INFRA,+ format!(+ "`hub pull` reported success but the ledger does not record {target} as \+ restored ({other:?})"+ ),+ ))+ }+ }++ carry_last_used(old, landed.as_ref().map(|c| c.dir.as_path()));+ let stuck: Vec<String> = old+ .iter()+ .filter_map(|p| {+ remove_pin_dir(&p.dir)+ .err()+ .map(|e| format!("{}: {e}", p.pin8))+ })+ .collect();+ if stuck.is_empty() {+ Ok(())+ } else {+ Err((+ EXIT_INFRA,+ format!(+ "the new pin is in place but the old pin could not be removed ({})",+ stuck.join("; ")+ ),+ ))+ }+}++/// Remove one pin directory. One that is already gone — a pull-time eviction+/// running beside this sync can take another corpus's pack — counts as removed.+fn remove_pin_dir(dir: &Path) -> std::io::Result<()> {+ match fs::remove_dir_all(dir) {+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),+ other => other,+ }+}++/// Remove pin directories, returning the pin8s that went. Failures are skipped+/// (the next sync tries again).+fn remove_pins(pins: &[CachedPin]) -> Vec<String> {+ pins.iter()+ .filter(|p| remove_pin_dir(&p.dir).is_ok())+ .map(|p| p.pin8.clone())+ .collect()+}++/// An update is not a use: give the new pin directory the newest recency of the+/// pins it replaces, so a refresh does not make an idle pack look hot.+fn carry_last_used(old: &[CachedPin], new_dir: Option<&Path>) {+ let (Some(new_dir), Some(when)) = (new_dir, old.iter().map(|p| p.last_used).max()) else {+ return;+ };+ let secs = when+ .duration_since(UNIX_EPOCH)+ .unwrap_or(Duration::ZERO)+ .as_secs();+ let marker = new_dir.join(LAST_USED_FILE);+ // Best effort: LRU order is a courtesy, never a reason to fail an update.+ let _ = fs::write(&marker, secs.to_string());+ let _ = fs::OpenOptions::new()+ .write(true)+ .open(&marker)+ .and_then(|f| f.set_modified(when));+}++/// Remove whole pack directories, least recently used first (incomplete ones+/// before any complete one), until the pin directories under `<hub>/packs` fit in+/// `budget` bytes. Slugs in `protect` are never candidates. Returns what was+/// removed and the size that remains.+///+/// The budget counts the pin directories — what this can remove — and not the+/// `<pin8>.tmp` / `<pin8>.lock` entries of pulls in flight (or abandoned by a+/// crash), which belong to the pull that made them: counting them would evict+/// good packs to make room for files this cannot free.+pub fn prune_to_budget(+ hub_home: &Path,+ budget: u64,+ protect: &BTreeSet<String>,+) -> (Vec<Pruned>, u64) {+ let mut sized: Vec<(CachedPin, u64)> = scan_cache(hub_home)+ .into_iter()+ .map(|c| {+ let bytes = dir_size(&c.dir);+ (c, bytes)+ })+ .collect();+ let mut total: u64 = sized.iter().map(|(_, b)| *b).sum();+ let mut pruned = Vec::new();+ if total <= budget {+ return (pruned, total);+ }+ sized.retain(|(c, _)| !protect.contains(&c.slug));+ sized.sort_by(|(a, _), (b, _)| {+ a.complete+ .cmp(&b.complete)+ .then_with(|| a.last_used.cmp(&b.last_used))+ .then_with(|| a.slug.cmp(&b.slug))+ .then_with(|| a.pin8.cmp(&b.pin8))+ });+ for (c, bytes) in sized {+ if total <= budget {+ break;+ }+ if fs::remove_dir_all(&c.dir).is_ok() {+ total = total.saturating_sub(bytes);+ // Drop the slug directory once its last pin is gone.+ if let Some(parent) = c.dir.parent() {+ let _ = fs::remove_dir(parent);+ }+ pruned.push(Pruned {+ slug: c.slug,+ pin8: c.pin8,+ bytes,+ });+ }+ }+ (pruned, total)+}++/// Stamp `<hub>/last_sync` with the current UTC time (RFC 3339); `xerj hub+/// status` reads its age.+pub fn write_last_sync(hub_home: &Path) -> Result<()> {+ fs::create_dir_all(hub_home).with_context(|| format!("create {}", hub_home.display()))?;+ let stamp = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true);+ let path = hub_home.join(LAST_SYNC_FILE);+ fs::write(&path, format!("{stamp}\n")).with_context(|| format!("write {}", path.display()))+}++// ── the timer ────────────────────────────────────────────────────────────────++/// The systemd unit pair's basename (`<name>.service` + `<name>.timer`).+pub const SYSTEMD_UNIT_BASENAME: &str = "xerj-hub-sync";+/// The launchd agent's label, and the plist's basename.+pub const LAUNCHD_LABEL: &str = "org.xerj.hub-sync";+/// `StartInterval` for launchd: six hours, in seconds.+const LAUNCHD_INTERVAL_SECS: u32 = 6 * 60 * 60;++/// Which scheduler a timer is written for.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum TimerKind {+ /// A systemd user timer (`~/.config/systemd/user/`).+ SystemdUser,+ /// A launchd agent (`~/Library/LaunchAgents/`).+ Launchd,+}++impl TimerKind {+ /// The scheduler of the OS this binary was built for; `None` where neither+ /// exists (Windows).+ pub fn for_this_os() -> Option<TimerKind> {+ if cfg!(target_os = "linux") {+ Some(TimerKind::SystemdUser)+ } else if cfg!(target_os = "macos") {+ Some(TimerKind::Launchd)+ } else {+ None+ }+ }+}++/// Where the timer files go. Injectable so the tests write into a temp dir.+#[derive(Debug, Clone)]+pub struct TimerDirs {+ /// `$XDG_CONFIG_HOME/systemd/user`, else `~/.config/systemd/user`.+ pub systemd_user: PathBuf,+ /// `~/Library/LaunchAgents`.+ pub launch_agents: PathBuf,+ /// The hub home: launchd's log lives here (`sync.log`).+ pub hub_home: PathBuf,+}++impl TimerDirs {+ pub fn from_env(env: &dyn Fn(&str) -> Option<OsString>) -> Result<TimerDirs> {+ let home = env("HOME")+ .filter(|h| !h.is_empty())+ .map(PathBuf::from)+ .ok_or_else(|| anyhow!("HOME is not set; cannot place a user timer"))?;+ // XDG says a relative XDG_CONFIG_HOME is invalid and must be ignored.+ let config = env("XDG_CONFIG_HOME")+ .map(PathBuf::from)+ .filter(|p| p.is_absolute())+ .unwrap_or_else(|| home.join(".config"));+ Ok(TimerDirs {+ systemd_user: config.join("systemd").join("user"),+ launch_agents: home.join("Library").join("LaunchAgents"),+ hub_home: hub_root_from(env),+ })+ }+}++/// A path as one `ExecStart=` token: verbatim when it is plainly safe, else+/// double-quoted with systemd's escapes (`\\`, `\"`, `%%` for the specifier+/// character, `$$` for the variable one).+fn systemd_exec_token(path: &Path) -> Result<String> {+ let s = path+ .to_str()+ .ok_or_else(|| anyhow!("{} is not valid UTF-8", path.display()))?;+ if s.chars().any(|c| c.is_control()) {+ bail!("{s:?} contains a control character; refusing to write it into a unit");+ }+ let plain = |c: char| c.is_ascii_alphanumeric() || "/._-+:@,=".contains(c);+ if !s.is_empty() && s.chars().all(plain) {+ return Ok(s.to_string());+ }+ let mut out = String::from("\"");+ for c in s.chars() {+ match c {+ '\\' => out.push_str("\\\\"),+ '"' => out.push_str("\\\""),+ '%' => out.push_str("%%"),+ '$' => out.push_str("$$"),+ c => out.push(c),+ }+ }+ out.push('"');+ Ok(out)+}++/// `<name>.service`: one `xerj hub sync` run. `Type=oneshot` so systemd never+/// overlaps two runs, a timeout so a wedged run cannot block every later one.+pub fn render_systemd_service(exe: &Path) -> Result<String> {+ Ok(format!(+ "# Written by `xerj hub sync --install-timer`. Enable the schedule with:\n\+ # systemctl --user daemon-reload && systemctl --user enable --now {SYSTEMD_UNIT_BASENAME}.timer\n\+ [Unit]\n\+ Description=Refresh the cached XERJ hub corpus packs\n\+ \n\+ [Service]\n\+ Type=oneshot\n\+ ExecStart={} hub sync\n\+ TimeoutStartSec=2h\n\+ Nice=10\n",+ systemd_exec_token(exe)?+ ))+}++/// `<name>.timer`: every six hours on the wall clock. `Persistent=true` runs a+/// missed slot at the next boot or wake (a laptop that slept through 06:00 still+/// syncs), and the randomised delay keeps every client from asking the hub in+/// the same second.+pub fn render_systemd_timer() -> String {+ format!(+ "# Written by `xerj hub sync --install-timer`; runs {SYSTEMD_UNIT_BASENAME}.service.\n\+ [Unit]\n\+ Description=Run `xerj hub sync` every 6 hours\n\+ \n\+ [Timer]\n\+ OnCalendar=*-*-* 00/6:00:00\n\+ RandomizedDelaySec=15min\n\+ Persistent=true\n\+ \n\+ [Install]\n\+ WantedBy=timers.target\n"+ )+}++fn xml_escape(s: &str) -> String {+ let mut out = String::with_capacity(s.len());+ for c in s.chars() {+ match c {+ '&' => out.push_str("&"),+ '<' => out.push_str("<"),+ '>' => out.push_str(">"),+ '"' => out.push_str("""),+ '\'' => out.push_str("'"),+ c => out.push(c),+ }+ }+ out+}++/// The launchd agent: `xerj hub sync` every 6 hours (missed intervals during+/// sleep coalesce into one run at wake), output appended to `log`.+pub fn render_launchd_plist(exe: &Path, log: &Path) -> Result<String> {+ let exe = exe+ .to_str()+ .ok_or_else(|| anyhow!("{} is not valid UTF-8", exe.display()))?;+ let log = log+ .to_str()+ .ok_or_else(|| anyhow!("{} is not valid UTF-8", log.display()))?;+ let (exe, log) = (xml_escape(exe), xml_escape(log));+ Ok(format!(+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\+ <!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n\+ <!-- Written by `xerj hub sync --install-timer`. Load it with: launchctl bootstrap gui/$(id -u) {LAUNCHD_LABEL}.plist -->\n\+ <plist version=\"1.0\">\n\+ <dict>\n\+ \t<key>Label</key>\n\+ \t<string>{LAUNCHD_LABEL}</string>\n\+ \t<key>ProgramArguments</key>\n\+ \t<array>\n\+ \t\t<string>{exe}</string>\n\+ \t\t<string>hub</string>\n\+ \t\t<string>sync</string>\n\+ \t</array>\n\+ \t<key>StartInterval</key>\n\+ \t<integer>{LAUNCHD_INTERVAL_SECS}</integer>\n\+ \t<key>RunAtLoad</key>\n\+ \t<false/>\n\+ \t<key>ProcessType</key>\n\+ \t<string>Background</string>\n\+ \t<key>LowPriorityIO</key>\n\+ \t<true/>\n\+ \t<key>StandardOutPath</key>\n\+ \t<string>{log}</string>\n\+ \t<key>StandardErrorPath</key>\n\+ \t<string>{log}</string>\n\+ </dict>\n\+ </plist>\n"+ ))+}++/// Write `contents` to `path` through a sibling file and a rename, so a reader+/// (systemd, launchd) never sees a half-written unit.+fn write_file_atomic(path: &Path, contents: &str) -> Result<()> {+ let parent = path+ .parent()+ .ok_or_else(|| anyhow!("{} has no parent directory", path.display()))?;+ fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;+ let mut name = path.file_name().unwrap_or_default().to_os_string();+ name.push(".new");+ let tmp = path.with_file_name(name);+ fs::write(&tmp, contents).with_context(|| format!("write {}", tmp.display()))?;+ fs::rename(&tmp, path).with_context(|| format!("rename into {}", path.display()))+}++/// Write the timer files for `kind` and return the path to the one the user+/// enables: the `.timer` (systemd; its `.service` is written beside it) or the+/// `.plist` (launchd). Enables, loads and starts nothing. Re-running overwrites,+/// so it is also how a moved binary's path is refreshed.+pub fn install_timer(kind: TimerKind, exe: &Path, dirs: &TimerDirs) -> Result<PathBuf> {+ match kind {+ TimerKind::SystemdUser => {+ let service = dirs+ .systemd_user+ .join(format!("{SYSTEMD_UNIT_BASENAME}.service"));+ let timer = dirs+ .systemd_user+ .join(format!("{SYSTEMD_UNIT_BASENAME}.timer"));+ // Render both before writing either: a bad exe path leaves nothing behind.+ let service_body = render_systemd_service(exe)?;+ let timer_body = render_systemd_timer();+ write_file_atomic(&service, &service_body)?;+ write_file_atomic(&timer, &timer_body)?;+ Ok(timer)+ }+ TimerKind::Launchd => {+ let plist = dirs.launch_agents.join(format!("{LAUNCHD_LABEL}.plist"));+ let log = dirs.hub_home.join("sync.log");+ let body = render_launchd_plist(exe, &log)?;+ // launchd opens the log path at job start and will not create its directory.+ fs::create_dir_all(&dirs.hub_home)+ .with_context(|| format!("create {}", dirs.hub_home.display()))?;+ write_file_atomic(&plist, &body)?;+ Ok(plist)+ }+ }+}++// ── the subcommand ───────────────────────────────────────────────────────────++/// `XERJ_HUB_URL`, else the public hub.+fn hub_url() -> String {+ std::env::var("XERJ_HUB_URL")+ .ok()+ .filter(|u| !u.is_empty())+ .unwrap_or_else(|| DEFAULT_HUB_URL.to_string())+}++/// `xerj hub sync [--url <node>]` and `xerj hub sync --install-timer`.+pub fn run_sync(args: &[String]) -> i32 {+ run_sync_using(args, &|url| {+ Ok(Box::new(CliOps::current_exe(url)?) as Box<dyn SyncOps>)+ })+}++/// [`run_sync`] with the pull/ledger side injectable, so the whole command —+/// argument handling, the catalogue refresh and its signature check, the update,+/// the exit code, `last_sync` — can be driven end to end by a test that has no+/// `xerj` binary to spawn. `make_ops` receives `--url` and is called only after+/// the catalogue has been refreshed, so a usage error, `--install-timer` or a+/// failed refresh never builds one.+pub fn run_sync_using(+ args: &[String],+ make_ops: &dyn Fn(Option<String>) -> Result<Box<dyn SyncOps>>,+) -> i32 {+ let usage = sync_usage();+ let mut url: Option<String> = None;+ let mut install = false;+ let mut i = 0;+ while i < args.len() {+ match args[i].as_str() {+ "-h" | "--help" => {+ println!("{usage}");+ return EXIT_OK;+ }+ "--install-timer" => install = true,+ "--url" => match args.get(i + 1) {+ Some(v) => {+ url = Some(v.clone());+ i += 1;+ }+ None => {+ eprintln!("xerj hub sync: --url needs a value\n\n{usage}");+ return EXIT_USAGE;+ }+ },+ a if a.starts_with('-') => {+ eprintln!("xerj hub sync: unknown flag '{a}'\n\n{usage}");+ return EXIT_USAGE;+ }+ a => {+ eprintln!("xerj hub sync: unexpected argument '{a}'\n\n{usage}");+ return EXIT_USAGE;+ }+ }+ i += 1;+ }+ if install {+ if url.is_some() {+ eprintln!("xerj hub sync: --install-timer takes no other flag\n\n{usage}");+ return EXIT_USAGE;+ }+ return run_install_timer();+ }++ let hub_home = default_hub_root();+ let cat = match refresh_catalogue(&hub_url(), &hub_home) {+ Ok(c) => c,+ Err(e) => {+ eprintln!("xerj hub sync: cannot refresh the catalogue; nothing was changed: {e:#}");+ return EXIT_INFRA;+ }+ };+ println!(+ "hub sync: catalogue refreshed (generated {}, {} corpora)",+ cat.generated_at,+ cat.corpora.len()+ );+ let ops = match make_ops(url) {+ Ok(o) => o,+ Err(e) => {+ eprintln!("xerj hub sync: {e:#}");+ return EXIT_INFRA;+ }+ };+ let report = run_sync_with(+ &hub_home,+ &cat,+ ops.as_ref(),+ DISK_BUDGET_BYTES,+ &mut |level, line| match level {+ Level::Info => println!("{line}"),+ Level::Warn => eprintln!("{line}"),+ },+ );+ let code = report.exit_code();+ if code == EXIT_OK {+ if let Err(e) = write_last_sync(&hub_home) {+ eprintln!("xerj hub sync: synced, but could not write last_sync: {e:#}");+ return EXIT_INFRA;+ }+ }+ code+}++/// `--install-timer`: write the units for this OS, print the path, nothing else.+fn run_install_timer() -> i32 {+ let Some(kind) = TimerKind::for_this_os() else {+ eprintln!(+ "xerj hub sync --install-timer: only systemd (Linux) and launchd (macOS) are \+ supported; schedule `xerj hub sync` with your platform's scheduler instead"+ );+ return EXIT_INFRA;+ };+ let exe = match std::env::current_exe() {+ Ok(p) => p,+ Err(e) => {+ eprintln!("xerj hub sync --install-timer: cannot locate the running binary: {e}");+ return EXIT_INFRA;+ }+ };+ let dirs = match TimerDirs::from_env(&|k| std::env::var_os(k)) {+ Ok(d) => d,+ Err(e) => {+ eprintln!("xerj hub sync --install-timer: {e:#}");+ return EXIT_INFRA;+ }+ };+ match install_timer(kind, &exe, &dirs) {+ Ok(path) => {+ println!("{}", path.display());+ EXIT_OK+ }+ Err(e) => {+ eprintln!("xerj hub sync --install-timer: {e:#}");+ EXIT_INFRA+ }+ }+}++#[cfg(test)]+mod tests;diff --git a/engine/crates/xerj-hub/src/sync/tests.rs b/engine/crates/xerj-hub/src/sync/tests.rsnew file mode 100644index 000000000..6f85ad953--- /dev/null+++ b/engine/crates/xerj-hub/src/sync/tests.rs@@ -0,0 +1,1367 @@+//! Unit tests for `xerj hub sync` (plan §8): every one runs on throwaway+//! directories with a scripted [`SyncOps`], so no node, network or port is+//! involved. `tests/sync_e2e.rs` covers the same flow against real nodes and the+//! real `hub pull`.++use std::cell::RefCell;+use std::collections::BTreeMap;+use std::path::Path;+use std::time::{Duration, SystemTime};++use tempfile::tempdir;+use xerj_common::hub::{LicenceClass, PackSource, Sketch, PACK_FORMAT_VERSION};++use super::*;++// ── fixtures ─────────────────────────────────────────────────────────────────++fn full_sha(pin8: &str) -> String {+ format!("{pin8}{}", "0".repeat(32))+}++/// A complete extracted pack: `pack.json`, a payload file, a snapshot member and+/// the `SHA256SUMS` that lists them.+fn make_pack(hub: &Path, slug: &str, pin8: &str, pin_date: &str) -> PathBuf {+ let dir = hub.join("packs").join(slug).join(pin8);+ fs::create_dir_all(dir.join("snapshot")).unwrap();+ let manifest = PackManifest {+ format_version: PACK_FORMAT_VERSION,+ slug: slug.to_string(),+ pin_sha: full_sha(pin8),+ pin_date: pin_date.to_string(),+ built_at: "2026-10-04T00:00:00Z".to_string(),+ builder_version: "test".to_string(),+ index_prefix: format!("xc-{slug}-b1"),+ indices: vec![],+ snapshot_name: format!("{slug}-{pin8}"),+ licence_spdx: "MIT".to_string(),+ licence_class: LicenceClass::AdaptWithAttribution,+ sources: vec![PackSource {+ repo: slug.to_string(),+ url: format!("https://example.test/{slug}"),+ sha: full_sha(pin8),+ licence: "MIT".to_string(),+ }],+ defs_count: 0,+ files_count: 0,+ };+ fs::write(+ dir.join("pack.json"),+ serde_json::to_vec(&manifest).unwrap(),+ )+ .unwrap();+ fs::write(dir.join("defs.jsonl"), b"{}\n").unwrap();+ fs::write(dir.join("snapshot").join("manifest.json"), b"{}").unwrap();+ fs::write(+ dir.join("SHA256SUMS"),+ "00 defs.jsonl\n00 pack.json\n00 snapshot/manifest.json\n",+ )+ .unwrap();+ dir+}++fn entry(slug: &str, pin8: &str, pin_date: &str) -> CatalogueEntry {+ CatalogueEntry {+ slug: slug.to_string(),+ domain: "NET".to_string(),+ topics: vec![],+ ask_for: "x".to_string(),+ pin_sha: full_sha(pin8),+ pin_date: pin_date.to_string(),+ refresh_cadence_days: 7,+ licence_spdx: "MIT".to_string(),+ licence_class: LicenceClass::AdaptWithAttribution,+ pack_bytes: 1_000,+ pack_url: format!("https://example.test/{slug}-{pin8}.hubpack.tar.zst"),+ pack_sha256: "ab".repeat(32),+ sig_url: format!("https://example.test/{slug}-{pin8}.hubpack.tar.zst.sig"),+ dependency_names: vec![],+ quality_grade: None,+ term_sketch: Sketch {+ m_bits: 64,+ k_hashes: 7,+ bits_base64: "AAAAAAAAAAA=".to_string(),+ },+ }+}++fn catalogue(entries: Vec<CatalogueEntry>) -> Catalogue {+ Catalogue {+ format_version: 1,+ generated_at: "2026-10-14T00:00:00Z".to_string(),+ corpora: entries,+ }+}++/// What a scripted pull of one slug does.+#[derive(Clone)]+enum Script {+ /// Land `pin8` the way a real pull does: extract the pin directory, then+ /// stamp the ledger.+ Land { pin8: String, pin_date: String },+ /// Exit non-zero.+ Fail { code: i32, message: String },+ /// Exit 0 without having extracted anything.+ SucceedWithoutADirectory,+ /// Extract the pin directory but exit 0 without ever stamping the ledger.+ SucceedWithoutALedger { pin8: String, pin_date: String },+}++struct Fake<'a> {+ hub: &'a Path,+ ledger: RefCell<BTreeMap<String, Ledger>>,+ script: BTreeMap<String, Script>,+ pulls: RefCell<Vec<String>>,+ /// The pin directories that existed under `packs/<slug>/` at the moment the+ /// pull ran — proof the old pin outlives the pull.+ present_at_pull: RefCell<BTreeMap<String, Vec<String>>>,+}++impl<'a> Fake<'a> {+ fn new(hub: &'a Path) -> Self {+ Fake {+ hub,+ ledger: RefCell::new(BTreeMap::new()),+ script: BTreeMap::new(),+ pulls: RefCell::new(Vec::new()),+ present_at_pull: RefCell::new(BTreeMap::new()),+ }+ }+ fn with_ledger(self, slug: &str, pin8: &str) -> Self {+ self.ledger.borrow_mut().insert(+ slug.to_string(),+ Ledger::Hubpack {+ pin8: Some(pin8.to_string()),+ },+ );+ self+ }+ fn with_script(mut self, slug: &str, script: Script) -> Self {+ self.script.insert(slug.to_string(), script);+ self+ }+ fn pulled(&self) -> Vec<String> {+ self.pulls.borrow().clone()+ }+}++impl SyncOps for Fake<'_> {+ fn pull(&self, slug: &str) -> PullOutcome {+ self.pulls.borrow_mut().push(slug.to_string());+ let listing = scan_cache(self.hub)+ .into_iter()+ .filter(|c| c.slug == slug)+ .map(|c| c.pin8)+ .collect();+ self.present_at_pull+ .borrow_mut()+ .insert(slug.to_string(), listing);+ match self.script.get(slug).cloned() {+ Some(Script::Land { pin8, pin_date }) => {+ make_pack(self.hub, slug, &pin8, &pin_date);+ self.ledger.borrow_mut().insert(+ slug.to_string(),+ Ledger::Hubpack {+ pin8: Some(pin8.clone()),+ },+ );+ PullOutcome::Done+ }+ Some(Script::Fail { code, message }) => PullOutcome::Failed { code, message },+ Some(Script::SucceedWithoutADirectory) => PullOutcome::Done,+ Some(Script::SucceedWithoutALedger { pin8, pin_date }) => {+ make_pack(self.hub, slug, &pin8, &pin_date);+ PullOutcome::Done+ }+ None => PullOutcome::Failed {+ code: 1,+ message: format!("the test scripted no pull of {slug}"),+ },+ }+ }++ fn ledger(&self, slug: &str) -> Ledger {+ self.ledger+ .borrow()+ .get(slug)+ .cloned()+ .unwrap_or(Ledger::Missing)+ }+}++/// Run a sync, returning the report and every line it printed.+fn sync(+ hub: &Path,+ cat: &Catalogue,+ ops: &dyn SyncOps,+ budget: u64,+) -> (SyncReport, Vec<(Level, String)>) {+ let mut lines = Vec::new();+ let report = run_sync_with(hub, cat, ops, budget, &mut |level, line| {+ lines.push((level, line.to_string()))+ });+ (report, lines)+}++fn has_line(lines: &[(Level, String)], needle: &str) -> bool {+ lines.iter().any(|(_, l)| l.contains(needle))+}++fn set_mtime(path: &Path, when: SystemTime) {+ fs::OpenOptions::new()+ .write(true)+ .open(path)+ .unwrap()+ .set_modified(when)+ .unwrap();+}++fn days_ago(n: u64) -> SystemTime {+ SystemTime::now() - Duration::from_secs(n * 86_400)+}++// ── §8: sync with a newer pin replaces the pack and the old dir is gone ──────++#[test]+fn a_newer_pin_replaces_the_pack_and_the_old_dir_is_gone() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(+ "quinn",+ Script::Land {+ pin8: "bbbb2222".into(),+ pin_date: "2026-10-09".into(),+ },+ );++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(!old.exists(), "the old pin's directory is gone");+ let new = hub.join("packs/quinn/bbbb2222");+ assert!(new.join("pack.json").is_file(), "the new pin is in place");+ assert_eq!(ops.pulled(), vec!["quinn"], "exactly one pull");+ assert_eq!(+ ops.present_at_pull.borrow()["quinn"],+ vec!["aaaa1111"],+ "the old pin was still on disk while the new one was pulled — it is removed \+ only AFTER the new pin restores"+ );+ assert_eq!(report.updated, vec!["quinn"]);+ assert!(report.failed.is_empty());+ assert_eq!(report.exit_code(), 0);+ assert!(+ has_line(+ &lines,+ "hub sync: quinn aaaa1111 -> bbbb2222: updated; old pin removed"+ ),+ "lines were: {lines:?}"+ );+ // `status` lists exactly what is on disk: only the new pin.+ let on_disk: Vec<_> = crate::lockfile::gather_status(hub)+ .packs+ .iter()+ .map(|p| (p.slug.clone(), p.pin8.clone()))+ .collect();+ assert_eq!(on_disk, vec![("quinn".to_string(), "bbbb2222".to_string())]);+}++#[test]+fn every_old_pin_of_the_slug_goes_once_the_new_one_has_restored() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let a = make_pack(hub, "quinn", "aaaa1111", "2026-09-01");+ let b = make_pack(hub, "quinn", "bbbb2222", "2026-09-20");+ let cat = catalogue(vec![entry("quinn", "cccc3333", "2026-10-09")]);+ let ops = Fake::new(hub).with_ledger("quinn", "bbbb2222").with_script(+ "quinn",+ Script::Land {+ pin8: "cccc3333".into(),+ pin_date: "2026-10-09".into(),+ },+ );++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(!a.exists() && !b.exists());+ assert!(hub.join("packs/quinn/cccc3333").is_dir());+ assert_eq!(report.updated, vec!["quinn"]);+ assert!(+ has_line(&lines, "quinn aaaa1111,bbbb2222 -> cccc3333: updated"),+ "lines were: {lines:?}"+ );+}++// ── a pull that does not complete never costs the old pin ────────────────────++#[test]+fn a_failed_pull_keeps_the_old_pin_and_exits_1() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(+ "quinn",+ Script::Fail {+ code: 1,+ message: "connect refused".into(),+ },+ );++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(old.join("pack.json").is_file(), "the old pin is untouched");+ assert!(!hub.join("packs/quinn/bbbb2222").exists());+ assert_eq!(report.failed.len(), 1);+ assert_eq!(report.failed[0].slug, "quinn");+ assert_eq!(report.exit_code(), 1);+ assert!(report.updated.is_empty());+ let failed: Vec<_> = lines.iter().filter(|(_, l)| l.contains("FAILED")).collect();+ assert_eq!(failed.len(), 1);+ assert_eq!(failed[0].0, Level::Warn, "failures go to stderr");+ assert!(failed[0].1.contains("connect refused"));+ assert!(failed[0].1.contains("old pin aaaa1111 kept"));+}++#[test]+fn a_pack_that_fails_verification_exits_3_and_keeps_the_old_pin() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let lz4 = make_pack(hub, "lz4", "dddd4444", "2026-10-02");+ let cat = catalogue(vec![+ entry("quinn", "bbbb2222", "2026-10-09"),+ entry("lz4", "eeee5555", "2026-10-09"),+ ]);+ let ops = Fake::new(hub)+ .with_ledger("quinn", "aaaa1111")+ .with_ledger("lz4", "dddd4444")+ .with_script(+ "quinn",+ Script::Fail {+ code: 3,+ message: "pack verification failed: signature does not verify".into(),+ },+ )+ .with_script(+ "lz4",+ Script::Land {+ pin8: "eeee5555".into(),+ pin_date: "2026-10-09".into(),+ },+ );++ let (report, _lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(+ old.is_dir(),+ "the unverifiable pack did not displace the old pin"+ );+ assert!(!lz4.exists() && hub.join("packs/lz4/eeee5555").is_dir());+ assert_eq!(+ report.updated,+ vec!["lz4"],+ "one slug failing does not stop the others"+ );+ assert_eq!(+ report.exit_code(),+ 3,+ "verification failure outranks a plain failure"+ );+}++#[test]+fn a_pull_that_claims_success_without_a_new_directory_keeps_the_old_pin() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);+ let ops = Fake::new(hub)+ .with_ledger("quinn", "aaaa1111")+ .with_script("quinn", Script::SucceedWithoutADirectory);++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(old.is_dir());+ assert_eq!(report.failed.len(), 1);+ assert!(+ has_line(&lines, "missing or incomplete"),+ "lines: {lines:?}"+ );+}++#[test]+fn a_pull_that_never_stamped_the_ledger_keeps_the_old_pin() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(+ "quinn",+ Script::SucceedWithoutALedger {+ pin8: "bbbb2222".into(),+ pin_date: "2026-10-09".into(),+ },+ );++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(+ old.is_dir(),+ "the new pin is extracted but not recorded as restored, so the old pin stays"+ );+ assert_eq!(report.failed.len(), 1);+ assert!(+ has_line(&lines, "does not record bbbb2222 as restored"),+ "lines: {lines:?}"+ );+}++// ── nothing to do ────────────────────────────────────────────────────────────++#[test]+fn a_pack_already_at_the_catalogue_pin_is_not_pulled() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let dir = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let cat = catalogue(vec![entry("quinn", "aaaa1111", "2026-10-02")]);+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111");++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(ops.pulled().is_empty(), "no pull, so no download");+ assert!(dir.is_dir());+ assert_eq!(report.current, vec!["quinn"]);+ assert_eq!(report.exit_code(), 0);+ assert!(has_line(&lines, "hub sync: quinn aaaa1111: current"));+}++#[test]+fn an_empty_cache_is_a_clean_no_op() {+ let tmp = tempdir().unwrap();+ let cat = catalogue(vec![entry("quinn", "aaaa1111", "2026-10-02")]);+ let ops = Fake::new(tmp.path());++ let (report, lines) = sync(tmp.path(), &cat, &ops, DISK_BUDGET_BYTES);++ assert_eq!(report, SyncReport::default());+ assert!(has_line(&lines, "no cached packs"));+ assert!(ops.pulled().is_empty());+}++#[test]+fn superseded_pins_beside_a_restored_current_pin_are_removed() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let stale = make_pack(hub, "quinn", "aaaa1111", "2026-09-01");+ let current = make_pack(hub, "quinn", "bbbb2222", "2026-10-02");+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-02")]);+ let ops = Fake::new(hub).with_ledger("quinn", "bbbb2222");++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(ops.pulled().is_empty());+ assert!(!stale.exists(), "a leftover older pin is housekeeping");+ assert!(current.is_dir());+ assert_eq!(report.current, vec!["quinn"]);+ assert!(+ has_line(&lines, "removed superseded pin aaaa1111"),+ "{lines:?}"+ );+}++// ── "restored", not just "extracted" ─────────────────────────────────────────++#[test]+fn a_cached_pin_the_ledger_does_not_record_is_pulled_again_and_kept() {+ // The catalogue pin's directory exists (an interrupted pull extracted it)+ // but the ledger still names the old pin: it was never restored.+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let target = make_pack(hub, "quinn", "bbbb2222", "2026-10-09");+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(+ "quinn",+ Script::Land {+ pin8: "bbbb2222".into(),+ pin_date: "2026-10-09".into(),+ },+ );++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert_eq!(+ ops.pulled(),+ vec!["quinn"],+ "present is not the same as restored"+ );+ assert!(target.is_dir(), "the pin being repaired is never removed");+ assert!(!old.exists());+ assert_eq!(report.updated, vec!["quinn"]);+ assert!(has_line(+ &lines,+ "re-pulled (the pin was cached but not restored)"+ ));+}++#[test]+fn a_half_extracted_current_pin_is_pulled_again() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let dir = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ // The extractor writes pack.json before the snapshot: lose a listed file.+ fs::remove_file(dir.join("snapshot").join("manifest.json")).unwrap();+ let cat = catalogue(vec![entry("quinn", "aaaa1111", "2026-10-02")]);+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(+ "quinn",+ Script::Land {+ pin8: "aaaa1111".into(),+ pin_date: "2026-10-02".into(),+ },+ );++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert_eq!(ops.pulled(), vec!["quinn"]);+ assert!(+ dir.join("snapshot").join("manifest.json").is_file(),+ "the pull rewrote the directory in place"+ );+ assert_eq!(report.updated, vec!["quinn"]);+ assert!(has_line(+ &lines,+ "re-pulled (the cached pin was incomplete)"+ ));+}++#[test]+fn scan_cache_sees_packs_and_nothing_else() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ // In-flight pull artefacts and strays sit in the same directory.+ let slug_dir = hub.join("packs/quinn");+ fs::create_dir(slug_dir.join("bbbb2222.tmp")).unwrap();+ fs::create_dir(slug_dir.join("bbbb2222.lock")).unwrap();+ fs::create_dir(slug_dir.join("NOTHEX00")).unwrap();+ fs::write(slug_dir.join("cccc3333"), b"a file, not a pack").unwrap();+ fs::write(hub.join("packs/stray-file"), b"x").unwrap();+ // A pin directory with no pack.json is listed, but not complete.+ fs::create_dir_all(hub.join("packs/lz4/dddd4444")).unwrap();++ let cached = scan_cache(hub);++ let seen: Vec<_> = cached+ .iter()+ .map(|c| (c.slug.as_str(), c.pin8.as_str(), c.complete))+ .collect();+ assert_eq!(+ seen,+ vec![("lz4", "dddd4444", false), ("quinn", "aaaa1111", true)]+ );+ assert_eq!(cached[1].pin_date.as_deref(), Some("2026-10-02"));+}++#[test]+fn a_sums_entry_that_climbs_out_of_the_pack_dir_makes_it_incomplete() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let dir = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ fs::write(hub.join("outside"), b"x").unwrap();+ fs::write(+ dir.join("SHA256SUMS"),+ "00 pack.json\n00 ../../../outside\n",+ )+ .unwrap();+ assert!(!scan_cache(hub)[0].complete);+}++// ── refusals ─────────────────────────────────────────────────────────────────++#[test]+fn a_catalogue_older_than_the_cache_never_downgrades_it() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let cached = make_pack(hub, "quinn", "bbbb2222", "2026-10-09");+ // A replayed (validly signed, but old) catalogue still lists yesterday's pin.+ let cat = catalogue(vec![entry("quinn", "aaaa1111", "2026-10-02")]);+ let ops = Fake::new(hub).with_ledger("quinn", "bbbb2222");++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(ops.pulled().is_empty());+ assert!(cached.is_dir());+ assert_eq!(report.kept, vec!["quinn"]);+ assert_eq!(report.exit_code(), 0, "a refusal is not a failure");+ assert!(+ has_line(&lines, "older than the cached one (2026-10-09)"),+ "lines: {lines:?}"+ );+}++#[test]+fn a_slug_the_catalogue_no_longer_lists_is_kept() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let dir = make_pack(hub, "gone", "aaaa1111", "2026-10-02");+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);+ let ops = Fake::new(hub);++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(dir.is_dir());+ assert!(ops.pulled().is_empty());+ assert_eq!(report.kept, vec!["gone"]);+ assert!(has_line(+ &lines,+ "gone aaaa1111: kept (not in the catalogue)"+ ));+}++#[test]+fn excluded_corpora_and_oversize_packs_are_never_pulled() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let a = make_pack(hub, "closed", "aaaa1111", "2026-10-02");+ let b = make_pack(hub, "huge", "bbbb2222", "2026-10-02");+ let mut closed = entry("closed", "cccc3333", "2026-10-09");+ closed.licence_class = LicenceClass::Excluded;+ let mut huge = entry("huge", "dddd4444", "2026-10-09");+ huge.pack_bytes = AUTO_PULL_MAX_BYTES + 1;+ let cat = catalogue(vec![closed, huge]);+ let ops = Fake::new(hub)+ .with_ledger("closed", "aaaa1111")+ .with_ledger("huge", "bbbb2222");++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(ops.pulled().is_empty(), "neither is downloaded");+ assert!(a.is_dir() && b.is_dir(), "both cached copies are kept");+ assert_eq!(report.kept, vec!["closed", "huge"]);+ assert_eq!(report.exit_code(), 0);+ assert!(has_line(&lines, "classes it excluded"));+ assert!(has_line(&lines, "over the 52428800-byte auto-pull cap"));+}++#[test]+fn a_current_pack_stays_current_when_the_catalogue_later_calls_it_excluded() {+ // `excluded` stops a NEW pin being pulled; it is not a reason to report a+ // pack that is already at the catalogue's pin as something to refuse.+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let dir = make_pack(hub, "closed", "aaaa1111", "2026-10-02");+ let mut same_pin = entry("closed", "aaaa1111", "2026-10-02");+ same_pin.licence_class = LicenceClass::Excluded;+ let ops = Fake::new(hub).with_ledger("closed", "aaaa1111");++ let (report, lines) = sync(hub, &catalogue(vec![same_pin]), &ops, DISK_BUDGET_BYTES);++ assert!(dir.is_dir() && ops.pulled().is_empty());+ assert_eq!(report.current, vec!["closed"]);+ assert!(+ has_line(&lines, "hub sync: closed aaaa1111: current"),+ "{lines:?}"+ );+}++#[test]+fn a_slug_served_from_a_local_index_is_left_alone() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let dir = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);+ let ops = Fake::new(hub);+ ops.ledger+ .borrow_mut()+ .insert("quinn".into(), Ledger::Local);++ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ assert!(+ ops.pulled().is_empty(),+ "a pull would flip the ledger back to the pack"+ );+ assert!(dir.is_dir());+ assert_eq!(report.kept, vec!["quinn"]);+ assert!(has_line(&lines, "served from a local index"));+}++#[test]+fn a_catalogue_pin_that_is_not_a_commit_is_refused() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ make_pack(hub, "quinn", "aaaa1111", "2026-10-02");+ let mut bad = entry("quinn", "bbbb2222", "2026-10-09");+ bad.pin_sha = "../../etc".into();+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111");++ let (report, lines) = sync(hub, &catalogue(vec![bad]), &ops, DISK_BUDGET_BYTES);++ assert!(ops.pulled().is_empty());+ assert_eq!(report.kept, vec!["quinn"]);+ assert!(has_line(&lines, "is not a hex commit"));+}++// ── the disk budget ──────────────────────────────────────────────────────────++/// A complete pack padded to `size` bytes with a given recency.+fn sized_pack(hub: &Path, slug: &str, pin8: &str, size: usize, used: SystemTime) -> PathBuf {+ let dir = make_pack(hub, slug, pin8, "2026-10-02");+ fs::write(dir.join("snapshot").join("segment.bin"), vec![0u8; size]).unwrap();+ let marker = dir.join("last_used");+ fs::write(&marker, b"x").unwrap();+ set_mtime(&marker, used);+ dir+}++#[test]+fn the_budget_pass_removes_least_recently_used_first_until_it_fits() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let oldest = sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(30));+ let middle = sized_pack(hub, "beta", "bbbb2222", 10_000, days_ago(10));+ let newest = sized_pack(hub, "gamma", "cccc3333", 10_000, days_ago(1));+ let total = dir_size(&hub.join("packs"));+ // Room for two of the three packs.+ let budget = total - 10_000;++ let (pruned, left) = prune_to_budget(hub, budget, &BTreeSet::new());++ assert_eq!(+ pruned.iter().map(|p| p.slug.as_str()).collect::<Vec<_>>(),+ vec!["alpha"],+ "only the least recently used pack had to go"+ );+ assert!(left <= budget);+ assert!(!oldest.exists() && middle.is_dir() && newest.is_dir());+ assert!(+ !hub.join("packs/alpha").exists(),+ "an emptied slug directory is dropped too"+ );+}++#[test]+fn the_budget_pass_keeps_removing_until_the_cache_fits() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(30));+ sized_pack(hub, "beta", "bbbb2222", 10_000, days_ago(10));+ let newest = sized_pack(hub, "gamma", "cccc3333", 10_000, days_ago(1));+ let one_pack = dir_size(&newest);++ let (pruned, left) = prune_to_budget(hub, one_pack + 1, &BTreeSet::new());++ assert_eq!(+ pruned.iter().map(|p| p.slug.as_str()).collect::<Vec<_>>(),+ vec!["alpha", "beta"],+ "oldest first"+ );+ assert!(left <= one_pack + 1);+ assert!(newest.is_dir());+}++#[test]+fn a_cache_inside_the_budget_is_left_exactly_alone() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let dir = sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(300));++ let (pruned, _) = prune_to_budget(hub, DISK_BUDGET_BYTES, &BTreeSet::new());++ assert!(pruned.is_empty());+ assert!(dir.is_dir());+}++#[test]+fn leftovers_of_pulls_do_not_count_against_the_budget() {+ // The budget measures what sync can remove. A crashed pull's `.tmp` is the+ // pull's to clean up; counting it would evict a good pack to make room for+ // bytes this can never free.+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let dir = sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(300));+ let junk = hub.join("packs/alpha/bbbb2222.tmp");+ fs::create_dir_all(&junk).unwrap();+ fs::write(junk.join("partial.tar.zst"), vec![0u8; 1_000_000]).unwrap();+ let budget = dir_size(&dir) + 100;++ let (pruned, left) = prune_to_budget(hub, budget, &BTreeSet::new());++ assert!(+ pruned.is_empty(),+ "the pack fits; a megabyte of junk elsewhere is not its problem"+ );+ assert!(left <= budget);+ assert!(dir.is_dir());+ assert!(+ junk.is_dir(),+ "and sync does not touch another process's temp directory"+ );+}++#[test]+fn incomplete_directories_are_pruned_before_any_pack() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let good = sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(300));+ let junk = hub.join("packs/beta/bbbb2222");+ fs::create_dir_all(&junk).unwrap();+ fs::write(junk.join("partial.bin"), vec![0u8; 10_000]).unwrap();+ let budget = dir_size(&hub.join("packs")) - 5_000;++ let (pruned, _) = prune_to_budget(hub, budget, &BTreeSet::new());++ assert_eq!(pruned.len(), 1);+ assert_eq!(+ pruned[0].slug, "beta",+ "the half-extracted leftover goes first"+ );+ assert!(+ good.is_dir(),+ "even though the real pack was used less recently"+ );+}++#[test]+fn packs_updated_in_this_run_are_not_pruned() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ sized_pack(hub, "quinn", "aaaa1111", 10_000, days_ago(30));+ let other = sized_pack(hub, "lz4", "dddd4444", 10_000, days_ago(1));+ let cat = catalogue(vec![+ entry("quinn", "bbbb2222", "2026-10-09"),+ entry("lz4", "dddd4444", "2026-10-02"),+ ]);+ let ops = Fake::new(hub)+ .with_ledger("quinn", "aaaa1111")+ .with_ledger("lz4", "dddd4444")+ .with_script(+ "quinn",+ Script::Land {+ pin8: "bbbb2222".into(),+ pin_date: "2026-10-09".into(),+ },+ );+ // Far too small for either pack: lz4 (used yesterday) is the only candidate,+ // because quinn was just fetched — so it goes, and quinn stays over budget.+ let budget = 100;++ let (report, lines) = sync(hub, &cat, &ops, budget);++ assert!(+ hub.join("packs/quinn/bbbb2222").is_dir(),+ "the fresh pack stays"+ );+ assert!(!other.exists(), "the only candidate went");+ assert_eq!(report.pruned.len(), 1);+ assert_eq!(report.pruned[0].slug, "lz4");+ assert!(has_line(&lines, "pruned lz4 dddd4444"), "lines: {lines:?}");+ assert!(+ has_line(&lines, "nothing more can be pruned"),+ "still over budget, and it says so: {lines:?}"+ );+}++#[test]+fn an_update_keeps_the_old_pins_recency() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path();+ let long_idle = days_ago(40);+ sized_pack(hub, "quinn", "aaaa1111", 100, long_idle);+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(+ "quinn",+ Script::Land {+ pin8: "bbbb2222".into(),+ pin_date: "2026-10-09".into(),+ },+ );++ sync(hub, &cat, &ops, DISK_BUDGET_BYTES);++ let new = scan_cache(hub);+ assert_eq!(new.len(), 1);+ assert_eq!(new[0].pin8, "bbbb2222");+ let (early, late) = if new[0].last_used < long_idle {+ (new[0].last_used, long_idle)+ } else {+ (long_idle, new[0].last_used)+ };+ let drift = late.duration_since(early).unwrap();+ assert!(+ drift < Duration::from_secs(2),+ "refreshing a pack is not using it: its recency must stay the old pin's, \+ but it moved by {drift:?}"+ );+}++// ── last_sync and the exit code ──────────────────────────────────────────────++#[test]+fn last_sync_is_an_rfc3339_stamp_that_status_can_read() {+ let tmp = tempdir().unwrap();+ let hub = tmp.path().join("fresh-hub-dir");++ write_last_sync(&hub).unwrap();++ let raw = fs::read_to_string(hub.join("last_sync")).unwrap();+ chrono::DateTime::parse_from_rfc3339(raw.trim()).expect("RFC 3339");+ assert_eq!(+ crate::lockfile::gather_status(&hub).last_sync_age_days,+ Some(0),+ "`xerj hub status` reads the file's age"+ );+}++#[test]+fn the_exit_code_ranks_verification_over_infrastructure() {+ let mut r = SyncReport::default();+ assert_eq!(r.exit_code(), 0);+ r.failed.push(SyncFailure {+ slug: "a".into(),+ code: 1,+ message: String::new(),+ });+ assert_eq!(r.exit_code(), 1);+ r.failed.push(SyncFailure {+ slug: "b".into(),+ code: 3,+ message: String::new(),+ });+ assert_eq!(r.exit_code(), 3);+}++// ── the child-process pull ───────────────────────────────────────────────────++#[cfg(unix)]+mod child_process {+ use super::*;++ /// A `sh` script standing in for `xerj`: records its arguments, writes a+ /// line to stderr, exits with `code`. Run as `sh <script> hub pull …` so no+ /// freshly written executable is ever exec'd (no ETXTBSY race under the+ /// parallel test runner).+ fn fake_xerj(dir: &Path, code: i32, stderr: &str) -> (PathBuf, PathBuf) {+ let args = dir.join("args.txt");+ let script = dir.join("fake-xerj.sh");+ fs::write(+ &script,+ format!(+ "printf '%s\\n' \"$@\" > '{}'\necho '{stderr}' >&2\nexit {code}\n",+ args.display()+ ),+ )+ .unwrap();+ (script, args)+ }++ fn ops(script: &Path, url: Option<&str>, code_root: &Path) -> CliOps {+ CliOps::with_program(+ "sh",+ vec![script.as_os_str().to_owned()],+ url.map(str::to_string),+ code_root.to_path_buf(),+ )+ }++ #[test]+ fn pull_runs_the_existing_pull_subcommand_quietly() {+ let tmp = tempdir().unwrap();+ let (script, args) = fake_xerj(tmp.path(), 0, "");++ let out = ops(&script, Some("http://127.0.0.1:1"), tmp.path()).pull("quinn");++ assert_eq!(out, PullOutcome::Done);+ assert_eq!(+ fs::read_to_string(args)+ .unwrap()+ .lines()+ .collect::<Vec<_>>(),+ vec![+ "hub",+ "pull",+ "quinn",+ "--quiet",+ "--url",+ "http://127.0.0.1:1"+ ],+ "sync drives `xerj hub pull`, it does not reimplement it"+ );+ }++ #[test]+ fn pull_omits_url_when_none_was_given() {+ let tmp = tempdir().unwrap();+ let (script, args) = fake_xerj(tmp.path(), 0, "");++ ops(&script, None, tmp.path()).pull("quinn");++ assert_eq!(+ fs::read_to_string(args)+ .unwrap()+ .lines()+ .collect::<Vec<_>>(),+ vec!["hub", "pull", "quinn", "--quiet"]+ );+ }++ #[test]+ fn a_nonzero_exit_carries_its_code_and_stderr() {+ let tmp = tempdir().unwrap();+ let (script, _) = fake_xerj(+ tmp.path(),+ 3,+ "xerj hub pull: pack verification failed: pack signature",+ );++ let out = ops(&script, None, tmp.path()).pull("quinn");++ assert_eq!(+ out,+ PullOutcome::Failed {+ code: 3,+ message: "xerj hub pull: pack verification failed: pack signature".into()+ }+ );+ }++ #[test]+ fn a_missing_program_is_an_infrastructure_failure() {+ let tmp = tempdir().unwrap();+ let ops = CliOps::with_program(+ tmp.path().join("no-such-xerj"),+ vec![],+ None,+ tmp.path().to_path_buf(),+ );++ match ops.pull("quinn") {+ PullOutcome::Failed { code, message } => {+ assert_eq!(code, 1);+ assert!(message.contains("cannot run"), "{message}");+ }+ other => panic!("expected a failure, got {other:?}"),+ }+ }++ #[test]+ fn the_ledger_is_read_from_the_code_root() {+ let tmp = tempdir().unwrap();+ let state = tmp.path().join("state");+ fs::create_dir_all(&state).unwrap();+ let (script, _) = fake_xerj(tmp.path(), 0, "");+ let ops = ops(&script, None, tmp.path());++ assert_eq!(ops.ledger("quinn"), Ledger::Missing);++ fs::write(+ state.join("quinn.json"),+ format!(+ r#"{{"corpus":"quinn","source":"hubpack","pin_sha":"{}"}}"#,+ full_sha("bbbb2222")+ ),+ )+ .unwrap();+ assert_eq!(+ ops.ledger("quinn"),+ Ledger::Hubpack {+ pin8: Some("bbbb2222".into())+ }+ );++ fs::write(+ state.join("lz4.json"),+ r#"{"corpus":"lz4","index_prefix":"xc-lz4-b1700000000"}"#,+ )+ .unwrap();+ assert_eq!(ops.ledger("lz4"), Ledger::Local, "no `source: hubpack`");+ }+}++#[test]+fn one_line_flattens_and_caps_stderr() {+ assert_eq!(one_line("a\n b\t c\n", 100), "a b c");+ assert_eq!(one_line("abcdefghij", 4), "abcd…");+}++// ── the timer ────────────────────────────────────────────────────────────────++fn timer_dirs(root: &Path) -> TimerDirs {+ TimerDirs {+ systemd_user: root.join("cfg/systemd/user"),+ launch_agents: root.join("Library/LaunchAgents"),+ hub_home: root.join("hub"),+ }+}++#[test]+fn install_timer_writes_a_six_hourly_systemd_user_timer_and_nothing_else() {+ let tmp = tempdir().unwrap();+ let dirs = timer_dirs(tmp.path());+ let exe = Path::new("/home/u/.local/bin/xerj");++ let printed = install_timer(TimerKind::SystemdUser, exe, &dirs).unwrap();++ // The path printed is the timer — the unit the user enables.+ assert_eq!(printed, dirs.systemd_user.join("xerj-hub-sync.timer"));+ let timer = fs::read_to_string(&printed).unwrap();+ let service = fs::read_to_string(dirs.systemd_user.join("xerj-hub-sync.service")).unwrap();++ let timer_lines: Vec<&str> = timer.lines().collect();+ for want in [+ "[Timer]",+ "OnCalendar=*-*-* 00/6:00:00",+ "Persistent=true",+ "[Install]",+ "WantedBy=timers.target",+ ] {+ assert!(+ timer_lines.contains(&want),+ "timer lacks {want:?}:\n{timer}"+ );+ }+ let service_lines: Vec<&str> = service.lines().collect();+ for want in [+ "[Service]",+ "Type=oneshot",+ "ExecStart=/home/u/.local/bin/xerj hub sync",+ ] {+ assert!(+ service_lines.contains(&want),+ "service lacks {want:?}:\n{service}"+ );+ }++ // Nothing else: two files in the systemd dir, no hub dir, no launchd dir.+ let mut written: Vec<String> = fs::read_dir(&dirs.systemd_user)+ .unwrap()+ .flatten()+ .map(|e| e.file_name().to_string_lossy().into_owned())+ .collect();+ written.sort();+ assert_eq!(+ written,+ vec!["xerj-hub-sync.service", "xerj-hub-sync.timer"]+ );+ assert!(!dirs.hub_home.exists() && !dirs.launch_agents.exists());+}++#[test]+fn install_timer_overwrites_so_a_moved_binary_is_picked_up() {+ let tmp = tempdir().unwrap();+ let dirs = timer_dirs(tmp.path());++ install_timer(TimerKind::SystemdUser, Path::new("/old/xerj"), &dirs).unwrap();+ install_timer(TimerKind::SystemdUser, Path::new("/new/xerj"), &dirs).unwrap();++ let service = fs::read_to_string(dirs.systemd_user.join("xerj-hub-sync.service")).unwrap();+ assert!(service.contains("ExecStart=/new/xerj hub sync"));+ assert!(!service.contains("/old/xerj"));+}++#[test]+fn a_binary_path_that_needs_quoting_is_quoted_for_systemd() {+ let tok = |p: &str| systemd_exec_token(Path::new(p)).unwrap();+ assert_eq!(tok("/usr/bin/xerj"), "/usr/bin/xerj");+ assert_eq!(tok("/opt/my tools/xerj"), "\"/opt/my tools/xerj\"");+ // `%` starts a specifier and `$` a variable even inside quotes.+ assert_eq!(tok("/opt/100%/xerj"), "\"/opt/100%%/xerj\"");+ assert_eq!(tok("/opt/$HOME/xerj"), "\"/opt/$$HOME/xerj\"");+ assert_eq!(tok("/opt/a\"b\\c/xerj"), "\"/opt/a\\\"b\\\\c/xerj\"");+ assert!(systemd_exec_token(Path::new("/opt/a\nb")).is_err());++ let service = render_systemd_service(Path::new("/opt/my tools/xerj")).unwrap();+ assert!(service.contains("ExecStart=\"/opt/my tools/xerj\" hub sync"));+}++#[test]+fn a_binary_path_that_cannot_be_written_safely_leaves_nothing_behind() {+ let tmp = tempdir().unwrap();+ let dirs = timer_dirs(tmp.path());++ let err = install_timer(TimerKind::SystemdUser, Path::new("/opt/a\nb/xerj"), &dirs)+ .expect_err("a newline cannot ride in a unit file");++ assert!(format!("{err:#}").contains("control character"));+ assert!(!dirs.systemd_user.exists(), "no half-installed units");+}++#[test]+fn install_timer_writes_a_six_hourly_launchd_agent() {+ let tmp = tempdir().unwrap();+ let dirs = timer_dirs(tmp.path());+ let exe = Path::new("/Users/u/bin/xerj & co/xerj");++ let printed = install_timer(TimerKind::Launchd, exe, &dirs).unwrap();++ assert_eq!(printed, dirs.launch_agents.join("org.xerj.hub-sync.plist"));+ let plist = fs::read_to_string(&printed).unwrap();+ for want in [+ "<key>Label</key>\n\t<string>org.xerj.hub-sync</string>",+ // argv in order: the binary, then `hub sync`.+ "<string>/Users/u/bin/xerj & co/xerj</string>\n\t\t<string>hub</string>\n\t\t<string>sync</string>",+ "<key>StartInterval</key>\n\t<integer>21600</integer>",+ "<key>RunAtLoad</key>\n\t<false/>",+ ] {+ assert!(plist.contains(want), "plist lacks {want:?}:\n{plist}");+ }+ let log = dirs.hub_home.join("sync.log");+ assert!(+ plist.contains(&format!("<string>{}</string>", log.display())),+ "launchd output lands in the hub home"+ );+ assert!(+ dirs.hub_home.is_dir(),+ "launchd will not create the log's directory"+ );+ assert!(plist.starts_with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"));+ assert!(plist.trim_end().ends_with("</plist>"));+ assert!(+ !dirs.systemd_user.exists(),+ "only the launchd file is written"+ );+}++#[test]+fn timer_dirs_follow_xdg_config_home_and_ignore_a_relative_one() {+ let env = |pairs: &'static [(&'static str, &'static str)]| {+ move |k: &str| -> Option<OsString> {+ pairs+ .iter()+ .find(|(name, _)| *name == k)+ .map(|(_, v)| OsString::from(*v))+ }+ };++ let dirs =+ TimerDirs::from_env(&env(&[("HOME", "/home/u"), ("XDG_CONFIG_HOME", "/cfg")])).unwrap();+ assert_eq!(dirs.systemd_user, PathBuf::from("/cfg/systemd/user"));+ assert_eq!(+ dirs.launch_agents,+ PathBuf::from("/home/u/Library/LaunchAgents")+ );+ assert_eq!(dirs.hub_home, PathBuf::from("/home/u/.xerj/hub"));++ // XDG: a relative XDG_CONFIG_HOME is invalid and ignored.+ let dirs =+ TimerDirs::from_env(&env(&[("HOME", "/home/u"), ("XDG_CONFIG_HOME", "rel")])).unwrap();+ assert_eq!(+ dirs.systemd_user,+ PathBuf::from("/home/u/.config/systemd/user")+ );++ // The same hub-home override every other subcommand honours.+ let dirs =+ TimerDirs::from_env(&env(&[("HOME", "/home/u"), ("XERJ_HUB_HOME", "/srv/hub")])).unwrap();+ assert_eq!(dirs.hub_home, PathBuf::from("/srv/hub"));++ assert!(+ TimerDirs::from_env(&env(&[])).is_err(),+ "no HOME, no user timer"+ );+}++/// systemd is the arbiter of what a unit means, so when the machine has+/// `systemd-analyze` ask it: the calendar spec must fire every six hours, and+/// `verify` must have no complaint about the files we wrote. (Skipped where it+/// is absent: it is not a build dependency.)+#[cfg(target_os = "linux")]+#[test]+fn systemd_reads_the_units_as_a_six_hourly_oneshot() {+ let have = Command::new("systemd-analyze").arg("--version").output();+ if !have.is_ok_and(|o| o.status.success()) {+ eprintln!("systemd-analyze not available; skipping");+ return;+ }++ // The calendar spec in the timer we render, in UTC so the host's zone is moot.+ let spec = render_systemd_timer()+ .lines()+ .find_map(|l| l.strip_prefix("OnCalendar=").map(str::to_string))+ .expect("the timer has an OnCalendar line");+ let cal = Command::new("systemd-analyze")+ .env("TZ", "UTC")+ .args(["calendar", "--iterations=5", &spec])+ .output()+ .unwrap();+ let text = String::from_utf8_lossy(&cal.stdout).into_owned();+ assert!(cal.status.success(), "calendar spec rejected:\n{text}");+ let hours: Vec<&str> = text+ .lines()+ .filter(|l| l.contains("UTC") && !l.contains("From now"))+ .filter_map(|l| l.split_whitespace().rev().nth(1))+ .collect();+ assert_eq!(+ hours,+ vec!["00:00:00", "06:00:00", "12:00:00", "18:00:00", "00:00:00"],+ "five successive firings are six hours apart:\n{text}"+ );++ // Whole-unit check. `/bin/true` exists everywhere systemd does.+ let tmp = tempdir().unwrap();+ let dirs = timer_dirs(tmp.path());+ install_timer(TimerKind::SystemdUser, Path::new("/bin/true"), &dirs).unwrap();+ let verify = Command::new("systemd-analyze")+ .arg("verify")+ .arg(dirs.systemd_user.join("xerj-hub-sync.service"))+ .arg(dirs.systemd_user.join("xerj-hub-sync.timer"))+ .output()+ .unwrap();+ let diag = String::from_utf8_lossy(&verify.stderr);+ assert!(+ verify.status.success() && !diag.contains("xerj-hub-sync"),+ "systemd-analyze verify complained:\n{diag}"+ );+}++// ── the subcommand's argument handling ───────────────────────────────────────++fn args(a: &[&str]) -> Vec<String> {+ a.iter().map(|s| s.to_string()).collect()+}++#[test]+fn sync_rejects_what_it_does_not_understand_before_touching_anything() {+ assert_eq!(run_sync(&args(&["--bogus"])), 2);+ assert_eq!(run_sync(&args(&["some-slug"])), 2, "sync takes no slug");+ assert_eq!(run_sync(&args(&["--url"])), 2, "--url needs a value");+ assert_eq!(+ run_sync(&args(&["--install-timer", "--url", "http://x"])),+ 2,+ "--install-timer takes no other flag"+ );+ assert_eq!(run_sync(&args(&["--help"])), 0);+ assert_eq!(run_sync(&args(&["-h"])), 0);+}++#[test]+fn the_hub_command_routes_sync_here() {+ // `--help` exits 0 only if `sync` is a known subcommand (an unknown one+ // exits 2), and it does so without touching the network or the cache.+ assert_eq!(crate::cli::run_hub_cli(&args(&["sync", "--help"])), 0);+ assert_eq!(crate::cli::run_hub_cli(&args(&["sync", "--bogus"])), 2);+}++#[test]+fn the_usage_names_every_flag_and_exit_code() {+ let usage = sync_usage();+ for needle in [+ "--url <node>",+ "--install-timer",+ "every 6 hours",+ "exit codes: 0",+ "3 a pack",+ ] {+ assert!(usage.contains(needle), "usage lacks {needle:?}");+ }+ // The budget it names is the one the code enforces, not a typed-in number.+ assert!(+ usage.contains(&format!(+ "the {} disk budget",+ human_bytes(DISK_BUDGET_BYTES)+ )),+ "{usage}"+ );+}diff --git a/engine/crates/xerj-hub/tests/sync_e2e.rs b/engine/crates/xerj-hub/tests/sync_e2e.rsnew file mode 100644index 000000000..bf770c681--- /dev/null+++ b/engine/crates/xerj-hub/tests/sync_e2e.rs@@ -0,0 +1,687 @@+//! Corpus Hub v2 — `xerj hub sync` end to end (plan §8 "Tests").+//!+//! Real nodes (an ephemeral TCP port each, the way `pack_roundtrip.rs` does), the+//! real `xerj hub pull`, a real signed catalogue served over HTTP from a throwaway+//! local "hub", and the real `sync` command — everything except the process+//! boundary: sync normally runs `pull` as a child of the `xerj` binary, which a+//! test binary cannot be, so the pull is driven in-process through the same+//! `run_hub_cli(["pull", …])` entry point instead.+//!+//! The story, in one `#[tokio::test]` because it mutates process-global env+//! (`XERJ_CODE_HOME`, `XERJ_HUB_HOME`, `XERJ_HUB_PUBKEY`, `XERJ_HUB_URL`):+//!+//! 1. two packs of one corpus (`mini-c`) — an older pin and a newer one — are built+//! and signed on a builder node;+//! 2. a client node pulls the older pin; `def` answers `MINI_LIMIT = 128`;+//! 3. the hub's catalogue moves to the newer pin and `sync` runs: the pack is+//! replaced, the OLD directory is gone, `def` now answers `256` (the restore+//! really happened), `last_sync` is written;+//! 4. a second sync pulls nothing;+//! 5. the catalogue moves to a pin whose archive is corrupted: sync exits 3, the+//! pin that works stays in place and keeps answering, `last_sync` is untouched;+//! 6. a catalogue whose signature does not verify is refused (exit 1) and changes+//! nothing.++use std::io::{Read, Write};+use std::net::{TcpListener, TcpStream};+use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::{Arc, Mutex};+use std::time::{Duration, SystemTime};++use serde_json::json;+use sha2::{Digest, Sha256};++use xerj_autoindex::harvest::sign::{generate_keypair, sign_bytes};+use xerj_hub::catalogue::canonical_json;+use xerj_hub::node::HubNode;+use xerj_hub::pack::{self, PackFileEntry};+use xerj_hub::sync::{run_sync_using, CliOps, Ledger, PullOutcome, SyncOps};+use xerj_hub::{+ Catalogue, CatalogueEntry, DefRecord, LicenceClass, PackIndex, PackManifest, PackSource,+ Sketch, CATALOGUE_FORMAT_VERSION, PACK_FORMAT_VERSION,+};++/// One pin of the `mini-c` corpus: the older and the newer differ in pin, date,+/// index prefix and in the value `MINI_LIMIT` carries, so a `def` lookup says+/// which pack is live.+#[derive(Clone, Copy)]+struct PackSpec {+ pin_sha: &'static str,+ pin_date: &'static str,+ prefix: &'static str,+ limit: &'static str,+}++const OLD: PackSpec = PackSpec {+ pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5",+ pin_date: "2026-10-02",+ prefix: "xc-mini-c-b1",+ limit: "128",+};+const NEW: PackSpec = PackSpec {+ pin_sha: "9f3c1a2b4d5e60718293a4b5c6d7e8f901234567",+ pin_date: "2026-10-09",+ prefix: "xc-mini-c-b2",+ limit: "256",+};+/// A pin the catalogue will advertise with a corrupted archive behind it.+const BAD_SHA: &str = "c0ffee0011223344556677889900aabbccddeeff";++impl PackSpec {+ fn pin8(&self) -> &'static str {+ &self.pin_sha[..8]+ }+}++/// Boot an es-compat node on an ephemeral port with `data_dir` and the given+/// snapshot-repo allowlist, and wait until it answers.+async fn spawn_node(data_dir: &Path, allowlist: Vec<String>) -> String {+ std::fs::create_dir_all(data_dir).unwrap();+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = data_dir.to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ config.limits.snapshot_repo_allowlist = allowlist;+ let metrics = xerj_common::metrics::Metrics::new().unwrap();+ let engine = xerj_engine::Engine::new(config.clone()).unwrap();+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ let app = xerj_api::router::build_es_compat_router(state);+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let port = listener.local_addr().unwrap().port();+ tokio::spawn(async move {+ let _ = axum::serve(listener, app.into_make_service()).await;+ });+ let url = format!("http://127.0.0.1:{port}");+ let probe = url.clone();+ tokio::task::spawn_blocking(move || {+ HubNode::connect(&probe)+ .unwrap()+ .wait_until_reachable(Duration::from_secs(30))+ .unwrap();+ })+ .await+ .unwrap();+ url+}++fn sample_defs(spec: &PackSpec) -> Vec<DefRecord> {+ vec![+ DefRecord {+ symbol: "MINI_LIMIT".into(),+ kind: "define".into(),+ lang: "c".into(),+ file: "limits.h".into(),+ line: 5,+ text: format!("#define MINI_LIMIT {}", spec.limit),+ value: Some(spec.limit.into()),+ },+ DefRecord {+ symbol: "mini_clamp".into(),+ kind: "function".into(),+ lang: "c".into(),+ file: "clamp.c".into(),+ line: 4,+ text: "int mini_clamp(int value, int lo, int hi)".into(),+ value: None,+ },+ ]+}++fn manifest(spec: &PackSpec) -> PackManifest {+ PackManifest {+ format_version: PACK_FORMAT_VERSION,+ slug: "mini-c".into(),+ pin_sha: spec.pin_sha.into(),+ pin_date: spec.pin_date.into(),+ built_at: "2026-10-14T00:00:00Z".into(),+ builder_version: "xerj test".into(),+ index_prefix: spec.prefix.into(),+ indices: vec![+ PackIndex {+ name: format!("{}-defs", spec.prefix),+ docs: 2,+ bytes: 0,+ },+ PackIndex {+ name: format!("{}-docs", spec.prefix),+ docs: 1,+ bytes: 0,+ },+ ],+ snapshot_name: format!("mini-c-{}", spec.pin8()),+ licence_spdx: "MIT".into(),+ licence_class: LicenceClass::AdaptWithAttribution,+ sources: vec![PackSource {+ repo: "mini-c".into(),+ url: "file:///fixtures/mini-c".into(),+ sha: spec.pin_sha.into(),+ licence: "MIT".into(),+ }],+ defs_count: 2,+ files_count: 2,+ }+}++/// Create the docs + defs indices for `spec` on the builder node and snapshot+/// `<prefix>-*` into `location`.+fn seed_and_snapshot(url: &str, spec: &PackSpec, location: &Path) {+ let node = HubNode::connect(url).unwrap();+ let prefix = spec.prefix;+ node.request_ok(+ "PUT",+ &format!("/{prefix}-docs"),+ Some(&json!({"mappings":{"properties":{"body":{"type":"text"}}}})),+ )+ .unwrap();+ node.bulk_ndjson(+ &format!("/{prefix}-docs/_bulk"),+ "{\"index\":{}}\n{\"body\":\"mini c reference\"}\n".to_string(),+ )+ .unwrap();+ node.request_ok(+ "PUT",+ &format!("/{prefix}-defs"),+ Some(&json!({"mappings":{"properties":{+ "symbol":{"type":"keyword"},+ "symbol_text":{"type":"text"},+ "kind":{"type":"keyword"},+ "lang":{"type":"keyword"},+ "file":{"type":"keyword"},+ "line":{"type":"integer"},+ "text":{"type":"text"},+ "value":{"type":"keyword"}+ }}})),+ )+ .unwrap();+ let mut ndjson = String::new();+ for d in sample_defs(spec) {+ ndjson.push_str("{\"index\":{}}\n");+ ndjson.push_str(+ &json!({+ "symbol": d.symbol, "symbol_text": d.symbol, "kind": d.kind,+ "lang": d.lang, "file": d.file, "line": d.line, "text": d.text, "value": d.value,+ })+ .to_string(),+ );+ ndjson.push('\n');+ }+ node.bulk_ndjson(&format!("/{prefix}-defs/_bulk"), ndjson)+ .unwrap();+ node.request_ok("POST", &format!("/{prefix}-defs/_refresh"), None)+ .ok();++ let repo = format!("hub-build-{}", spec.pin8());+ node.request_ok(+ "PUT",+ &format!("/_snapshot/{repo}"),+ Some(&json!({"type":"fs","settings":{"location": location.to_string_lossy()}})),+ )+ .unwrap();+ node.request_ok(+ "PUT",+ &format!(+ "/_snapshot/{repo}/mini-c-{}?wait_for_completion=true",+ spec.pin8()+ ),+ Some(&json!({"indices": format!("{prefix}-*")})),+ )+ .unwrap();+}++/// Build and sign one pack on the builder node; returns the archive path and its+/// sha256.+fn build_pack(builder_url: &str, spec: &PackSpec, out: &Path, seed: &str) -> (PathBuf, String) {+ let location = out.join(format!("snapshot-{}", spec.pin8()));+ seed_and_snapshot(builder_url, spec, &location);+ let files = vec![PackFileEntry {+ path: "limits.h".into(),+ lang: "c".into(),+ bytes: 300,+ licence: "MIT".into(),+ }];+ let archive = out.join(format!("mini-c-{}.hubpack.tar.zst", spec.pin8()));+ pack::write_pack(+ &archive,+ &manifest(spec),+ &sample_defs(spec),+ &files,+ &location,+ seed,+ )+ .unwrap();+ let sha = sha256_hex(&archive);+ (archive, sha)+}++fn sha256_hex(path: &Path) -> String {+ Sha256::digest(std::fs::read(path).unwrap())+ .iter()+ .map(|b| format!("{b:02x}"))+ .collect()+}++fn sig_of(archive: &Path) -> PathBuf {+ let mut s = archive.as_os_str().to_os_string();+ s.push(".sig");+ PathBuf::from(s)+}++/// A catalogue entry whose pack is a `file://` archive.+fn catalogue_entry(+ pin_sha: &str,+ pin_date: &str,+ archive: &Path,+ pack_sha256: &str,+) -> CatalogueEntry {+ CatalogueEntry {+ slug: "mini-c".into(),+ domain: "DEV".into(),+ topics: vec!["c".into()],+ ask_for: "a tiny C fixture corpus".into(),+ pin_sha: pin_sha.into(),+ pin_date: pin_date.into(),+ refresh_cadence_days: 7,+ licence_spdx: "MIT".into(),+ licence_class: LicenceClass::AdaptWithAttribution,+ pack_bytes: std::fs::metadata(archive).map(|m| m.len()).unwrap_or(0),+ pack_url: format!("file://{}", archive.display()),+ pack_sha256: pack_sha256.into(),+ sig_url: format!("file://{}", sig_of(archive).display()),+ dependency_names: vec![],+ quality_grade: None,+ term_sketch: Sketch {+ m_bits: 64,+ k_hashes: 7,+ bits_base64: "AAAAAAAAAAA=".into(),+ },+ }+}++/// The signed bytes of a one-corpus catalogue: `(catalogue.json, signature)`.+fn signed_catalogue(entry: CatalogueEntry, seed: &str) -> (Vec<u8>, String) {+ let cat = Catalogue {+ format_version: CATALOGUE_FORMAT_VERSION,+ generated_at: "2026-10-14T00:00:00Z".into(),+ corpora: vec![entry],+ };+ let bytes = canonical_json(&cat).unwrap();+ let sig = sign_bytes(&bytes, seed).unwrap();+ (bytes, sig)+}++/// A throwaway local "hub": serves whatever catalogue it currently holds at+/// `/api/v1/catalogue.json` and its signature at `….sig`, for as many requests as+/// come.+struct TinyHub {+ url: String,+ addr: std::net::SocketAddr,+ body: Arc<Mutex<(Vec<u8>, String)>>,+ stop: Arc<AtomicBool>,+ handle: Option<std::thread::JoinHandle<()>>,+}++impl TinyHub {+ fn start(initial: (Vec<u8>, String)) -> TinyHub {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let addr = listener.local_addr().unwrap();+ let body = Arc::new(Mutex::new(initial));+ let stop = Arc::new(AtomicBool::new(false));+ let (b, s) = (body.clone(), stop.clone());+ let handle = std::thread::spawn(move || {+ while let Ok((mut stream, _)) = listener.accept() {+ if s.load(Ordering::SeqCst) {+ break;+ }+ let mut buf = [0u8; 4096];+ let n = stream.read(&mut buf).unwrap_or(0);+ let req = String::from_utf8_lossy(&buf[..n]).into_owned();+ let first = req.lines().next().unwrap_or("");+ let (cat, sig) = b.lock().unwrap().clone();+ let payload: Vec<u8> = if first.contains(".sig") {+ sig.into_bytes()+ } else {+ cat+ };+ let head = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\+ Content-Length: {}\r\nConnection: close\r\n\r\n",+ payload.len()+ );+ let _ = stream.write_all(head.as_bytes());+ let _ = stream.write_all(&payload);+ let _ = stream.flush();+ }+ });+ TinyHub {+ url: format!("http://{addr}"),+ addr,+ body,+ stop,+ handle: Some(handle),+ }+ }++ fn serve(&self, bytes: Vec<u8>, sig: String) {+ *self.body.lock().unwrap() = (bytes, sig);+ }++ fn shutdown(mut self) {+ self.stop.store(true, Ordering::SeqCst);+ let _ = TcpStream::connect(self.addr); // wake the accept loop+ if let Some(h) = self.handle.take() {+ h.join().unwrap();+ }+ }+}++/// Drives the real `hub pull` in-process and reads the real ledger, counting the+/// pulls and noting what was on disk around each one.+struct InProcessOps {+ node_url: String,+ hub_home: PathBuf,+ code_home: PathBuf,+ pulls: Arc<AtomicUsize>,+ /// For each pull: was the OLD pin's directory present before it, and were+ /// BOTH directories present when it returned (i.e. sync had not yet removed+ /// the old one).+ around_pull: Arc<Mutex<Vec<(bool, bool)>>>,+}++impl SyncOps for InProcessOps {+ fn pull(&self, slug: &str) -> PullOutcome {+ self.pulls.fetch_add(1, Ordering::SeqCst);+ let old = self.hub_home.join("packs").join(slug).join(OLD.pin8());+ let new = self.hub_home.join("packs").join(slug).join(NEW.pin8());+ let old_before = old.is_dir();+ let rc = xerj_hub::cli::run_hub_cli(&[+ "pull".into(),+ slug.into(),+ "--url".into(),+ self.node_url.clone(),+ "--quiet".into(),+ ]);+ self.around_pull+ .lock()+ .unwrap()+ .push((old_before, old.is_dir() && new.is_dir()));+ if rc == 0 {+ PullOutcome::Done+ } else {+ PullOutcome::Failed {+ code: rc,+ message: format!("hub pull exited {rc}"),+ }+ }+ }++ fn ledger(&self, slug: &str) -> Ledger {+ // The real ledger reader, pointed at this test's code root.+ CliOps::with_program("unused", vec![], None, self.code_home.clone()).ledger(slug)+ }+}++fn age_file(path: &Path, secs: u64) {+ let when = SystemTime::now() - Duration::from_secs(secs);+ std::fs::OpenOptions::new()+ .write(true)+ .open(path)+ .unwrap()+ .set_modified(when)+ .unwrap();+}++fn pin_dirs(hub_home: &Path) -> Vec<String> {+ let mut v: Vec<String> = std::fs::read_dir(hub_home.join("packs/mini-c"))+ .map(|rd| {+ rd.flatten()+ .map(|e| e.file_name().to_string_lossy().into_owned())+ .collect()+ })+ .unwrap_or_default();+ v.sort();+ v+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn sync_replaces_the_pack_with_a_newer_pin_and_the_old_dir_is_gone() {+ // One canonical work root so every snapshot location is inside the nodes'+ // allowlist (the engine refuses a repo location outside data_dir or the+ // allowlist; we configure the root, never weaken the check).+ let work_tmp = tempfile::tempdir().unwrap();+ let work = std::fs::canonicalize(work_tmp.path()).unwrap();+ let out = work.join("out");+ std::fs::create_dir_all(&out).unwrap();+ let allowlist = vec![work.to_string_lossy().into_owned()];+ let (seed, public) = generate_keypair().unwrap();++ // ── 1. Build + sign both pins on the builder node ─────────────────────+ let builder = spawn_node(&work.join("data-builder"), allowlist.clone()).await;+ let (old_archive, old_sha, new_archive, new_sha) = {+ let (builder, out, seed) = (builder.clone(), out.clone(), seed.clone());+ tokio::task::spawn_blocking(move || {+ let (oa, os) = build_pack(&builder, &OLD, &out, &seed);+ let (na, ns) = build_pack(&builder, &NEW, &out, &seed);+ (oa, os, na, ns)+ })+ .await+ .unwrap()+ };++ // A corrupted copy of the NEW archive, advertised under a third pin with the+ // original (now mismatching) sha256 — the way a tampered download looks.+ let bad_archive = out.join("mini-c-c0ffee00.hubpack.tar.zst");+ std::fs::copy(&new_archive, &bad_archive).unwrap();+ std::fs::copy(sig_of(&new_archive), sig_of(&bad_archive)).unwrap();+ let mut bytes = std::fs::read(&bad_archive).unwrap();+ let mid = bytes.len() / 2;+ bytes[mid] ^= 0xff;+ std::fs::write(&bad_archive, &bytes).unwrap();++ // ── 2. The local hub starts out advertising the OLD pin ───────────────+ let hub = TinyHub::start(signed_catalogue(+ catalogue_entry(OLD.pin_sha, OLD.pin_date, &old_archive, &old_sha),+ &seed,+ ));+ let client = spawn_node(&work.join("data-client"), allowlist).await;+ let hub_home = work.join("hub-home");+ let code_home = work.join("code-home");+ std::fs::create_dir_all(&hub_home).unwrap();+ std::env::set_var("XERJ_CODE_HOME", &code_home);+ std::env::set_var("XERJ_HUB_HOME", &hub_home);+ std::env::set_var("XERJ_HUB_PUBKEY", &public);+ std::env::set_var("XERJ_HUB_URL", &hub.url);++ // The first pull is the ordinary one: nothing cached yet, so sync has no+ // part in it. Seed the catalogue cache the way a first `ask` would.+ {+ let (bytes, _) = signed_catalogue(+ catalogue_entry(OLD.pin_sha, OLD.pin_date, &old_archive, &old_sha),+ &seed,+ );+ std::fs::write(hub_home.join("catalogue.json"), bytes).unwrap();+ }+ let rc = {+ let client = client.clone();+ tokio::task::spawn_blocking(move || {+ xerj_hub::cli::run_hub_cli(&[+ "pull".into(),+ "mini-c".into(),+ "--url".into(),+ client,+ "--quiet".into(),+ ])+ })+ .await+ .unwrap()+ };+ assert_eq!(rc, 0, "the older pin pulls cleanly");+ assert_eq!(pin_dirs(&hub_home), vec![OLD.pin8()]);+ let limit = |client: String| {+ tokio::task::spawn_blocking(move || {+ let node = HubNode::connect(&client).unwrap();+ let a = xerj_hub::def::lookup(&node, "mini-c", "MINI_LIMIT", 5);+ (+ a.hits.first().and_then(|h| h.value.clone()),+ a.pin_date.clone(),+ )+ })+ };+ assert_eq!(+ limit(client.clone()).await.unwrap(),+ (Some("128".to_string()), OLD.pin_date.to_string()),+ "the old pin answers 128"+ );++ // The shared ops factory: every sync below goes through the real command.+ let pulls = Arc::new(AtomicUsize::new(0));+ let around_pull = Arc::new(Mutex::new(Vec::new()));+ let run = {+ let (client, hub_home, code_home) = (client.clone(), hub_home.clone(), code_home.clone());+ let (pulls, around_pull) = (pulls.clone(), around_pull.clone());+ move || {+ let (client, hub_home, code_home) =+ (client.clone(), hub_home.clone(), code_home.clone());+ let (pulls, around_pull) = (pulls.clone(), around_pull.clone());+ tokio::task::spawn_blocking(move || {+ run_sync_using(&[], &move |_url| {+ Ok(Box::new(InProcessOps {+ node_url: client.clone(),+ hub_home: hub_home.clone(),+ code_home: code_home.clone(),+ pulls: pulls.clone(),+ around_pull: around_pull.clone(),+ }) as Box<dyn SyncOps>)+ })+ })+ }+ };++ // ── 3. The hub moves to the NEW pin; sync replaces the pack ───────────+ let (bytes, sig) = signed_catalogue(+ catalogue_entry(NEW.pin_sha, NEW.pin_date, &new_archive, &new_sha),+ &seed,+ );+ hub.serve(bytes, sig);+ assert!(+ !hub_home.join("last_sync").exists(),+ "precondition: sync has never run"+ );++ assert_eq!(run().await.unwrap(), 0, "a sync with a newer pin succeeds");++ assert_eq!(+ pin_dirs(&hub_home),+ vec![NEW.pin8()],+ "the new pin replaced the old one and the old directory is gone"+ );+ assert!(hub_home+ .join("packs/mini-c")+ .join(NEW.pin8())+ .join("pack.json")+ .is_file());+ assert_eq!(pulls.load(Ordering::SeqCst), 1, "exactly one pull");+ assert_eq!(+ *around_pull.lock().unwrap(),+ vec![(true, true)],+ "the old pin was on disk before the pull and STILL on disk when it returned: \+ it is removed only after the new pin has restored"+ );+ let ledger: serde_json::Value = serde_json::from_slice(+ &std::fs::read(code_home.join("state").join("mini-c.json")).unwrap(),+ )+ .unwrap();+ assert_eq!(ledger["source"], json!("hubpack"));+ assert_eq!(ledger["pin_sha"], json!(NEW.pin_sha));+ assert_eq!(ledger["index_prefix"], json!(NEW.prefix));+ assert_eq!(+ limit(client.clone()).await.unwrap(),+ (Some("256".to_string()), NEW.pin_date.to_string()),+ "the new pin is the one answering: the restore really ran"+ );+ let stamp = std::fs::read_to_string(hub_home.join("last_sync")).unwrap();+ chrono::DateTime::parse_from_rfc3339(stamp.trim()).expect("last_sync is RFC 3339");+ assert!(+ hub_home.join("catalogue.json").is_file(),+ "the refreshed catalogue was cached"+ );++ // ── 4. Nothing moved: a second sync pulls nothing ─────────────────────+ assert_eq!(run().await.unwrap(), 0);+ assert_eq!(+ pulls.load(Ordering::SeqCst),+ 1,+ "a pack already at the catalogue's pin costs no download"+ );+ assert_eq!(pin_dirs(&hub_home), vec![NEW.pin8()]);++ // ── 5. The hub advertises a pin whose archive is corrupt ──────────────+ age_file(&hub_home.join("last_sync"), 3 * 86_400);+ let before = std::fs::metadata(hub_home.join("last_sync"))+ .unwrap()+ .modified()+ .unwrap();+ let (bytes, sig) = signed_catalogue(+ catalogue_entry(BAD_SHA, "2026-10-16", &bad_archive, &new_sha),+ &seed,+ );+ hub.serve(bytes, sig);++ assert_eq!(+ run().await.unwrap(),+ 3,+ "a pack that fails verification is exit 3"+ );+ assert_eq!(+ pin_dirs(&hub_home),+ vec![NEW.pin8()],+ "nothing was extracted for the bad pin and the working pin was not touched"+ );+ assert!(!hub_home.join("packs/mini-c/c0ffee00.tmp").exists());+ assert_eq!(+ limit(client.clone()).await.unwrap().0,+ Some("256".to_string()),+ "the pin that works keeps answering"+ );+ assert_eq!(+ std::fs::metadata(hub_home.join("last_sync"))+ .unwrap()+ .modified()+ .unwrap(),+ before,+ "a failed sync does not claim to have synced"+ );++ // ── 6. A catalogue that does not verify is refused, and changes nothing ─+ let (good, good_sig) = signed_catalogue(+ catalogue_entry(OLD.pin_sha, OLD.pin_date, &old_archive, &old_sha),+ &seed,+ );+ let mut tampered = good.clone();+ let mid = tampered.len() / 2;+ tampered[mid] ^= 0x01;+ hub.serve(tampered, good_sig);+ let pulls_before = pulls.load(Ordering::SeqCst);++ assert_eq!(+ run().await.unwrap(),+ 1,+ "a catalogue whose signature fails is a refresh failure, exit 1"+ );+ assert_eq!(+ pulls.load(Ordering::SeqCst),+ pulls_before,+ "nothing was pulled"+ );+ assert_eq!(pin_dirs(&hub_home), vec![NEW.pin8()]);++ // Clean up so the env does not leak and the local hub thread ends.+ hub.shutdown();+ for k in [+ "XERJ_CODE_HOME",+ "XERJ_HUB_HOME",+ "XERJ_HUB_PUBKEY",+ "XERJ_HUB_URL",+ ] {+ std::env::remove_var(k);+ }+}