Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(hub): xerj hub sync and sync --install-timer (plan §8)

xerj-org3bfc6fd3cf09
diff --git a/engine/crates/xerj-hub/src/catalogue.rs b/engine/crates/xerj-hub/src/catalogue.rs
index 26de98dce..59c289cfb 100644
--- a/engine/crates/xerj-hub/src/catalogue.rs
+++ b/engine/crates/xerj-hub/src/catalogue.rs
@@ -364,6 +364,25 @@ pub fn load_catalogue(hub_url: &str, cache_dir: &Path, offline: bool) -> Result<
}
}
+/// Ask the hub for the catalogue NOW, ignoring the 24 h freshness window that
+/// [`load_catalogue`] honours. `xerj hub sync` calls this: a sync that trusted a
+/// young cache would never see a pin that moved this morning.
+///
+/// The request is still conditional (`If-None-Match` from `catalogue.etag`, so an
+/// unchanged catalogue costs a 304) and the detached signature is still verified
+/// BEFORE the fetched bytes replace the cache. Unlike [`load_catalogue`] it does
+/// not fall back to the cached copy on failure: the caller asked "is there
+/// something newer?", and answering from a stale copy would be a wrong "no". On
+/// any network, signature or parse failure the error is returned and the cache
+/// is left exactly as it was.
+pub fn refresh_catalogue(hub_url: &str, cache_dir: &Path) -> Result<Catalogue> {
+ fetch_and_store(
+ hub_url,
+ &cache_dir.join(CACHE_FILE),
+ &cache_dir.join(ETAG_FILE),
+ )
+}
+
/// Read and validate the cached catalogue, or `None` if it is missing,
/// unreadable, unparsable or a format version this client does not speak.
fn read_cached(cache_file: &Path) -> Option<Catalogue> {
@@ -887,6 +906,71 @@ mod tests {
assert_eq!(on_disk, good_bytes);
}
+ #[test]
+ fn refresh_ignores_a_young_cache_and_fetches_the_newer_catalogue() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ let fx = tempfile::tempdir().unwrap();
+ let inputs = two_inputs(fx.path());
+
+ // A YOUNG cache holding yesterday's catalogue: `load_catalogue` trusts it.
+ let (old_seed, _old_public) = keypair();
+ let (old_cat, old_sig) =
+ build_catalogue(&inputs, "2026-10-13T00:00:00Z", &old_seed).unwrap();
+ let old_bytes = canonical_json(&old_cat).unwrap();
+ let cache = tempfile::tempdir().unwrap();
+ seed_cache(cache.path(), &old_bytes, &old_sig);
+
+ // The hub now serves a newer catalogue, signed by the key the client trusts.
+ let (new_seed, new_public) = keypair();
+ let (new_cat, new_sig) =
+ build_catalogue(&inputs, "2026-10-14T00:00:00Z", &new_seed).unwrap();
+ let new_bytes = canonical_json(&new_cat).unwrap();
+ let (url, handle) = spawn_hub(new_bytes.clone(), new_sig);
+
+ std::env::set_var("XERJ_HUB_PUBKEY", &new_public);
+ let trusted = load_catalogue(&url, cache.path(), false).expect("young cache is served");
+ assert_eq!(
+ trusted.generated_at, "2026-10-13T00:00:00Z",
+ "load_catalogue trusts a young cache without asking the hub"
+ );
+ let refreshed = refresh_catalogue(&url, cache.path()).expect("forced refresh");
+ std::env::remove_var("XERJ_HUB_PUBKEY");
+ handle.join().unwrap();
+
+ assert_eq!(
+ refreshed.generated_at, "2026-10-14T00:00:00Z",
+ "refresh_catalogue asked the hub despite the young cache"
+ );
+ let on_disk = fs::read(cache.path().join(CACHE_FILE)).unwrap();
+ assert_eq!(on_disk, new_bytes, "the verified bytes replaced the cache");
+ }
+
+ #[test]
+ fn refresh_reports_a_tampered_catalogue_instead_of_serving_the_cache() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ let fx = tempfile::tempdir().unwrap();
+ let (good_bytes, good_sig, public) = fixture_bytes(fx.path());
+ let cache = tempfile::tempdir().unwrap();
+ seed_cache(cache.path(), &good_bytes, &good_sig);
+
+ let mut tampered = good_bytes.clone();
+ tampered[good_bytes.len() / 2] ^= 0x01;
+ let (url, handle) = spawn_hub(tampered, good_sig.clone());
+
+ std::env::set_var("XERJ_HUB_PUBKEY", &public);
+ let err = refresh_catalogue(&url, cache.path())
+ .expect_err("a refresh that cannot be verified is an error, not a stale answer");
+ std::env::remove_var("XERJ_HUB_PUBKEY");
+ handle.join().unwrap();
+
+ assert!(
+ format!("{err:#}").contains("signature"),
+ "the error names the signature check: {err:#}"
+ );
+ let on_disk = fs::read(cache.path().join(CACHE_FILE)).unwrap();
+ assert_eq!(on_disk, good_bytes, "the cache is untouched");
+ }
+
#[test]
fn load_without_a_key_skips_verification_rather_than_failing() {
let _guard = ENV_LOCK.lock().unwrap();
diff --git a/engine/crates/xerj-hub/src/cli.rs b/engine/crates/xerj-hub/src/cli.rs
index 6f4d5022f..ba925fc12 100644
--- a/engine/crates/xerj-hub/src/cli.rs
+++ b/engine/crates/xerj-hub/src/cli.rs
@@ -42,6 +42,9 @@ const HUB_USAGE: &str = "usage: xerj hub <command>
build-pack <slug> --registry <dir> --out <dir> --url <node> --sign-key <seed-file>
Build a signed, pre-indexed pack from a registry manifest (CI).
+ sync [--url <node>] | sync --install-timer
+ Refresh the catalogue and bring every cached pack up to its pin;
+ --install-timer schedules that every 6 hours (see `xerj hub sync -h`).
pull <slug> [--quiet] [--url <node>]
Download, verify and restore a pack, then `xerj code <slug>` works.
@@ -57,6 +60,7 @@ pub fn run_hub_cli(args: &[String]) -> i32 {
let rest = &args[1..];
match sub.as_str() {
"build-pack" => run_build_pack(rest),
+ "sync" => crate::sync::run_sync(rest),
"pull" => run_pull(rest),
"-h" | "--help" | "help" => {
println!("{HUB_USAGE}");
diff --git a/engine/crates/xerj-hub/src/lib.rs b/engine/crates/xerj-hub/src/lib.rs
index e46f16250..f6175a75c 100644
--- a/engine/crates/xerj-hub/src/lib.rs
+++ b/engine/crates/xerj-hub/src/lib.rs
@@ -21,6 +21,7 @@ pub mod node;
pub mod pack;
pub mod router;
pub mod sketch;
+pub mod sync;
pub use xerj_common::hub::{
render, Answer, AnswerMode, AnsweredFrom, Catalogue, CatalogueEntry, DefRecord, Hit,
diff --git a/engine/crates/xerj-hub/src/lockfile.rs b/engine/crates/xerj-hub/src/lockfile.rs
index f8be0cd3f..d07f386ff 100644
--- a/engine/crates/xerj-hub/src/lockfile.rs
+++ b/engine/crates/xerj-hub/src/lockfile.rs
@@ -14,9 +14,9 @@
//! - the `xerj hub status` data model + plain-text renderer
//! ([`gather_status`] / [`render_status`]).
//!
-//! `xerj hub sync` is **DEFERRED (step 8b)**: it needs the step-2 pull/restore
-//! path another agent is building, so [`sync`] returns an explanatory error
-//! rather than faking a no-op. See its doc comment.
+//! `xerj hub sync` is not here: it drives the pull path, so it lives next to the
+//! other subcommands that do (`crate::sync`). This module only owns the on-disk
+//! layout names it shares (`packs/`, `pack.json`, `last_used`, `last_sync`).
use std::fs;
use std::path::{Path, PathBuf};
@@ -238,23 +238,34 @@ fn question_asks_for_current(question: &str) -> bool {
// xerj hub status
// ---------------------------------------------------------------------------
-const PACKS_DIR: &str = "packs";
-const PACK_MANIFEST_FILE: &str = "pack.json";
-const LAST_USED_FILE: &str = "last_used";
+pub(crate) const PACKS_DIR: &str = "packs";
+pub(crate) const PACK_MANIFEST_FILE: &str = "pack.json";
+pub(crate) const LAST_USED_FILE: &str = "last_used";
const NODE_PID_FILE: &str = "node.pid";
const NODE_PORT_FILE: &str = "node.port";
const CONFIG_FILE: &str = "config.toml";
-const LAST_SYNC_FILE: &str = "last_sync";
+pub(crate) const LAST_SYNC_FILE: &str = "last_sync";
const CATALOGUE_FILE: &str = "catalogue.json";
/// The default value reported for the `hosted` setting when `config.toml` is
/// absent or does not set it.
const DEFAULT_HOSTED: &str = "auto";
-/// The default hub root, `~/.xerj/hub`. `status` takes the root as a parameter
-/// (so tests inject a temp dir); callers that want the real location pass this.
+/// The default hub root: `XERJ_HUB_HOME` when set (the override every `xerj hub`
+/// subcommand honours), else `~/.xerj/hub`. `status` takes the root as a
+/// parameter (so tests inject a temp dir); callers that want the real location
+/// pass this.
pub fn default_hub_root() -> PathBuf {
- let home = std::env::var_os("HOME")
+ hub_root_from(&|key| std::env::var_os(key))
+}
+
+/// [`default_hub_root`] over an injectable environment, so the rule is testable
+/// without mutating the process-global one.
+pub(crate) fn hub_root_from(env: &dyn Fn(&str) -> Option<std::ffi::OsString>) -> PathBuf {
+ if let Some(h) = env("XERJ_HUB_HOME").filter(|h| !h.is_empty()) {
+ return PathBuf::from(h);
+ }
+ let home = env("HOME")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("."));
home.join(".xerj").join("hub")
@@ -443,26 +454,6 @@ pub fn render_status(status: &HubStatus) -> String {
lines.join("\n")
}
-// ---------------------------------------------------------------------------
-// xerj hub sync — DEFERRED (step 8b): needs step-2 pull
-// ---------------------------------------------------------------------------
-
-/// `xerj hub sync` — **DEFERRED (step 8b): needs step-2 pull**.
-///
-/// The full sync (refresh the catalogue; for every cached pack whose catalogue
-/// pin is newer, pull the new pin and remove the old dir after the new one is
-/// restored; verify signatures; prune to the disk budget; write
-/// `~/.xerj/hub/last_sync`) all hinges on the step-2 pull/restore path another
-/// agent is building. Rather than fake a no-op sync, this returns an error that
-/// names the dependency, so a caller cannot mistake "nothing happened" for a
-/// successful refresh. Wire the real body here once `pack::pull` lands.
-pub fn sync(_hub_root: &Path) -> Result<()> {
- anyhow::bail!(
- "xerj hub sync is not implemented yet: it depends on the step-2 pull/restore path \
- (deferred to step 8b)"
- )
-}
-
// ---------------------------------------------------------------------------
// small helpers
// ---------------------------------------------------------------------------
@@ -480,7 +471,7 @@ fn read_trimmed(path: &Path) -> Option<String> {
/// Total size in bytes of a directory tree (files only; symlinks skipped so a
/// cycle cannot hang the walk).
-fn dir_size(dir: &Path) -> u64 {
+pub(crate) fn dir_size(dir: &Path) -> u64 {
let mut total = 0;
let Ok(entries) = fs::read_dir(dir) else {
return 0;
@@ -539,7 +530,7 @@ fn days_ago(age: Option<i64>) -> String {
}
/// Human-readable byte size, e.g. `0 B`, `512 B`, `2.0 MB`.
-fn human_bytes(bytes: u64) -> String {
+pub(crate) fn human_bytes(bytes: u64) -> String {
const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"];
let mut v = bytes as f64;
let mut unit = 0;
@@ -868,10 +859,33 @@ mod tests {
}
#[test]
- fn sync_is_deferred() {
- let tmp = tempdir().unwrap();
- let err = sync(tmp.path()).unwrap_err().to_string();
- assert!(err.contains("step-2"), "err was: {err}");
+ fn hub_root_honours_the_override_then_home() {
+ use std::ffi::OsString;
+ let env = |pairs: &'static [(&'static str, &'static str)]| {
+ move |k: &str| -> Option<OsString> {
+ pairs
+ .iter()
+ .find(|(name, _)| *name == k)
+ .map(|(_, v)| OsString::from(*v))
+ }
+ };
+ // XERJ_HUB_HOME wins — the same override `xerj hub pull` honours, so a
+ // sync and the pulls it drives always look at one cache.
+ assert_eq!(
+ hub_root_from(&env(&[("XERJ_HUB_HOME", "/srv/hub"), ("HOME", "/home/u")])),
+ PathBuf::from("/srv/hub")
+ );
+ // An EMPTY override is not an override.
+ assert_eq!(
+ hub_root_from(&env(&[("XERJ_HUB_HOME", ""), ("HOME", "/home/u")])),
+ PathBuf::from("/home/u/.xerj/hub")
+ );
+ assert_eq!(
+ hub_root_from(&env(&[("HOME", "/home/u")])),
+ PathBuf::from("/home/u/.xerj/hub")
+ );
+ // No HOME at all degrades to a relative path, never a panic.
+ assert_eq!(hub_root_from(&env(&[])), PathBuf::from("./.xerj/hub"));
}
// ---- small helpers ---------------------------------------------------
diff --git a/engine/crates/xerj-hub/src/sync.rs b/engine/crates/xerj-hub/src/sync.rs
new file mode 100644
index 000000000..fbfa36090
--- /dev/null
+++ b/engine/crates/xerj-hub/src/sync.rs
@@ -0,0 +1,1195 @@
+//! Corpus Hub v2 — `xerj hub sync` and `xerj hub sync --install-timer` (plan §8).
+//!
+//! `sync` keeps the pack cache (`~/.xerj/hub/packs/<slug>/<pin8>/`) in step with
+//! the catalogue, in this order:
+//!
+//! 1. **Refresh the catalogue** ([`crate::catalogue::refresh_catalogue`]): a
+//! conditional GET that ignores the 24 h cache window, with the detached
+//! signature verified before the cached copy is replaced. If the hub cannot be
+//! reached or the signature does not verify, sync stops and changes nothing.
+//! 2. **Update every cached pack whose catalogue pin moved.** The new pin is
+//! pulled by `xerj hub pull <slug>` — the existing pull path, run as a child
+//! process of this same binary — so the archive's sha256 and its ed25519
+//! signature are verified before a byte is extracted, exactly as for a first
+//! pull. The OLD pin directory is removed only after the pull exited 0 and
+//! the new directory is complete and recorded in the state ledger; a failed
+//! pull (network, verification, restore) leaves the old pin in place and
+//! serving.
+//! 3. **Prune to the disk budget**, least recently used first. Packs updated in
+//! this run are not candidates (they were just fetched), and an update keeps
+//! the old pin's `last_used` recency: refreshing a pack is not using it.
+//! 4. **Write `last_sync`** (only when nothing failed), the file `xerj hub status`
+//! reads for "last sync".
+//!
+//! Only extracted pack directories are removed. The restored indices live in the
+//! node's own data directory and are not touched here — deleting indices from a
+//! node is not sync's business (the same rule the pull-time eviction follows).
+//!
+//! The pure parts — [`scan_cache`], [`plan_slug`], [`prune_to_budget`], the unit
+//! renderers — take plain arguments, and the parts that touch the outside world
+//! (pulling, reading the ledger) sit behind [`SyncOps`], so the tests drive the
+//! whole flow with throwaway directories and no network, and `tests/sync_e2e.rs`
+//! drives it against real nodes and the real pull.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::ffi::OsString;
+use std::fs;
+use std::path::{Component, Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use anyhow::{anyhow, bail, Context, Result};
+use chrono::NaiveDate;
+use xerj_common::hub::{
+ Catalogue, CatalogueEntry, LicenceClass, PackManifest, AUTO_PULL_MAX_BYTES, DEFAULT_HUB_URL,
+ DISK_BUDGET_BYTES,
+};
+use xerj_common::xccode::state;
+
+use crate::catalogue::refresh_catalogue;
+use crate::lockfile::{
+ default_hub_root, dir_size, hub_root_from, human_bytes, LAST_SYNC_FILE, LAST_USED_FILE,
+ PACKS_DIR, PACK_MANIFEST_FILE,
+};
+
+/// Exit codes of `xerj hub sync`. 3 is "a pack failed verification", kept apart
+/// from 1 ("the machine or network got in the way") for the same reason
+/// `xerj hub pull` keeps them apart.
+const EXIT_OK: i32 = 0;
+const EXIT_INFRA: i32 = 1;
+const EXIT_USAGE: i32 = 2;
+const EXIT_VERIFY: i32 = 3;
+
+/// `xerj hub sync -h`. A function, not a constant, so the budget it names is the
+/// one the code enforces.
+fn sync_usage() -> String {
+ format!(
+ "usage: xerj hub sync [--url <node>]
+ xerj hub sync --install-timer
+
+Refresh the catalogue, then bring every cached pack up to the catalogue's pin:
+the new pin is pulled by `xerj hub pull` (sha256 and signature are verified
+before anything is extracted), the old pin's directory is removed only after the
+new one is restored, and the cache is pruned to the {budget} disk budget, least
+recently used first. `~/.xerj/hub/last_sync` is written when nothing failed.
+
+ --url <node> the node the packs are restored into (passed to `hub pull`)
+ --install-timer write a systemd user timer (Linux) or a launchd agent
+ (macOS) that runs `xerj hub sync` every 6 hours, and print
+ its path. Nothing is enabled, loaded or started: the
+ comment at the top of the written file says how.
+
+exit codes: 0 synced (or nothing to do); 1 the catalogue could not be refreshed
+or a pack could not be updated (its old pin is kept); 2 usage error; 3 a pack
+failed verification (its old pin is kept)",
+ budget = human_bytes(DISK_BUDGET_BYTES)
+ )
+}
+
+// ── the cache on disk ────────────────────────────────────────────────────────
+
+/// One extracted pack: `<hub>/packs/<slug>/<pin8>/`.
+#[derive(Debug, Clone)]
+pub struct CachedPin {
+ pub slug: String,
+ /// The directory name: the first 8 hex characters of the pack's `pin_sha`.
+ pub pin8: String,
+ pub dir: PathBuf,
+ /// `pin_date` from the pack's own `pack.json`, when it parses.
+ pub pin_date: Option<String>,
+ /// `pack.json` parses and every file `SHA256SUMS` names is on disk. A
+ /// directory that fails this is a half-extracted leftover, not a pack.
+ pub complete: bool,
+ /// Recency for LRU: the mtime of the pack's `last_used` file, else of the
+ /// directory itself (the same rule the pull-time eviction uses).
+ pub last_used: SystemTime,
+}
+
+/// `^[0-9a-f]{8}$` — what a pin directory is called. Anything else under a slug
+/// directory (`<pin8>.tmp`, `<pin8>.lock`, stray files) is not a pack.
+fn is_pin8(name: &str) -> bool {
+ name.len() == 8 && name.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
+}
+
+/// First 8 hex characters of a commit, lower-cased — the `pin8` in a pack name.
+/// `None` for anything that is not at least 8 hex characters.
+fn pin8_of(pin_sha: &str) -> Option<String> {
+ let s = pin_sha.trim();
+ (s.len() >= 8 && s.bytes().all(|b| b.is_ascii_hexdigit())).then(|| s[..8].to_ascii_lowercase())
+}
+
+/// Every pin directory under `<hub>/packs`, sorted by slug then pin. Cheap: no
+/// directory is walked for its size (only the files each `SHA256SUMS` names are
+/// stat'ed). Symlinks are never followed.
+pub fn scan_cache(hub_home: &Path) -> Vec<CachedPin> {
+ let mut out = Vec::new();
+ let Ok(slugs) = fs::read_dir(hub_home.join(PACKS_DIR)) else {
+ return out;
+ };
+ for slug_ent in slugs.flatten() {
+ if !slug_ent.file_type().is_ok_and(|t| t.is_dir()) {
+ continue;
+ }
+ let slug = slug_ent.file_name().to_string_lossy().into_owned();
+ let Ok(pins) = fs::read_dir(slug_ent.path()) else {
+ continue;
+ };
+ for pin_ent in pins.flatten() {
+ let pin8 = pin_ent.file_name().to_string_lossy().into_owned();
+ if !pin_ent.file_type().is_ok_and(|t| t.is_dir()) || !is_pin8(&pin8) {
+ continue;
+ }
+ let dir = pin_ent.path();
+ let mtime = |p: &Path| fs::metadata(p).and_then(|m| m.modified()).ok();
+ let manifest = read_pack_manifest(&dir);
+ out.push(CachedPin {
+ slug: slug.clone(),
+ complete: manifest.is_some() && every_listed_file_exists(&dir),
+ pin_date: manifest.map(|m| m.pin_date),
+ last_used: mtime(&dir.join(LAST_USED_FILE))
+ .or_else(|| mtime(&dir))
+ .unwrap_or(UNIX_EPOCH),
+ pin8,
+ dir,
+ });
+ }
+ }
+ out.sort_by(|a, b| a.slug.cmp(&b.slug).then_with(|| a.pin8.cmp(&b.pin8)));
+ out
+}
+
+fn read_pack_manifest(dir: &Path) -> Option<PackManifest> {
+ let raw = fs::read(dir.join(PACK_MANIFEST_FILE)).ok()?;
+ serde_json::from_slice(&raw).ok()
+}
+
+/// Every path `SHA256SUMS` lists exists under `dir`. The extractor writes
+/// `pack.json` before the snapshot files, so a parsing `pack.json` alone does not
+/// prove the extraction finished; the listing does. Existence only — re-hashing
+/// hundreds of MB every six hours would cost more than it protects. A path that
+/// is absolute or climbs out of `dir` makes the listing, and so the pack, invalid.
+fn every_listed_file_exists(dir: &Path) -> bool {
+ let Ok(sums) = fs::read_to_string(dir.join("SHA256SUMS")) else {
+ return false;
+ };
+ sums.lines().filter(|l| !l.trim().is_empty()).all(|line| {
+ let Some((_, rel)) = line.split_once(" ") else {
+ return false;
+ };
+ let rel = Path::new(rel);
+ rel.components().all(|c| matches!(c, Component::Normal(_))) && dir.join(rel).is_file()
+ })
+}
+
+// ── what to do about one slug ────────────────────────────────────────────────
+
+/// What the state ledger (`<code root>/state/<slug>.json`) says about a slug.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Ledger {
+ /// No readable ledger: the pack was never fully restored, or the ledger was
+ /// deleted.
+ Missing,
+ /// The slug is served from a LOCAL index (`xerj corpus index`), not a hub
+ /// pack. Sync does not flip that choice back behind the user's back.
+ Local,
+ /// A hub pack was restored; `pin8` is the first 8 hex of its `pin_sha`.
+ Hubpack { pin8: Option<String> },
+}
+
+/// What [`run_sync_with`] decided for one slug.
+#[derive(Debug, Clone)]
+pub enum Verdict {
+ /// The catalogue pin is cached, complete and restored. `superseded` are
+ /// older pin directories still lying around, now safe to remove.
+ Current { superseded: Vec<CachedPin> },
+ /// Pull `target`; remove `old` afterwards, and only then.
+ Pull {
+ target: String,
+ old: Vec<CachedPin>,
+ why: PullWhy,
+ },
+ /// Leave the slug alone and say why.
+ Keep(KeepWhy),
+}
+
+/// Why a pull is needed.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PullWhy {
+ /// The catalogue's pin is not cached.
+ NewPin,
+ /// The pin directory is there but the ledger does not record it as restored
+ /// (an interrupted pull, a deleted ledger).
+ NotRestored,
+ /// The pin directory is there but incomplete.
+ Damaged,
+}
+
+/// Why a slug was left alone.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum KeepWhy {
+ NotInCatalogue,
+ /// The catalogue's `pin_date` is older than the newest cached pin's: a
+ /// replayed or rolled-back catalogue must not downgrade a cache.
+ CatalogueOlder {
+ catalogue: String,
+ cached: String,
+ },
+ /// The catalogue now classes the corpus `excluded`; it is never pulled.
+ Excluded,
+ /// The new pack is over `AUTO_PULL_MAX_BYTES`; sync never auto-pulls those.
+ OverCap {
+ bytes: u64,
+ },
+ /// The catalogue's `pin_sha` is not a hex commit.
+ BadPin(String),
+ /// Served from a local index ([`Ledger::Local`]).
+ LocalIndex,
+}
+
+impl KeepWhy {
+ fn describe(&self) -> String {
+ match self {
+ KeepWhy::NotInCatalogue => "not in the catalogue".into(),
+ KeepWhy::CatalogueOlder { catalogue, cached } => {
+ format!("the catalogue's pin ({catalogue}) is older than the cached one ({cached})")
+ }
+ KeepWhy::Excluded => {
+ "the catalogue now classes it excluded, so no new pin is pulled".into()
+ }
+ KeepWhy::OverCap { bytes } => format!(
+ "its new pack is {} bytes, over the {}-byte auto-pull cap",
+ bytes, AUTO_PULL_MAX_BYTES
+ ),
+ KeepWhy::BadPin(p) => format!("the catalogue's pin '{p}' is not a hex commit"),
+ KeepWhy::LocalIndex => {
+ "it is served from a local index (`xerj corpus index`), not a hub pack".into()
+ }
+ }
+ }
+}
+
+fn parse_day(s: &str) -> Option<NaiveDate> {
+ NaiveDate::parse_from_str(s.trim(), "%Y-%m-%d").ok()
+}
+
+/// Decide what to do about one slug. Pure: `entry` is the catalogue's entry for
+/// the slug (if any), `cached` its pin directories on disk (never empty), and
+/// `ledger` what the state ledger records.
+pub fn plan_slug(entry: Option<&CatalogueEntry>, cached: &[CachedPin], ledger: &Ledger) -> Verdict {
+ let Some(entry) = entry else {
+ return Verdict::Keep(KeepWhy::NotInCatalogue);
+ };
+ let Some(target) = pin8_of(&entry.pin_sha) else {
+ return Verdict::Keep(KeepWhy::BadPin(entry.pin_sha.clone()));
+ };
+ if *ledger == Ledger::Local {
+ return Verdict::Keep(KeepWhy::LocalIndex);
+ }
+
+ let others: Vec<CachedPin> = cached
+ .iter()
+ .filter(|c| c.pin8 != target)
+ .cloned()
+ .collect();
+ let have = cached.iter().find(|c| c.pin8 == target);
+ let restored = matches!(ledger, Ledger::Hubpack { pin8: Some(p) } if *p == target);
+
+ if let Some(have) = have {
+ if have.complete && restored {
+ return Verdict::Current { superseded: others };
+ }
+ }
+
+ // A pull is needed. Refuse what sync must never auto-pull, and refuse to go
+ // BACKWARDS in time, before spending a download on it.
+ if entry.licence_class == LicenceClass::Excluded {
+ return Verdict::Keep(KeepWhy::Excluded);
+ }
+ if let (Some(cat_day), Some(cached_day)) = (
+ parse_day(&entry.pin_date),
+ others
+ .iter()
+ .filter_map(|c| c.pin_date.as_deref().and_then(parse_day))
+ .max(),
+ ) {
+ if cat_day < cached_day {
+ return Verdict::Keep(KeepWhy::CatalogueOlder {
+ catalogue: entry.pin_date.clone(),
+ cached: cached_day.format("%Y-%m-%d").to_string(),
+ });
+ }
+ }
+ if entry.pack_bytes > AUTO_PULL_MAX_BYTES {
+ return Verdict::Keep(KeepWhy::OverCap {
+ bytes: entry.pack_bytes,
+ });
+ }
+
+ let why = match have {
+ None => PullWhy::NewPin,
+ Some(h) if !h.complete => PullWhy::Damaged,
+ Some(_) => PullWhy::NotRestored,
+ };
+ Verdict::Pull {
+ target,
+ old: others,
+ why,
+ }
+}
+
+// ── the outside world ────────────────────────────────────────────────────────
+
+/// What a pull attempt came to.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum PullOutcome {
+ Done,
+ /// The pull failed with `code` (the exit code of `xerj hub pull`: 3 means
+ /// the pack failed verification, anything else is the machine or network).
+ Failed {
+ code: i32,
+ message: String,
+ },
+}
+
+/// The two things sync needs from the outside world. The production
+/// implementation is [`CliOps`]; tests substitute their own.
+pub trait SyncOps {
+ /// Pull `slug`'s catalogue pin through the verified pull path.
+ fn pull(&self, slug: &str) -> PullOutcome;
+ /// What the state ledger records for `slug`.
+ fn ledger(&self, slug: &str) -> Ledger;
+}
+
+/// Runs `xerj hub pull <slug> --quiet [--url <node>]` as a child process of this
+/// same binary and reads the ledger the pull writes.
+///
+/// A child process, not a function call, so sync gets whatever the pull
+/// subcommand does — node selection, locks, the size cap, signature
+/// verification — without re-implementing any of it, and so a change to the
+/// pull path cannot silently diverge from what sync runs.
+pub struct CliOps {
+ program: PathBuf,
+ /// Arguments placed before `hub pull …` (empty for the real binary).
+ pre_args: Vec<OsString>,
+ url: Option<String>,
+ code_root: PathBuf,
+}
+
+impl CliOps {
+ /// The running binary, with the ledger at `XERJ_CODE_HOME` (else
+ /// `~/.xerj-code`) — the same place `hub pull` writes it.
+ pub fn current_exe(url: Option<String>) -> Result<CliOps> {
+ let program = std::env::current_exe().context("cannot locate the running xerj binary")?;
+ Ok(CliOps {
+ program,
+ pre_args: Vec::new(),
+ url,
+ code_root: xerj_autoindex::xc::code_root(),
+ })
+ }
+
+ /// Any program that answers `hub pull` — the unit tests point this at a
+ /// shell script.
+ pub fn with_program(
+ program: impl Into<PathBuf>,
+ pre_args: Vec<OsString>,
+ url: Option<String>,
+ code_root: PathBuf,
+ ) -> CliOps {
+ CliOps {
+ program: program.into(),
+ pre_args,
+ url,
+ code_root,
+ }
+ }
+}
+
+impl SyncOps for CliOps {
+ fn pull(&self, slug: &str) -> PullOutcome {
+ let mut cmd = Command::new(&self.program);
+ cmd.args(&self.pre_args)
+ .args(["hub", "pull"])
+ .arg(slug)
+ .arg("--quiet");
+ if let Some(url) = &self.url {
+ cmd.arg("--url").arg(url);
+ }
+ cmd.stdin(Stdio::null());
+ match cmd.output() {
+ Ok(out) if out.status.success() => PullOutcome::Done,
+ Ok(out) => PullOutcome::Failed {
+ // No code means the child was killed by a signal.
+ code: out.status.code().unwrap_or(EXIT_INFRA),
+ message: one_line(&String::from_utf8_lossy(&out.stderr), 400),
+ },
+ Err(e) => PullOutcome::Failed {
+ code: EXIT_INFRA,
+ message: format!("cannot run {}: {e}", self.program.display()),
+ },
+ }
+ }
+
+ fn ledger(&self, slug: &str) -> Ledger {
+ match state::load_state(&self.code_root, slug) {
+ Err(_) => Ledger::Missing,
+ Ok(st) if !st.is_hubpack() => Ledger::Local,
+ Ok(st) => Ledger::Hubpack {
+ pin8: st.pin_sha.as_deref().and_then(pin8_of),
+ },
+ }
+ }
+}
+
+/// Collapse whitespace (including newlines) to single spaces and cap the length,
+/// so a child's stderr becomes one readable line.
+fn one_line(s: &str, max: usize) -> String {
+ let joined = s.split_whitespace().collect::<Vec<_>>().join(" ");
+ if joined.chars().count() <= max {
+ joined
+ } else {
+ let mut cut: String = joined.chars().take(max).collect();
+ cut.push('…');
+ cut
+ }
+}
+
+// ── the sync itself ──────────────────────────────────────────────────────────
+
+/// Where a progress line goes: stdout (`Info`) or stderr (`Warn`).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Level {
+ Info,
+ Warn,
+}
+
+/// One pack that could not be updated.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SyncFailure {
+ pub slug: String,
+ pub code: i32,
+ pub message: String,
+}
+
+/// A pack directory the budget pass removed.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Pruned {
+ pub slug: String,
+ pub pin8: String,
+ pub bytes: u64,
+}
+
+/// The outcome of one sync, slug by slug.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct SyncReport {
+ pub updated: Vec<String>,
+ pub current: Vec<String>,
+ pub kept: Vec<String>,
+ pub failed: Vec<SyncFailure>,
+ pub pruned: Vec<Pruned>,
+}
+
+impl SyncReport {
+ /// 3 when any pack failed verification, else 1 when any pack failed, else 0.
+ pub fn exit_code(&self) -> i32 {
+ if self.failed.iter().any(|f| f.code == EXIT_VERIFY) {
+ EXIT_VERIFY
+ } else if !self.failed.is_empty() {
+ EXIT_INFRA
+ } else {
+ EXIT_OK
+ }
+ }
+}
+
+fn pin_list(pins: &[CachedPin]) -> String {
+ pins.iter()
+ .map(|p| p.pin8.as_str())
+ .collect::<Vec<_>>()
+ .join(",")
+}
+
+/// Sync `hub_home` against `cat`: update, then prune to `budget` bytes. Progress
+/// goes to `emit` line by line as it happens, so a long sync is never silent.
+pub fn run_sync_with(
+ hub_home: &Path,
+ cat: &Catalogue,
+ ops: &dyn SyncOps,
+ budget: u64,
+ emit: &mut dyn FnMut(Level, &str),
+) -> SyncReport {
+ let mut report = SyncReport::default();
+ let mut by_slug: BTreeMap<String, Vec<CachedPin>> = BTreeMap::new();
+ for pin in scan_cache(hub_home) {
+ by_slug.entry(pin.slug.clone()).or_default().push(pin);
+ }
+ if by_slug.is_empty() {
+ emit(Level::Info, "hub sync: no cached packs to update");
+ }
+
+ // Slugs this run fetched: not candidates for the budget pass below.
+ let mut fresh: BTreeSet<String> = BTreeSet::new();
+
+ for (slug, pins) in &by_slug {
+ let entry = cat.corpora.iter().find(|c| &c.slug == slug);
+ match plan_slug(entry, pins, &ops.ledger(slug)) {
+ Verdict::Current { superseded } => {
+ let target = entry.and_then(|e| pin8_of(&e.pin_sha)).unwrap_or_default();
+ if superseded.is_empty() {
+ emit(Level::Info, &format!("hub sync: {slug} {target}: current"));
+ } else if remove_pins(&superseded).len() == superseded.len() {
+ emit(
+ Level::Info,
+ &format!(
+ "hub sync: {slug} {target}: current; removed superseded pin {}",
+ pin_list(&superseded)
+ ),
+ );
+ } else {
+ emit(
+ Level::Warn,
+ &format!(
+ "hub sync: {slug} {target}: current; could not remove superseded pin {}",
+ pin_list(&superseded)
+ ),
+ );
+ }
+ report.current.push(slug.clone());
+ }
+ Verdict::Keep(why) => {
+ emit(
+ Level::Info,
+ &format!(
+ "hub sync: {slug} {}: kept ({})",
+ pin_list(pins),
+ why.describe()
+ ),
+ );
+ report.kept.push(slug.clone());
+ }
+ Verdict::Pull { target, old, why } => {
+ let from = if old.is_empty() {
+ String::new()
+ } else {
+ format!("{} -> ", pin_list(&old))
+ };
+ match pull_and_replace(hub_home, slug, &target, &old, ops) {
+ Ok(()) => {
+ let what = match why {
+ PullWhy::NewPin => "updated",
+ PullWhy::NotRestored => {
+ "re-pulled (the pin was cached but not restored)"
+ }
+ PullWhy::Damaged => "re-pulled (the cached pin was incomplete)",
+ };
+ let tail = if old.is_empty() {
+ String::new()
+ } else {
+ "; old pin removed".to_string()
+ };
+ emit(
+ Level::Info,
+ &format!("hub sync: {slug} {from}{target}: {what}{tail}"),
+ );
+ fresh.insert(slug.clone());
+ report.updated.push(slug.clone());
+ }
+ Err((code, message)) => {
+ let kept = if old.is_empty() {
+ String::new()
+ } else {
+ format!("; old pin {} kept", pin_list(&old))
+ };
+ emit(
+ Level::Warn,
+ &format!(
+ "hub sync: {slug} {from}{target}: FAILED (exit {code}: {message}){kept}"
+ ),
+ );
+ report.failed.push(SyncFailure {
+ slug: slug.clone(),
+ code,
+ message,
+ });
+ }
+ }
+ }
+ }
+ }
+
+ let (pruned, total) = prune_to_budget(hub_home, budget, &fresh);
+ for p in &pruned {
+ emit(
+ Level::Info,
+ &format!(
+ "hub sync: pruned {} {} ({}, least recently used) to stay under the {} disk budget",
+ p.slug,
+ p.pin8,
+ human_bytes(p.bytes),
+ human_bytes(budget)
+ ),
+ );
+ }
+ if total > budget {
+ emit(
+ Level::Warn,
+ &format!(
+ "hub sync: the cache is {} over the {} disk budget and nothing more can be \
+ pruned (what is left was updated in this run, or could not be removed)",
+ human_bytes(total - budget),
+ human_bytes(budget)
+ ),
+ );
+ }
+ report.pruned = pruned;
+
+ emit(
+ Level::Info,
+ &format!(
+ "hub sync: {} updated, {} current, {} kept, {} failed, {} pruned",
+ report.updated.len(),
+ report.current.len(),
+ report.kept.len(),
+ report.failed.len(),
+ report.pruned.len()
+ ),
+ );
+ report
+}
+
+/// Pull `slug`'s catalogue pin and, only if that left the new pin complete and
+/// recorded as restored, remove the `old` pin directories. On any failure the old
+/// pins are untouched. The error is `(exit code, message)`.
+fn pull_and_replace(
+ hub_home: &Path,
+ slug: &str,
+ target: &str,
+ old: &[CachedPin],
+ ops: &dyn SyncOps,
+) -> std::result::Result<(), (i32, String)> {
+ if let PullOutcome::Failed { code, message } = ops.pull(slug) {
+ return Err((code, message));
+ }
+
+ // `hub pull` exited 0. Trust, but check what "restored" is supposed to mean
+ // before the only irreversible step: the new pin's directory is complete,
+ // and the ledger — written only after the restore — names it.
+ let landed = scan_cache(hub_home)
+ .into_iter()
+ .find(|c| c.slug == slug && c.pin8 == target);
+ if !landed.as_ref().is_some_and(|c| c.complete) {
+ return Err((
+ EXIT_INFRA,
+ format!("`hub pull` reported success but {slug}/{target} is missing or incomplete"),
+ ));
+ }
+ match ops.ledger(slug) {
+ Ledger::Hubpack { pin8: Some(p) } if p == target => {}
+ other => {
+ return Err((
+ EXIT_INFRA,
+ format!(
+ "`hub pull` reported success but the ledger does not record {target} as \
+ restored ({other:?})"
+ ),
+ ))
+ }
+ }
+
+ carry_last_used(old, landed.as_ref().map(|c| c.dir.as_path()));
+ let stuck: Vec<String> = old
+ .iter()
+ .filter_map(|p| {
+ remove_pin_dir(&p.dir)
+ .err()
+ .map(|e| format!("{}: {e}", p.pin8))
+ })
+ .collect();
+ if stuck.is_empty() {
+ Ok(())
+ } else {
+ Err((
+ EXIT_INFRA,
+ format!(
+ "the new pin is in place but the old pin could not be removed ({})",
+ stuck.join("; ")
+ ),
+ ))
+ }
+}
+
+/// Remove one pin directory. One that is already gone — a pull-time eviction
+/// running beside this sync can take another corpus's pack — counts as removed.
+fn remove_pin_dir(dir: &Path) -> std::io::Result<()> {
+ match fs::remove_dir_all(dir) {
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ other => other,
+ }
+}
+
+/// Remove pin directories, returning the pin8s that went. Failures are skipped
+/// (the next sync tries again).
+fn remove_pins(pins: &[CachedPin]) -> Vec<String> {
+ pins.iter()
+ .filter(|p| remove_pin_dir(&p.dir).is_ok())
+ .map(|p| p.pin8.clone())
+ .collect()
+}
+
+/// An update is not a use: give the new pin directory the newest recency of the
+/// pins it replaces, so a refresh does not make an idle pack look hot.
+fn carry_last_used(old: &[CachedPin], new_dir: Option<&Path>) {
+ let (Some(new_dir), Some(when)) = (new_dir, old.iter().map(|p| p.last_used).max()) else {
+ return;
+ };
+ let secs = when
+ .duration_since(UNIX_EPOCH)
+ .unwrap_or(Duration::ZERO)
+ .as_secs();
+ let marker = new_dir.join(LAST_USED_FILE);
+ // Best effort: LRU order is a courtesy, never a reason to fail an update.
+ let _ = fs::write(&marker, secs.to_string());
+ let _ = fs::OpenOptions::new()
+ .write(true)
+ .open(&marker)
+ .and_then(|f| f.set_modified(when));
+}
+
+/// Remove whole pack directories, least recently used first (incomplete ones
+/// before any complete one), until the pin directories under `<hub>/packs` fit in
+/// `budget` bytes. Slugs in `protect` are never candidates. Returns what was
+/// removed and the size that remains.
+///
+/// The budget counts the pin directories — what this can remove — and not the
+/// `<pin8>.tmp` / `<pin8>.lock` entries of pulls in flight (or abandoned by a
+/// crash), which belong to the pull that made them: counting them would evict
+/// good packs to make room for files this cannot free.
+pub fn prune_to_budget(
+ hub_home: &Path,
+ budget: u64,
+ protect: &BTreeSet<String>,
+) -> (Vec<Pruned>, u64) {
+ let mut sized: Vec<(CachedPin, u64)> = scan_cache(hub_home)
+ .into_iter()
+ .map(|c| {
+ let bytes = dir_size(&c.dir);
+ (c, bytes)
+ })
+ .collect();
+ let mut total: u64 = sized.iter().map(|(_, b)| *b).sum();
+ let mut pruned = Vec::new();
+ if total <= budget {
+ return (pruned, total);
+ }
+ sized.retain(|(c, _)| !protect.contains(&c.slug));
+ sized.sort_by(|(a, _), (b, _)| {
+ a.complete
+ .cmp(&b.complete)
+ .then_with(|| a.last_used.cmp(&b.last_used))
+ .then_with(|| a.slug.cmp(&b.slug))
+ .then_with(|| a.pin8.cmp(&b.pin8))
+ });
+ for (c, bytes) in sized {
+ if total <= budget {
+ break;
+ }
+ if fs::remove_dir_all(&c.dir).is_ok() {
+ total = total.saturating_sub(bytes);
+ // Drop the slug directory once its last pin is gone.
+ if let Some(parent) = c.dir.parent() {
+ let _ = fs::remove_dir(parent);
+ }
+ pruned.push(Pruned {
+ slug: c.slug,
+ pin8: c.pin8,
+ bytes,
+ });
+ }
+ }
+ (pruned, total)
+}
+
+/// Stamp `<hub>/last_sync` with the current UTC time (RFC 3339); `xerj hub
+/// status` reads its age.
+pub fn write_last_sync(hub_home: &Path) -> Result<()> {
+ fs::create_dir_all(hub_home).with_context(|| format!("create {}", hub_home.display()))?;
+ let stamp = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
+ let path = hub_home.join(LAST_SYNC_FILE);
+ fs::write(&path, format!("{stamp}\n")).with_context(|| format!("write {}", path.display()))
+}
+
+// ── the timer ────────────────────────────────────────────────────────────────
+
+/// The systemd unit pair's basename (`<name>.service` + `<name>.timer`).
+pub const SYSTEMD_UNIT_BASENAME: &str = "xerj-hub-sync";
+/// The launchd agent's label, and the plist's basename.
+pub const LAUNCHD_LABEL: &str = "org.xerj.hub-sync";
+/// `StartInterval` for launchd: six hours, in seconds.
+const LAUNCHD_INTERVAL_SECS: u32 = 6 * 60 * 60;
+
+/// Which scheduler a timer is written for.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TimerKind {
+ /// A systemd user timer (`~/.config/systemd/user/`).
+ SystemdUser,
+ /// A launchd agent (`~/Library/LaunchAgents/`).
+ Launchd,
+}
+
+impl TimerKind {
+ /// The scheduler of the OS this binary was built for; `None` where neither
+ /// exists (Windows).
+ pub fn for_this_os() -> Option<TimerKind> {
+ if cfg!(target_os = "linux") {
+ Some(TimerKind::SystemdUser)
+ } else if cfg!(target_os = "macos") {
+ Some(TimerKind::Launchd)
+ } else {
+ None
+ }
+ }
+}
+
+/// Where the timer files go. Injectable so the tests write into a temp dir.
+#[derive(Debug, Clone)]
+pub struct TimerDirs {
+ /// `$XDG_CONFIG_HOME/systemd/user`, else `~/.config/systemd/user`.
+ pub systemd_user: PathBuf,
+ /// `~/Library/LaunchAgents`.
+ pub launch_agents: PathBuf,
+ /// The hub home: launchd's log lives here (`sync.log`).
+ pub hub_home: PathBuf,
+}
+
+impl TimerDirs {
+ pub fn from_env(env: &dyn Fn(&str) -> Option<OsString>) -> Result<TimerDirs> {
+ let home = env("HOME")
+ .filter(|h| !h.is_empty())
+ .map(PathBuf::from)
+ .ok_or_else(|| anyhow!("HOME is not set; cannot place a user timer"))?;
+ // XDG says a relative XDG_CONFIG_HOME is invalid and must be ignored.
+ let config = env("XDG_CONFIG_HOME")
+ .map(PathBuf::from)
+ .filter(|p| p.is_absolute())
+ .unwrap_or_else(|| home.join(".config"));
+ Ok(TimerDirs {
+ systemd_user: config.join("systemd").join("user"),
+ launch_agents: home.join("Library").join("LaunchAgents"),
+ hub_home: hub_root_from(env),
+ })
+ }
+}
+
+/// A path as one `ExecStart=` token: verbatim when it is plainly safe, else
+/// double-quoted with systemd's escapes (`\\`, `\"`, `%%` for the specifier
+/// character, `$$` for the variable one).
+fn systemd_exec_token(path: &Path) -> Result<String> {
+ let s = path
+ .to_str()
+ .ok_or_else(|| anyhow!("{} is not valid UTF-8", path.display()))?;
+ if s.chars().any(|c| c.is_control()) {
+ bail!("{s:?} contains a control character; refusing to write it into a unit");
+ }
+ let plain = |c: char| c.is_ascii_alphanumeric() || "/._-+:@,=".contains(c);
+ if !s.is_empty() && s.chars().all(plain) {
+ return Ok(s.to_string());
+ }
+ let mut out = String::from("\"");
+ for c in s.chars() {
+ match c {
+ '\\' => out.push_str("\\\\"),
+ '"' => out.push_str("\\\""),
+ '%' => out.push_str("%%"),
+ '$' => out.push_str("$$"),
+ c => out.push(c),
+ }
+ }
+ out.push('"');
+ Ok(out)
+}
+
+/// `<name>.service`: one `xerj hub sync` run. `Type=oneshot` so systemd never
+/// overlaps two runs, a timeout so a wedged run cannot block every later one.
+pub fn render_systemd_service(exe: &Path) -> Result<String> {
+ Ok(format!(
+ "# Written by `xerj hub sync --install-timer`. Enable the schedule with:\n\
+ # systemctl --user daemon-reload && systemctl --user enable --now {SYSTEMD_UNIT_BASENAME}.timer\n\
+ [Unit]\n\
+ Description=Refresh the cached XERJ hub corpus packs\n\
+ \n\
+ [Service]\n\
+ Type=oneshot\n\
+ ExecStart={} hub sync\n\
+ TimeoutStartSec=2h\n\
+ Nice=10\n",
+ systemd_exec_token(exe)?
+ ))
+}
+
+/// `<name>.timer`: every six hours on the wall clock. `Persistent=true` runs a
+/// missed slot at the next boot or wake (a laptop that slept through 06:00 still
+/// syncs), and the randomised delay keeps every client from asking the hub in
+/// the same second.
+pub fn render_systemd_timer() -> String {
+ format!(
+ "# Written by `xerj hub sync --install-timer`; runs {SYSTEMD_UNIT_BASENAME}.service.\n\
+ [Unit]\n\
+ Description=Run `xerj hub sync` every 6 hours\n\
+ \n\
+ [Timer]\n\
+ OnCalendar=*-*-* 00/6:00:00\n\
+ RandomizedDelaySec=15min\n\
+ Persistent=true\n\
+ \n\
+ [Install]\n\
+ WantedBy=timers.target\n"
+ )
+}
+
+fn xml_escape(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ for c in s.chars() {
+ match c {
+ '&' => out.push_str("&amp;"),
+ '<' => out.push_str("&lt;"),
+ '>' => out.push_str("&gt;"),
+ '"' => out.push_str("&quot;"),
+ '\'' => out.push_str("&apos;"),
+ c => out.push(c),
+ }
+ }
+ out
+}
+
+/// The launchd agent: `xerj hub sync` every 6 hours (missed intervals during
+/// sleep coalesce into one run at wake), output appended to `log`.
+pub fn render_launchd_plist(exe: &Path, log: &Path) -> Result<String> {
+ let exe = exe
+ .to_str()
+ .ok_or_else(|| anyhow!("{} is not valid UTF-8", exe.display()))?;
+ let log = log
+ .to_str()
+ .ok_or_else(|| anyhow!("{} is not valid UTF-8", log.display()))?;
+ let (exe, log) = (xml_escape(exe), xml_escape(log));
+ Ok(format!(
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
+ <!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n\
+ <!-- Written by `xerj hub sync --install-timer`. Load it with: launchctl bootstrap gui/$(id -u) {LAUNCHD_LABEL}.plist -->\n\
+ <plist version=\"1.0\">\n\
+ <dict>\n\
+ \t<key>Label</key>\n\
+ \t<string>{LAUNCHD_LABEL}</string>\n\
+ \t<key>ProgramArguments</key>\n\
+ \t<array>\n\
+ \t\t<string>{exe}</string>\n\
+ \t\t<string>hub</string>\n\
+ \t\t<string>sync</string>\n\
+ \t</array>\n\
+ \t<key>StartInterval</key>\n\
+ \t<integer>{LAUNCHD_INTERVAL_SECS}</integer>\n\
+ \t<key>RunAtLoad</key>\n\
+ \t<false/>\n\
+ \t<key>ProcessType</key>\n\
+ \t<string>Background</string>\n\
+ \t<key>LowPriorityIO</key>\n\
+ \t<true/>\n\
+ \t<key>StandardOutPath</key>\n\
+ \t<string>{log}</string>\n\
+ \t<key>StandardErrorPath</key>\n\
+ \t<string>{log}</string>\n\
+ </dict>\n\
+ </plist>\n"
+ ))
+}
+
+/// Write `contents` to `path` through a sibling file and a rename, so a reader
+/// (systemd, launchd) never sees a half-written unit.
+fn write_file_atomic(path: &Path, contents: &str) -> Result<()> {
+ let parent = path
+ .parent()
+ .ok_or_else(|| anyhow!("{} has no parent directory", path.display()))?;
+ fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
+ let mut name = path.file_name().unwrap_or_default().to_os_string();
+ name.push(".new");
+ let tmp = path.with_file_name(name);
+ fs::write(&tmp, contents).with_context(|| format!("write {}", tmp.display()))?;
+ fs::rename(&tmp, path).with_context(|| format!("rename into {}", path.display()))
+}
+
+/// Write the timer files for `kind` and return the path to the one the user
+/// enables: the `.timer` (systemd; its `.service` is written beside it) or the
+/// `.plist` (launchd). Enables, loads and starts nothing. Re-running overwrites,
+/// so it is also how a moved binary's path is refreshed.
+pub fn install_timer(kind: TimerKind, exe: &Path, dirs: &TimerDirs) -> Result<PathBuf> {
+ match kind {
+ TimerKind::SystemdUser => {
+ let service = dirs
+ .systemd_user
+ .join(format!("{SYSTEMD_UNIT_BASENAME}.service"));
+ let timer = dirs
+ .systemd_user
+ .join(format!("{SYSTEMD_UNIT_BASENAME}.timer"));
+ // Render both before writing either: a bad exe path leaves nothing behind.
+ let service_body = render_systemd_service(exe)?;
+ let timer_body = render_systemd_timer();
+ write_file_atomic(&service, &service_body)?;
+ write_file_atomic(&timer, &timer_body)?;
+ Ok(timer)
+ }
+ TimerKind::Launchd => {
+ let plist = dirs.launch_agents.join(format!("{LAUNCHD_LABEL}.plist"));
+ let log = dirs.hub_home.join("sync.log");
+ let body = render_launchd_plist(exe, &log)?;
+ // launchd opens the log path at job start and will not create its directory.
+ fs::create_dir_all(&dirs.hub_home)
+ .with_context(|| format!("create {}", dirs.hub_home.display()))?;
+ write_file_atomic(&plist, &body)?;
+ Ok(plist)
+ }
+ }
+}
+
+// ── the subcommand ───────────────────────────────────────────────────────────
+
+/// `XERJ_HUB_URL`, else the public hub.
+fn hub_url() -> String {
+ std::env::var("XERJ_HUB_URL")
+ .ok()
+ .filter(|u| !u.is_empty())
+ .unwrap_or_else(|| DEFAULT_HUB_URL.to_string())
+}
+
+/// `xerj hub sync [--url <node>]` and `xerj hub sync --install-timer`.
+pub fn run_sync(args: &[String]) -> i32 {
+ run_sync_using(args, &|url| {
+ Ok(Box::new(CliOps::current_exe(url)?) as Box<dyn SyncOps>)
+ })
+}
+
+/// [`run_sync`] with the pull/ledger side injectable, so the whole command —
+/// argument handling, the catalogue refresh and its signature check, the update,
+/// the exit code, `last_sync` — can be driven end to end by a test that has no
+/// `xerj` binary to spawn. `make_ops` receives `--url` and is called only after
+/// the catalogue has been refreshed, so a usage error, `--install-timer` or a
+/// failed refresh never builds one.
+pub fn run_sync_using(
+ args: &[String],
+ make_ops: &dyn Fn(Option<String>) -> Result<Box<dyn SyncOps>>,
+) -> i32 {
+ let usage = sync_usage();
+ let mut url: Option<String> = None;
+ let mut install = false;
+ let mut i = 0;
+ while i < args.len() {
+ match args[i].as_str() {
+ "-h" | "--help" => {
+ println!("{usage}");
+ return EXIT_OK;
+ }
+ "--install-timer" => install = true,
+ "--url" => match args.get(i + 1) {
+ Some(v) => {
+ url = Some(v.clone());
+ i += 1;
+ }
+ None => {
+ eprintln!("xerj hub sync: --url needs a value\n\n{usage}");
+ return EXIT_USAGE;
+ }
+ },
+ a if a.starts_with('-') => {
+ eprintln!("xerj hub sync: unknown flag '{a}'\n\n{usage}");
+ return EXIT_USAGE;
+ }
+ a => {
+ eprintln!("xerj hub sync: unexpected argument '{a}'\n\n{usage}");
+ return EXIT_USAGE;
+ }
+ }
+ i += 1;
+ }
+ if install {
+ if url.is_some() {
+ eprintln!("xerj hub sync: --install-timer takes no other flag\n\n{usage}");
+ return EXIT_USAGE;
+ }
+ return run_install_timer();
+ }
+
+ let hub_home = default_hub_root();
+ let cat = match refresh_catalogue(&hub_url(), &hub_home) {
+ Ok(c) => c,
+ Err(e) => {
+ eprintln!("xerj hub sync: cannot refresh the catalogue; nothing was changed: {e:#}");
+ return EXIT_INFRA;
+ }
+ };
+ println!(
+ "hub sync: catalogue refreshed (generated {}, {} corpora)",
+ cat.generated_at,
+ cat.corpora.len()
+ );
+ let ops = match make_ops(url) {
+ Ok(o) => o,
+ Err(e) => {
+ eprintln!("xerj hub sync: {e:#}");
+ return EXIT_INFRA;
+ }
+ };
+ let report = run_sync_with(
+ &hub_home,
+ &cat,
+ ops.as_ref(),
+ DISK_BUDGET_BYTES,
+ &mut |level, line| match level {
+ Level::Info => println!("{line}"),
+ Level::Warn => eprintln!("{line}"),
+ },
+ );
+ let code = report.exit_code();
+ if code == EXIT_OK {
+ if let Err(e) = write_last_sync(&hub_home) {
+ eprintln!("xerj hub sync: synced, but could not write last_sync: {e:#}");
+ return EXIT_INFRA;
+ }
+ }
+ code
+}
+
+/// `--install-timer`: write the units for this OS, print the path, nothing else.
+fn run_install_timer() -> i32 {
+ let Some(kind) = TimerKind::for_this_os() else {
+ eprintln!(
+ "xerj hub sync --install-timer: only systemd (Linux) and launchd (macOS) are \
+ supported; schedule `xerj hub sync` with your platform's scheduler instead"
+ );
+ return EXIT_INFRA;
+ };
+ let exe = match std::env::current_exe() {
+ Ok(p) => p,
+ Err(e) => {
+ eprintln!("xerj hub sync --install-timer: cannot locate the running binary: {e}");
+ return EXIT_INFRA;
+ }
+ };
+ let dirs = match TimerDirs::from_env(&|k| std::env::var_os(k)) {
+ Ok(d) => d,
+ Err(e) => {
+ eprintln!("xerj hub sync --install-timer: {e:#}");
+ return EXIT_INFRA;
+ }
+ };
+ match install_timer(kind, &exe, &dirs) {
+ Ok(path) => {
+ println!("{}", path.display());
+ EXIT_OK
+ }
+ Err(e) => {
+ eprintln!("xerj hub sync --install-timer: {e:#}");
+ EXIT_INFRA
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/engine/crates/xerj-hub/src/sync/tests.rs b/engine/crates/xerj-hub/src/sync/tests.rs
new file mode 100644
index 000000000..6f85ad953
--- /dev/null
+++ b/engine/crates/xerj-hub/src/sync/tests.rs
@@ -0,0 +1,1367 @@
+//! Unit tests for `xerj hub sync` (plan §8): every one runs on throwaway
+//! directories with a scripted [`SyncOps`], so no node, network or port is
+//! involved. `tests/sync_e2e.rs` covers the same flow against real nodes and the
+//! real `hub pull`.
+
+use std::cell::RefCell;
+use std::collections::BTreeMap;
+use std::path::Path;
+use std::time::{Duration, SystemTime};
+
+use tempfile::tempdir;
+use xerj_common::hub::{LicenceClass, PackSource, Sketch, PACK_FORMAT_VERSION};
+
+use super::*;
+
+// ── fixtures ─────────────────────────────────────────────────────────────────
+
+fn full_sha(pin8: &str) -> String {
+ format!("{pin8}{}", "0".repeat(32))
+}
+
+/// A complete extracted pack: `pack.json`, a payload file, a snapshot member and
+/// the `SHA256SUMS` that lists them.
+fn make_pack(hub: &Path, slug: &str, pin8: &str, pin_date: &str) -> PathBuf {
+ let dir = hub.join("packs").join(slug).join(pin8);
+ fs::create_dir_all(dir.join("snapshot")).unwrap();
+ let manifest = PackManifest {
+ format_version: PACK_FORMAT_VERSION,
+ slug: slug.to_string(),
+ pin_sha: full_sha(pin8),
+ pin_date: pin_date.to_string(),
+ built_at: "2026-10-04T00:00:00Z".to_string(),
+ builder_version: "test".to_string(),
+ index_prefix: format!("xc-{slug}-b1"),
+ indices: vec![],
+ snapshot_name: format!("{slug}-{pin8}"),
+ licence_spdx: "MIT".to_string(),
+ licence_class: LicenceClass::AdaptWithAttribution,
+ sources: vec![PackSource {
+ repo: slug.to_string(),
+ url: format!("https://example.test/{slug}"),
+ sha: full_sha(pin8),
+ licence: "MIT".to_string(),
+ }],
+ defs_count: 0,
+ files_count: 0,
+ };
+ fs::write(
+ dir.join("pack.json"),
+ serde_json::to_vec(&manifest).unwrap(),
+ )
+ .unwrap();
+ fs::write(dir.join("defs.jsonl"), b"{}\n").unwrap();
+ fs::write(dir.join("snapshot").join("manifest.json"), b"{}").unwrap();
+ fs::write(
+ dir.join("SHA256SUMS"),
+ "00 defs.jsonl\n00 pack.json\n00 snapshot/manifest.json\n",
+ )
+ .unwrap();
+ dir
+}
+
+fn entry(slug: &str, pin8: &str, pin_date: &str) -> CatalogueEntry {
+ CatalogueEntry {
+ slug: slug.to_string(),
+ domain: "NET".to_string(),
+ topics: vec![],
+ ask_for: "x".to_string(),
+ pin_sha: full_sha(pin8),
+ pin_date: pin_date.to_string(),
+ refresh_cadence_days: 7,
+ licence_spdx: "MIT".to_string(),
+ licence_class: LicenceClass::AdaptWithAttribution,
+ pack_bytes: 1_000,
+ pack_url: format!("https://example.test/{slug}-{pin8}.hubpack.tar.zst"),
+ pack_sha256: "ab".repeat(32),
+ sig_url: format!("https://example.test/{slug}-{pin8}.hubpack.tar.zst.sig"),
+ dependency_names: vec![],
+ quality_grade: None,
+ term_sketch: Sketch {
+ m_bits: 64,
+ k_hashes: 7,
+ bits_base64: "AAAAAAAAAAA=".to_string(),
+ },
+ }
+}
+
+fn catalogue(entries: Vec<CatalogueEntry>) -> Catalogue {
+ Catalogue {
+ format_version: 1,
+ generated_at: "2026-10-14T00:00:00Z".to_string(),
+ corpora: entries,
+ }
+}
+
+/// What a scripted pull of one slug does.
+#[derive(Clone)]
+enum Script {
+ /// Land `pin8` the way a real pull does: extract the pin directory, then
+ /// stamp the ledger.
+ Land { pin8: String, pin_date: String },
+ /// Exit non-zero.
+ Fail { code: i32, message: String },
+ /// Exit 0 without having extracted anything.
+ SucceedWithoutADirectory,
+ /// Extract the pin directory but exit 0 without ever stamping the ledger.
+ SucceedWithoutALedger { pin8: String, pin_date: String },
+}
+
+struct Fake<'a> {
+ hub: &'a Path,
+ ledger: RefCell<BTreeMap<String, Ledger>>,
+ script: BTreeMap<String, Script>,
+ pulls: RefCell<Vec<String>>,
+ /// The pin directories that existed under `packs/<slug>/` at the moment the
+ /// pull ran — proof the old pin outlives the pull.
+ present_at_pull: RefCell<BTreeMap<String, Vec<String>>>,
+}
+
+impl<'a> Fake<'a> {
+ fn new(hub: &'a Path) -> Self {
+ Fake {
+ hub,
+ ledger: RefCell::new(BTreeMap::new()),
+ script: BTreeMap::new(),
+ pulls: RefCell::new(Vec::new()),
+ present_at_pull: RefCell::new(BTreeMap::new()),
+ }
+ }
+ fn with_ledger(self, slug: &str, pin8: &str) -> Self {
+ self.ledger.borrow_mut().insert(
+ slug.to_string(),
+ Ledger::Hubpack {
+ pin8: Some(pin8.to_string()),
+ },
+ );
+ self
+ }
+ fn with_script(mut self, slug: &str, script: Script) -> Self {
+ self.script.insert(slug.to_string(), script);
+ self
+ }
+ fn pulled(&self) -> Vec<String> {
+ self.pulls.borrow().clone()
+ }
+}
+
+impl SyncOps for Fake<'_> {
+ fn pull(&self, slug: &str) -> PullOutcome {
+ self.pulls.borrow_mut().push(slug.to_string());
+ let listing = scan_cache(self.hub)
+ .into_iter()
+ .filter(|c| c.slug == slug)
+ .map(|c| c.pin8)
+ .collect();
+ self.present_at_pull
+ .borrow_mut()
+ .insert(slug.to_string(), listing);
+ match self.script.get(slug).cloned() {
+ Some(Script::Land { pin8, pin_date }) => {
+ make_pack(self.hub, slug, &pin8, &pin_date);
+ self.ledger.borrow_mut().insert(
+ slug.to_string(),
+ Ledger::Hubpack {
+ pin8: Some(pin8.clone()),
+ },
+ );
+ PullOutcome::Done
+ }
+ Some(Script::Fail { code, message }) => PullOutcome::Failed { code, message },
+ Some(Script::SucceedWithoutADirectory) => PullOutcome::Done,
+ Some(Script::SucceedWithoutALedger { pin8, pin_date }) => {
+ make_pack(self.hub, slug, &pin8, &pin_date);
+ PullOutcome::Done
+ }
+ None => PullOutcome::Failed {
+ code: 1,
+ message: format!("the test scripted no pull of {slug}"),
+ },
+ }
+ }
+
+ fn ledger(&self, slug: &str) -> Ledger {
+ self.ledger
+ .borrow()
+ .get(slug)
+ .cloned()
+ .unwrap_or(Ledger::Missing)
+ }
+}
+
+/// Run a sync, returning the report and every line it printed.
+fn sync(
+ hub: &Path,
+ cat: &Catalogue,
+ ops: &dyn SyncOps,
+ budget: u64,
+) -> (SyncReport, Vec<(Level, String)>) {
+ let mut lines = Vec::new();
+ let report = run_sync_with(hub, cat, ops, budget, &mut |level, line| {
+ lines.push((level, line.to_string()))
+ });
+ (report, lines)
+}
+
+fn has_line(lines: &[(Level, String)], needle: &str) -> bool {
+ lines.iter().any(|(_, l)| l.contains(needle))
+}
+
+fn set_mtime(path: &Path, when: SystemTime) {
+ fs::OpenOptions::new()
+ .write(true)
+ .open(path)
+ .unwrap()
+ .set_modified(when)
+ .unwrap();
+}
+
+fn days_ago(n: u64) -> SystemTime {
+ SystemTime::now() - Duration::from_secs(n * 86_400)
+}
+
+// ── §8: sync with a newer pin replaces the pack and the old dir is gone ──────
+
+#[test]
+fn a_newer_pin_replaces_the_pack_and_the_old_dir_is_gone() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(
+ "quinn",
+ Script::Land {
+ pin8: "bbbb2222".into(),
+ pin_date: "2026-10-09".into(),
+ },
+ );
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(!old.exists(), "the old pin's directory is gone");
+ let new = hub.join("packs/quinn/bbbb2222");
+ assert!(new.join("pack.json").is_file(), "the new pin is in place");
+ assert_eq!(ops.pulled(), vec!["quinn"], "exactly one pull");
+ assert_eq!(
+ ops.present_at_pull.borrow()["quinn"],
+ vec!["aaaa1111"],
+ "the old pin was still on disk while the new one was pulled — it is removed \
+ only AFTER the new pin restores"
+ );
+ assert_eq!(report.updated, vec!["quinn"]);
+ assert!(report.failed.is_empty());
+ assert_eq!(report.exit_code(), 0);
+ assert!(
+ has_line(
+ &lines,
+ "hub sync: quinn aaaa1111 -> bbbb2222: updated; old pin removed"
+ ),
+ "lines were: {lines:?}"
+ );
+ // `status` lists exactly what is on disk: only the new pin.
+ let on_disk: Vec<_> = crate::lockfile::gather_status(hub)
+ .packs
+ .iter()
+ .map(|p| (p.slug.clone(), p.pin8.clone()))
+ .collect();
+ assert_eq!(on_disk, vec![("quinn".to_string(), "bbbb2222".to_string())]);
+}
+
+#[test]
+fn every_old_pin_of_the_slug_goes_once_the_new_one_has_restored() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let a = make_pack(hub, "quinn", "aaaa1111", "2026-09-01");
+ let b = make_pack(hub, "quinn", "bbbb2222", "2026-09-20");
+ let cat = catalogue(vec![entry("quinn", "cccc3333", "2026-10-09")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "bbbb2222").with_script(
+ "quinn",
+ Script::Land {
+ pin8: "cccc3333".into(),
+ pin_date: "2026-10-09".into(),
+ },
+ );
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(!a.exists() && !b.exists());
+ assert!(hub.join("packs/quinn/cccc3333").is_dir());
+ assert_eq!(report.updated, vec!["quinn"]);
+ assert!(
+ has_line(&lines, "quinn aaaa1111,bbbb2222 -> cccc3333: updated"),
+ "lines were: {lines:?}"
+ );
+}
+
+// ── a pull that does not complete never costs the old pin ────────────────────
+
+#[test]
+fn a_failed_pull_keeps_the_old_pin_and_exits_1() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(
+ "quinn",
+ Script::Fail {
+ code: 1,
+ message: "connect refused".into(),
+ },
+ );
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(old.join("pack.json").is_file(), "the old pin is untouched");
+ assert!(!hub.join("packs/quinn/bbbb2222").exists());
+ assert_eq!(report.failed.len(), 1);
+ assert_eq!(report.failed[0].slug, "quinn");
+ assert_eq!(report.exit_code(), 1);
+ assert!(report.updated.is_empty());
+ let failed: Vec<_> = lines.iter().filter(|(_, l)| l.contains("FAILED")).collect();
+ assert_eq!(failed.len(), 1);
+ assert_eq!(failed[0].0, Level::Warn, "failures go to stderr");
+ assert!(failed[0].1.contains("connect refused"));
+ assert!(failed[0].1.contains("old pin aaaa1111 kept"));
+}
+
+#[test]
+fn a_pack_that_fails_verification_exits_3_and_keeps_the_old_pin() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let lz4 = make_pack(hub, "lz4", "dddd4444", "2026-10-02");
+ let cat = catalogue(vec![
+ entry("quinn", "bbbb2222", "2026-10-09"),
+ entry("lz4", "eeee5555", "2026-10-09"),
+ ]);
+ let ops = Fake::new(hub)
+ .with_ledger("quinn", "aaaa1111")
+ .with_ledger("lz4", "dddd4444")
+ .with_script(
+ "quinn",
+ Script::Fail {
+ code: 3,
+ message: "pack verification failed: signature does not verify".into(),
+ },
+ )
+ .with_script(
+ "lz4",
+ Script::Land {
+ pin8: "eeee5555".into(),
+ pin_date: "2026-10-09".into(),
+ },
+ );
+
+ let (report, _lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(
+ old.is_dir(),
+ "the unverifiable pack did not displace the old pin"
+ );
+ assert!(!lz4.exists() && hub.join("packs/lz4/eeee5555").is_dir());
+ assert_eq!(
+ report.updated,
+ vec!["lz4"],
+ "one slug failing does not stop the others"
+ );
+ assert_eq!(
+ report.exit_code(),
+ 3,
+ "verification failure outranks a plain failure"
+ );
+}
+
+#[test]
+fn a_pull_that_claims_success_without_a_new_directory_keeps_the_old_pin() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);
+ let ops = Fake::new(hub)
+ .with_ledger("quinn", "aaaa1111")
+ .with_script("quinn", Script::SucceedWithoutADirectory);
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(old.is_dir());
+ assert_eq!(report.failed.len(), 1);
+ assert!(
+ has_line(&lines, "missing or incomplete"),
+ "lines: {lines:?}"
+ );
+}
+
+#[test]
+fn a_pull_that_never_stamped_the_ledger_keeps_the_old_pin() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(
+ "quinn",
+ Script::SucceedWithoutALedger {
+ pin8: "bbbb2222".into(),
+ pin_date: "2026-10-09".into(),
+ },
+ );
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(
+ old.is_dir(),
+ "the new pin is extracted but not recorded as restored, so the old pin stays"
+ );
+ assert_eq!(report.failed.len(), 1);
+ assert!(
+ has_line(&lines, "does not record bbbb2222 as restored"),
+ "lines: {lines:?}"
+ );
+}
+
+// ── nothing to do ────────────────────────────────────────────────────────────
+
+#[test]
+fn a_pack_already_at_the_catalogue_pin_is_not_pulled() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let dir = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let cat = catalogue(vec![entry("quinn", "aaaa1111", "2026-10-02")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111");
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(ops.pulled().is_empty(), "no pull, so no download");
+ assert!(dir.is_dir());
+ assert_eq!(report.current, vec!["quinn"]);
+ assert_eq!(report.exit_code(), 0);
+ assert!(has_line(&lines, "hub sync: quinn aaaa1111: current"));
+}
+
+#[test]
+fn an_empty_cache_is_a_clean_no_op() {
+ let tmp = tempdir().unwrap();
+ let cat = catalogue(vec![entry("quinn", "aaaa1111", "2026-10-02")]);
+ let ops = Fake::new(tmp.path());
+
+ let (report, lines) = sync(tmp.path(), &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert_eq!(report, SyncReport::default());
+ assert!(has_line(&lines, "no cached packs"));
+ assert!(ops.pulled().is_empty());
+}
+
+#[test]
+fn superseded_pins_beside_a_restored_current_pin_are_removed() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let stale = make_pack(hub, "quinn", "aaaa1111", "2026-09-01");
+ let current = make_pack(hub, "quinn", "bbbb2222", "2026-10-02");
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-02")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "bbbb2222");
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(ops.pulled().is_empty());
+ assert!(!stale.exists(), "a leftover older pin is housekeeping");
+ assert!(current.is_dir());
+ assert_eq!(report.current, vec!["quinn"]);
+ assert!(
+ has_line(&lines, "removed superseded pin aaaa1111"),
+ "{lines:?}"
+ );
+}
+
+// ── "restored", not just "extracted" ─────────────────────────────────────────
+
+#[test]
+fn a_cached_pin_the_ledger_does_not_record_is_pulled_again_and_kept() {
+ // The catalogue pin's directory exists (an interrupted pull extracted it)
+ // but the ledger still names the old pin: it was never restored.
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let old = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let target = make_pack(hub, "quinn", "bbbb2222", "2026-10-09");
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(
+ "quinn",
+ Script::Land {
+ pin8: "bbbb2222".into(),
+ pin_date: "2026-10-09".into(),
+ },
+ );
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert_eq!(
+ ops.pulled(),
+ vec!["quinn"],
+ "present is not the same as restored"
+ );
+ assert!(target.is_dir(), "the pin being repaired is never removed");
+ assert!(!old.exists());
+ assert_eq!(report.updated, vec!["quinn"]);
+ assert!(has_line(
+ &lines,
+ "re-pulled (the pin was cached but not restored)"
+ ));
+}
+
+#[test]
+fn a_half_extracted_current_pin_is_pulled_again() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let dir = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ // The extractor writes pack.json before the snapshot: lose a listed file.
+ fs::remove_file(dir.join("snapshot").join("manifest.json")).unwrap();
+ let cat = catalogue(vec![entry("quinn", "aaaa1111", "2026-10-02")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(
+ "quinn",
+ Script::Land {
+ pin8: "aaaa1111".into(),
+ pin_date: "2026-10-02".into(),
+ },
+ );
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert_eq!(ops.pulled(), vec!["quinn"]);
+ assert!(
+ dir.join("snapshot").join("manifest.json").is_file(),
+ "the pull rewrote the directory in place"
+ );
+ assert_eq!(report.updated, vec!["quinn"]);
+ assert!(has_line(
+ &lines,
+ "re-pulled (the cached pin was incomplete)"
+ ));
+}
+
+#[test]
+fn scan_cache_sees_packs_and_nothing_else() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ // In-flight pull artefacts and strays sit in the same directory.
+ let slug_dir = hub.join("packs/quinn");
+ fs::create_dir(slug_dir.join("bbbb2222.tmp")).unwrap();
+ fs::create_dir(slug_dir.join("bbbb2222.lock")).unwrap();
+ fs::create_dir(slug_dir.join("NOTHEX00")).unwrap();
+ fs::write(slug_dir.join("cccc3333"), b"a file, not a pack").unwrap();
+ fs::write(hub.join("packs/stray-file"), b"x").unwrap();
+ // A pin directory with no pack.json is listed, but not complete.
+ fs::create_dir_all(hub.join("packs/lz4/dddd4444")).unwrap();
+
+ let cached = scan_cache(hub);
+
+ let seen: Vec<_> = cached
+ .iter()
+ .map(|c| (c.slug.as_str(), c.pin8.as_str(), c.complete))
+ .collect();
+ assert_eq!(
+ seen,
+ vec![("lz4", "dddd4444", false), ("quinn", "aaaa1111", true)]
+ );
+ assert_eq!(cached[1].pin_date.as_deref(), Some("2026-10-02"));
+}
+
+#[test]
+fn a_sums_entry_that_climbs_out_of_the_pack_dir_makes_it_incomplete() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let dir = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ fs::write(hub.join("outside"), b"x").unwrap();
+ fs::write(
+ dir.join("SHA256SUMS"),
+ "00 pack.json\n00 ../../../outside\n",
+ )
+ .unwrap();
+ assert!(!scan_cache(hub)[0].complete);
+}
+
+// ── refusals ─────────────────────────────────────────────────────────────────
+
+#[test]
+fn a_catalogue_older_than_the_cache_never_downgrades_it() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let cached = make_pack(hub, "quinn", "bbbb2222", "2026-10-09");
+ // A replayed (validly signed, but old) catalogue still lists yesterday's pin.
+ let cat = catalogue(vec![entry("quinn", "aaaa1111", "2026-10-02")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "bbbb2222");
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(ops.pulled().is_empty());
+ assert!(cached.is_dir());
+ assert_eq!(report.kept, vec!["quinn"]);
+ assert_eq!(report.exit_code(), 0, "a refusal is not a failure");
+ assert!(
+ has_line(&lines, "older than the cached one (2026-10-09)"),
+ "lines: {lines:?}"
+ );
+}
+
+#[test]
+fn a_slug_the_catalogue_no_longer_lists_is_kept() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let dir = make_pack(hub, "gone", "aaaa1111", "2026-10-02");
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);
+ let ops = Fake::new(hub);
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(dir.is_dir());
+ assert!(ops.pulled().is_empty());
+ assert_eq!(report.kept, vec!["gone"]);
+ assert!(has_line(
+ &lines,
+ "gone aaaa1111: kept (not in the catalogue)"
+ ));
+}
+
+#[test]
+fn excluded_corpora_and_oversize_packs_are_never_pulled() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let a = make_pack(hub, "closed", "aaaa1111", "2026-10-02");
+ let b = make_pack(hub, "huge", "bbbb2222", "2026-10-02");
+ let mut closed = entry("closed", "cccc3333", "2026-10-09");
+ closed.licence_class = LicenceClass::Excluded;
+ let mut huge = entry("huge", "dddd4444", "2026-10-09");
+ huge.pack_bytes = AUTO_PULL_MAX_BYTES + 1;
+ let cat = catalogue(vec![closed, huge]);
+ let ops = Fake::new(hub)
+ .with_ledger("closed", "aaaa1111")
+ .with_ledger("huge", "bbbb2222");
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(ops.pulled().is_empty(), "neither is downloaded");
+ assert!(a.is_dir() && b.is_dir(), "both cached copies are kept");
+ assert_eq!(report.kept, vec!["closed", "huge"]);
+ assert_eq!(report.exit_code(), 0);
+ assert!(has_line(&lines, "classes it excluded"));
+ assert!(has_line(&lines, "over the 52428800-byte auto-pull cap"));
+}
+
+#[test]
+fn a_current_pack_stays_current_when_the_catalogue_later_calls_it_excluded() {
+ // `excluded` stops a NEW pin being pulled; it is not a reason to report a
+ // pack that is already at the catalogue's pin as something to refuse.
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let dir = make_pack(hub, "closed", "aaaa1111", "2026-10-02");
+ let mut same_pin = entry("closed", "aaaa1111", "2026-10-02");
+ same_pin.licence_class = LicenceClass::Excluded;
+ let ops = Fake::new(hub).with_ledger("closed", "aaaa1111");
+
+ let (report, lines) = sync(hub, &catalogue(vec![same_pin]), &ops, DISK_BUDGET_BYTES);
+
+ assert!(dir.is_dir() && ops.pulled().is_empty());
+ assert_eq!(report.current, vec!["closed"]);
+ assert!(
+ has_line(&lines, "hub sync: closed aaaa1111: current"),
+ "{lines:?}"
+ );
+}
+
+#[test]
+fn a_slug_served_from_a_local_index_is_left_alone() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let dir = make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);
+ let ops = Fake::new(hub);
+ ops.ledger
+ .borrow_mut()
+ .insert("quinn".into(), Ledger::Local);
+
+ let (report, lines) = sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ assert!(
+ ops.pulled().is_empty(),
+ "a pull would flip the ledger back to the pack"
+ );
+ assert!(dir.is_dir());
+ assert_eq!(report.kept, vec!["quinn"]);
+ assert!(has_line(&lines, "served from a local index"));
+}
+
+#[test]
+fn a_catalogue_pin_that_is_not_a_commit_is_refused() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ make_pack(hub, "quinn", "aaaa1111", "2026-10-02");
+ let mut bad = entry("quinn", "bbbb2222", "2026-10-09");
+ bad.pin_sha = "../../etc".into();
+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111");
+
+ let (report, lines) = sync(hub, &catalogue(vec![bad]), &ops, DISK_BUDGET_BYTES);
+
+ assert!(ops.pulled().is_empty());
+ assert_eq!(report.kept, vec!["quinn"]);
+ assert!(has_line(&lines, "is not a hex commit"));
+}
+
+// ── the disk budget ──────────────────────────────────────────────────────────
+
+/// A complete pack padded to `size` bytes with a given recency.
+fn sized_pack(hub: &Path, slug: &str, pin8: &str, size: usize, used: SystemTime) -> PathBuf {
+ let dir = make_pack(hub, slug, pin8, "2026-10-02");
+ fs::write(dir.join("snapshot").join("segment.bin"), vec![0u8; size]).unwrap();
+ let marker = dir.join("last_used");
+ fs::write(&marker, b"x").unwrap();
+ set_mtime(&marker, used);
+ dir
+}
+
+#[test]
+fn the_budget_pass_removes_least_recently_used_first_until_it_fits() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let oldest = sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(30));
+ let middle = sized_pack(hub, "beta", "bbbb2222", 10_000, days_ago(10));
+ let newest = sized_pack(hub, "gamma", "cccc3333", 10_000, days_ago(1));
+ let total = dir_size(&hub.join("packs"));
+ // Room for two of the three packs.
+ let budget = total - 10_000;
+
+ let (pruned, left) = prune_to_budget(hub, budget, &BTreeSet::new());
+
+ assert_eq!(
+ pruned.iter().map(|p| p.slug.as_str()).collect::<Vec<_>>(),
+ vec!["alpha"],
+ "only the least recently used pack had to go"
+ );
+ assert!(left <= budget);
+ assert!(!oldest.exists() && middle.is_dir() && newest.is_dir());
+ assert!(
+ !hub.join("packs/alpha").exists(),
+ "an emptied slug directory is dropped too"
+ );
+}
+
+#[test]
+fn the_budget_pass_keeps_removing_until_the_cache_fits() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(30));
+ sized_pack(hub, "beta", "bbbb2222", 10_000, days_ago(10));
+ let newest = sized_pack(hub, "gamma", "cccc3333", 10_000, days_ago(1));
+ let one_pack = dir_size(&newest);
+
+ let (pruned, left) = prune_to_budget(hub, one_pack + 1, &BTreeSet::new());
+
+ assert_eq!(
+ pruned.iter().map(|p| p.slug.as_str()).collect::<Vec<_>>(),
+ vec!["alpha", "beta"],
+ "oldest first"
+ );
+ assert!(left <= one_pack + 1);
+ assert!(newest.is_dir());
+}
+
+#[test]
+fn a_cache_inside_the_budget_is_left_exactly_alone() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let dir = sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(300));
+
+ let (pruned, _) = prune_to_budget(hub, DISK_BUDGET_BYTES, &BTreeSet::new());
+
+ assert!(pruned.is_empty());
+ assert!(dir.is_dir());
+}
+
+#[test]
+fn leftovers_of_pulls_do_not_count_against_the_budget() {
+ // The budget measures what sync can remove. A crashed pull's `.tmp` is the
+ // pull's to clean up; counting it would evict a good pack to make room for
+ // bytes this can never free.
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let dir = sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(300));
+ let junk = hub.join("packs/alpha/bbbb2222.tmp");
+ fs::create_dir_all(&junk).unwrap();
+ fs::write(junk.join("partial.tar.zst"), vec![0u8; 1_000_000]).unwrap();
+ let budget = dir_size(&dir) + 100;
+
+ let (pruned, left) = prune_to_budget(hub, budget, &BTreeSet::new());
+
+ assert!(
+ pruned.is_empty(),
+ "the pack fits; a megabyte of junk elsewhere is not its problem"
+ );
+ assert!(left <= budget);
+ assert!(dir.is_dir());
+ assert!(
+ junk.is_dir(),
+ "and sync does not touch another process's temp directory"
+ );
+}
+
+#[test]
+fn incomplete_directories_are_pruned_before_any_pack() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let good = sized_pack(hub, "alpha", "aaaa1111", 10_000, days_ago(300));
+ let junk = hub.join("packs/beta/bbbb2222");
+ fs::create_dir_all(&junk).unwrap();
+ fs::write(junk.join("partial.bin"), vec![0u8; 10_000]).unwrap();
+ let budget = dir_size(&hub.join("packs")) - 5_000;
+
+ let (pruned, _) = prune_to_budget(hub, budget, &BTreeSet::new());
+
+ assert_eq!(pruned.len(), 1);
+ assert_eq!(
+ pruned[0].slug, "beta",
+ "the half-extracted leftover goes first"
+ );
+ assert!(
+ good.is_dir(),
+ "even though the real pack was used less recently"
+ );
+}
+
+#[test]
+fn packs_updated_in_this_run_are_not_pruned() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ sized_pack(hub, "quinn", "aaaa1111", 10_000, days_ago(30));
+ let other = sized_pack(hub, "lz4", "dddd4444", 10_000, days_ago(1));
+ let cat = catalogue(vec![
+ entry("quinn", "bbbb2222", "2026-10-09"),
+ entry("lz4", "dddd4444", "2026-10-02"),
+ ]);
+ let ops = Fake::new(hub)
+ .with_ledger("quinn", "aaaa1111")
+ .with_ledger("lz4", "dddd4444")
+ .with_script(
+ "quinn",
+ Script::Land {
+ pin8: "bbbb2222".into(),
+ pin_date: "2026-10-09".into(),
+ },
+ );
+ // Far too small for either pack: lz4 (used yesterday) is the only candidate,
+ // because quinn was just fetched — so it goes, and quinn stays over budget.
+ let budget = 100;
+
+ let (report, lines) = sync(hub, &cat, &ops, budget);
+
+ assert!(
+ hub.join("packs/quinn/bbbb2222").is_dir(),
+ "the fresh pack stays"
+ );
+ assert!(!other.exists(), "the only candidate went");
+ assert_eq!(report.pruned.len(), 1);
+ assert_eq!(report.pruned[0].slug, "lz4");
+ assert!(has_line(&lines, "pruned lz4 dddd4444"), "lines: {lines:?}");
+ assert!(
+ has_line(&lines, "nothing more can be pruned"),
+ "still over budget, and it says so: {lines:?}"
+ );
+}
+
+#[test]
+fn an_update_keeps_the_old_pins_recency() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path();
+ let long_idle = days_ago(40);
+ sized_pack(hub, "quinn", "aaaa1111", 100, long_idle);
+ let cat = catalogue(vec![entry("quinn", "bbbb2222", "2026-10-09")]);
+ let ops = Fake::new(hub).with_ledger("quinn", "aaaa1111").with_script(
+ "quinn",
+ Script::Land {
+ pin8: "bbbb2222".into(),
+ pin_date: "2026-10-09".into(),
+ },
+ );
+
+ sync(hub, &cat, &ops, DISK_BUDGET_BYTES);
+
+ let new = scan_cache(hub);
+ assert_eq!(new.len(), 1);
+ assert_eq!(new[0].pin8, "bbbb2222");
+ let (early, late) = if new[0].last_used < long_idle {
+ (new[0].last_used, long_idle)
+ } else {
+ (long_idle, new[0].last_used)
+ };
+ let drift = late.duration_since(early).unwrap();
+ assert!(
+ drift < Duration::from_secs(2),
+ "refreshing a pack is not using it: its recency must stay the old pin's, \
+ but it moved by {drift:?}"
+ );
+}
+
+// ── last_sync and the exit code ──────────────────────────────────────────────
+
+#[test]
+fn last_sync_is_an_rfc3339_stamp_that_status_can_read() {
+ let tmp = tempdir().unwrap();
+ let hub = tmp.path().join("fresh-hub-dir");
+
+ write_last_sync(&hub).unwrap();
+
+ let raw = fs::read_to_string(hub.join("last_sync")).unwrap();
+ chrono::DateTime::parse_from_rfc3339(raw.trim()).expect("RFC 3339");
+ assert_eq!(
+ crate::lockfile::gather_status(&hub).last_sync_age_days,
+ Some(0),
+ "`xerj hub status` reads the file's age"
+ );
+}
+
+#[test]
+fn the_exit_code_ranks_verification_over_infrastructure() {
+ let mut r = SyncReport::default();
+ assert_eq!(r.exit_code(), 0);
+ r.failed.push(SyncFailure {
+ slug: "a".into(),
+ code: 1,
+ message: String::new(),
+ });
+ assert_eq!(r.exit_code(), 1);
+ r.failed.push(SyncFailure {
+ slug: "b".into(),
+ code: 3,
+ message: String::new(),
+ });
+ assert_eq!(r.exit_code(), 3);
+}
+
+// ── the child-process pull ───────────────────────────────────────────────────
+
+#[cfg(unix)]
+mod child_process {
+ use super::*;
+
+ /// A `sh` script standing in for `xerj`: records its arguments, writes a
+ /// line to stderr, exits with `code`. Run as `sh <script> hub pull …` so no
+ /// freshly written executable is ever exec'd (no ETXTBSY race under the
+ /// parallel test runner).
+ fn fake_xerj(dir: &Path, code: i32, stderr: &str) -> (PathBuf, PathBuf) {
+ let args = dir.join("args.txt");
+ let script = dir.join("fake-xerj.sh");
+ fs::write(
+ &script,
+ format!(
+ "printf '%s\\n' \"$@\" > '{}'\necho '{stderr}' >&2\nexit {code}\n",
+ args.display()
+ ),
+ )
+ .unwrap();
+ (script, args)
+ }
+
+ fn ops(script: &Path, url: Option<&str>, code_root: &Path) -> CliOps {
+ CliOps::with_program(
+ "sh",
+ vec![script.as_os_str().to_owned()],
+ url.map(str::to_string),
+ code_root.to_path_buf(),
+ )
+ }
+
+ #[test]
+ fn pull_runs_the_existing_pull_subcommand_quietly() {
+ let tmp = tempdir().unwrap();
+ let (script, args) = fake_xerj(tmp.path(), 0, "");
+
+ let out = ops(&script, Some("http://127.0.0.1:1"), tmp.path()).pull("quinn");
+
+ assert_eq!(out, PullOutcome::Done);
+ assert_eq!(
+ fs::read_to_string(args)
+ .unwrap()
+ .lines()
+ .collect::<Vec<_>>(),
+ vec![
+ "hub",
+ "pull",
+ "quinn",
+ "--quiet",
+ "--url",
+ "http://127.0.0.1:1"
+ ],
+ "sync drives `xerj hub pull`, it does not reimplement it"
+ );
+ }
+
+ #[test]
+ fn pull_omits_url_when_none_was_given() {
+ let tmp = tempdir().unwrap();
+ let (script, args) = fake_xerj(tmp.path(), 0, "");
+
+ ops(&script, None, tmp.path()).pull("quinn");
+
+ assert_eq!(
+ fs::read_to_string(args)
+ .unwrap()
+ .lines()
+ .collect::<Vec<_>>(),
+ vec!["hub", "pull", "quinn", "--quiet"]
+ );
+ }
+
+ #[test]
+ fn a_nonzero_exit_carries_its_code_and_stderr() {
+ let tmp = tempdir().unwrap();
+ let (script, _) = fake_xerj(
+ tmp.path(),
+ 3,
+ "xerj hub pull: pack verification failed: pack signature",
+ );
+
+ let out = ops(&script, None, tmp.path()).pull("quinn");
+
+ assert_eq!(
+ out,
+ PullOutcome::Failed {
+ code: 3,
+ message: "xerj hub pull: pack verification failed: pack signature".into()
+ }
+ );
+ }
+
+ #[test]
+ fn a_missing_program_is_an_infrastructure_failure() {
+ let tmp = tempdir().unwrap();
+ let ops = CliOps::with_program(
+ tmp.path().join("no-such-xerj"),
+ vec![],
+ None,
+ tmp.path().to_path_buf(),
+ );
+
+ match ops.pull("quinn") {
+ PullOutcome::Failed { code, message } => {
+ assert_eq!(code, 1);
+ assert!(message.contains("cannot run"), "{message}");
+ }
+ other => panic!("expected a failure, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn the_ledger_is_read_from_the_code_root() {
+ let tmp = tempdir().unwrap();
+ let state = tmp.path().join("state");
+ fs::create_dir_all(&state).unwrap();
+ let (script, _) = fake_xerj(tmp.path(), 0, "");
+ let ops = ops(&script, None, tmp.path());
+
+ assert_eq!(ops.ledger("quinn"), Ledger::Missing);
+
+ fs::write(
+ state.join("quinn.json"),
+ format!(
+ r#"{{"corpus":"quinn","source":"hubpack","pin_sha":"{}"}}"#,
+ full_sha("bbbb2222")
+ ),
+ )
+ .unwrap();
+ assert_eq!(
+ ops.ledger("quinn"),
+ Ledger::Hubpack {
+ pin8: Some("bbbb2222".into())
+ }
+ );
+
+ fs::write(
+ state.join("lz4.json"),
+ r#"{"corpus":"lz4","index_prefix":"xc-lz4-b1700000000"}"#,
+ )
+ .unwrap();
+ assert_eq!(ops.ledger("lz4"), Ledger::Local, "no `source: hubpack`");
+ }
+}
+
+#[test]
+fn one_line_flattens_and_caps_stderr() {
+ assert_eq!(one_line("a\n b\t c\n", 100), "a b c");
+ assert_eq!(one_line("abcdefghij", 4), "abcd…");
+}
+
+// ── the timer ────────────────────────────────────────────────────────────────
+
+fn timer_dirs(root: &Path) -> TimerDirs {
+ TimerDirs {
+ systemd_user: root.join("cfg/systemd/user"),
+ launch_agents: root.join("Library/LaunchAgents"),
+ hub_home: root.join("hub"),
+ }
+}
+
+#[test]
+fn install_timer_writes_a_six_hourly_systemd_user_timer_and_nothing_else() {
+ let tmp = tempdir().unwrap();
+ let dirs = timer_dirs(tmp.path());
+ let exe = Path::new("/home/u/.local/bin/xerj");
+
+ let printed = install_timer(TimerKind::SystemdUser, exe, &dirs).unwrap();
+
+ // The path printed is the timer — the unit the user enables.
+ assert_eq!(printed, dirs.systemd_user.join("xerj-hub-sync.timer"));
+ let timer = fs::read_to_string(&printed).unwrap();
+ let service = fs::read_to_string(dirs.systemd_user.join("xerj-hub-sync.service")).unwrap();
+
+ let timer_lines: Vec<&str> = timer.lines().collect();
+ for want in [
+ "[Timer]",
+ "OnCalendar=*-*-* 00/6:00:00",
+ "Persistent=true",
+ "[Install]",
+ "WantedBy=timers.target",
+ ] {
+ assert!(
+ timer_lines.contains(&want),
+ "timer lacks {want:?}:\n{timer}"
+ );
+ }
+ let service_lines: Vec<&str> = service.lines().collect();
+ for want in [
+ "[Service]",
+ "Type=oneshot",
+ "ExecStart=/home/u/.local/bin/xerj hub sync",
+ ] {
+ assert!(
+ service_lines.contains(&want),
+ "service lacks {want:?}:\n{service}"
+ );
+ }
+
+ // Nothing else: two files in the systemd dir, no hub dir, no launchd dir.
+ let mut written: Vec<String> = fs::read_dir(&dirs.systemd_user)
+ .unwrap()
+ .flatten()
+ .map(|e| e.file_name().to_string_lossy().into_owned())
+ .collect();
+ written.sort();
+ assert_eq!(
+ written,
+ vec!["xerj-hub-sync.service", "xerj-hub-sync.timer"]
+ );
+ assert!(!dirs.hub_home.exists() && !dirs.launch_agents.exists());
+}
+
+#[test]
+fn install_timer_overwrites_so_a_moved_binary_is_picked_up() {
+ let tmp = tempdir().unwrap();
+ let dirs = timer_dirs(tmp.path());
+
+ install_timer(TimerKind::SystemdUser, Path::new("/old/xerj"), &dirs).unwrap();
+ install_timer(TimerKind::SystemdUser, Path::new("/new/xerj"), &dirs).unwrap();
+
+ let service = fs::read_to_string(dirs.systemd_user.join("xerj-hub-sync.service")).unwrap();
+ assert!(service.contains("ExecStart=/new/xerj hub sync"));
+ assert!(!service.contains("/old/xerj"));
+}
+
+#[test]
+fn a_binary_path_that_needs_quoting_is_quoted_for_systemd() {
+ let tok = |p: &str| systemd_exec_token(Path::new(p)).unwrap();
+ assert_eq!(tok("/usr/bin/xerj"), "/usr/bin/xerj");
+ assert_eq!(tok("/opt/my tools/xerj"), "\"/opt/my tools/xerj\"");
+ // `%` starts a specifier and `$` a variable even inside quotes.
+ assert_eq!(tok("/opt/100%/xerj"), "\"/opt/100%%/xerj\"");
+ assert_eq!(tok("/opt/$HOME/xerj"), "\"/opt/$$HOME/xerj\"");
+ assert_eq!(tok("/opt/a\"b\\c/xerj"), "\"/opt/a\\\"b\\\\c/xerj\"");
+ assert!(systemd_exec_token(Path::new("/opt/a\nb")).is_err());
+
+ let service = render_systemd_service(Path::new("/opt/my tools/xerj")).unwrap();
+ assert!(service.contains("ExecStart=\"/opt/my tools/xerj\" hub sync"));
+}
+
+#[test]
+fn a_binary_path_that_cannot_be_written_safely_leaves_nothing_behind() {
+ let tmp = tempdir().unwrap();
+ let dirs = timer_dirs(tmp.path());
+
+ let err = install_timer(TimerKind::SystemdUser, Path::new("/opt/a\nb/xerj"), &dirs)
+ .expect_err("a newline cannot ride in a unit file");
+
+ assert!(format!("{err:#}").contains("control character"));
+ assert!(!dirs.systemd_user.exists(), "no half-installed units");
+}
+
+#[test]
+fn install_timer_writes_a_six_hourly_launchd_agent() {
+ let tmp = tempdir().unwrap();
+ let dirs = timer_dirs(tmp.path());
+ let exe = Path::new("/Users/u/bin/xerj & co/xerj");
+
+ let printed = install_timer(TimerKind::Launchd, exe, &dirs).unwrap();
+
+ assert_eq!(printed, dirs.launch_agents.join("org.xerj.hub-sync.plist"));
+ let plist = fs::read_to_string(&printed).unwrap();
+ for want in [
+ "<key>Label</key>\n\t<string>org.xerj.hub-sync</string>",
+ // argv in order: the binary, then `hub sync`.
+ "<string>/Users/u/bin/xerj &amp; co/xerj</string>\n\t\t<string>hub</string>\n\t\t<string>sync</string>",
+ "<key>StartInterval</key>\n\t<integer>21600</integer>",
+ "<key>RunAtLoad</key>\n\t<false/>",
+ ] {
+ assert!(plist.contains(want), "plist lacks {want:?}:\n{plist}");
+ }
+ let log = dirs.hub_home.join("sync.log");
+ assert!(
+ plist.contains(&format!("<string>{}</string>", log.display())),
+ "launchd output lands in the hub home"
+ );
+ assert!(
+ dirs.hub_home.is_dir(),
+ "launchd will not create the log's directory"
+ );
+ assert!(plist.starts_with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"));
+ assert!(plist.trim_end().ends_with("</plist>"));
+ assert!(
+ !dirs.systemd_user.exists(),
+ "only the launchd file is written"
+ );
+}
+
+#[test]
+fn timer_dirs_follow_xdg_config_home_and_ignore_a_relative_one() {
+ let env = |pairs: &'static [(&'static str, &'static str)]| {
+ move |k: &str| -> Option<OsString> {
+ pairs
+ .iter()
+ .find(|(name, _)| *name == k)
+ .map(|(_, v)| OsString::from(*v))
+ }
+ };
+
+ let dirs =
+ TimerDirs::from_env(&env(&[("HOME", "/home/u"), ("XDG_CONFIG_HOME", "/cfg")])).unwrap();
+ assert_eq!(dirs.systemd_user, PathBuf::from("/cfg/systemd/user"));
+ assert_eq!(
+ dirs.launch_agents,
+ PathBuf::from("/home/u/Library/LaunchAgents")
+ );
+ assert_eq!(dirs.hub_home, PathBuf::from("/home/u/.xerj/hub"));
+
+ // XDG: a relative XDG_CONFIG_HOME is invalid and ignored.
+ let dirs =
+ TimerDirs::from_env(&env(&[("HOME", "/home/u"), ("XDG_CONFIG_HOME", "rel")])).unwrap();
+ assert_eq!(
+ dirs.systemd_user,
+ PathBuf::from("/home/u/.config/systemd/user")
+ );
+
+ // The same hub-home override every other subcommand honours.
+ let dirs =
+ TimerDirs::from_env(&env(&[("HOME", "/home/u"), ("XERJ_HUB_HOME", "/srv/hub")])).unwrap();
+ assert_eq!(dirs.hub_home, PathBuf::from("/srv/hub"));
+
+ assert!(
+ TimerDirs::from_env(&env(&[])).is_err(),
+ "no HOME, no user timer"
+ );
+}
+
+/// systemd is the arbiter of what a unit means, so when the machine has
+/// `systemd-analyze` ask it: the calendar spec must fire every six hours, and
+/// `verify` must have no complaint about the files we wrote. (Skipped where it
+/// is absent: it is not a build dependency.)
+#[cfg(target_os = "linux")]
+#[test]
+fn systemd_reads_the_units_as_a_six_hourly_oneshot() {
+ let have = Command::new("systemd-analyze").arg("--version").output();
+ if !have.is_ok_and(|o| o.status.success()) {
+ eprintln!("systemd-analyze not available; skipping");
+ return;
+ }
+
+ // The calendar spec in the timer we render, in UTC so the host's zone is moot.
+ let spec = render_systemd_timer()
+ .lines()
+ .find_map(|l| l.strip_prefix("OnCalendar=").map(str::to_string))
+ .expect("the timer has an OnCalendar line");
+ let cal = Command::new("systemd-analyze")
+ .env("TZ", "UTC")
+ .args(["calendar", "--iterations=5", &spec])
+ .output()
+ .unwrap();
+ let text = String::from_utf8_lossy(&cal.stdout).into_owned();
+ assert!(cal.status.success(), "calendar spec rejected:\n{text}");
+ let hours: Vec<&str> = text
+ .lines()
+ .filter(|l| l.contains("UTC") && !l.contains("From now"))
+ .filter_map(|l| l.split_whitespace().rev().nth(1))
+ .collect();
+ assert_eq!(
+ hours,
+ vec!["00:00:00", "06:00:00", "12:00:00", "18:00:00", "00:00:00"],
+ "five successive firings are six hours apart:\n{text}"
+ );
+
+ // Whole-unit check. `/bin/true` exists everywhere systemd does.
+ let tmp = tempdir().unwrap();
+ let dirs = timer_dirs(tmp.path());
+ install_timer(TimerKind::SystemdUser, Path::new("/bin/true"), &dirs).unwrap();
+ let verify = Command::new("systemd-analyze")
+ .arg("verify")
+ .arg(dirs.systemd_user.join("xerj-hub-sync.service"))
+ .arg(dirs.systemd_user.join("xerj-hub-sync.timer"))
+ .output()
+ .unwrap();
+ let diag = String::from_utf8_lossy(&verify.stderr);
+ assert!(
+ verify.status.success() && !diag.contains("xerj-hub-sync"),
+ "systemd-analyze verify complained:\n{diag}"
+ );
+}
+
+// ── the subcommand's argument handling ───────────────────────────────────────
+
+fn args(a: &[&str]) -> Vec<String> {
+ a.iter().map(|s| s.to_string()).collect()
+}
+
+#[test]
+fn sync_rejects_what_it_does_not_understand_before_touching_anything() {
+ assert_eq!(run_sync(&args(&["--bogus"])), 2);
+ assert_eq!(run_sync(&args(&["some-slug"])), 2, "sync takes no slug");
+ assert_eq!(run_sync(&args(&["--url"])), 2, "--url needs a value");
+ assert_eq!(
+ run_sync(&args(&["--install-timer", "--url", "http://x"])),
+ 2,
+ "--install-timer takes no other flag"
+ );
+ assert_eq!(run_sync(&args(&["--help"])), 0);
+ assert_eq!(run_sync(&args(&["-h"])), 0);
+}
+
+#[test]
+fn the_hub_command_routes_sync_here() {
+ // `--help` exits 0 only if `sync` is a known subcommand (an unknown one
+ // exits 2), and it does so without touching the network or the cache.
+ assert_eq!(crate::cli::run_hub_cli(&args(&["sync", "--help"])), 0);
+ assert_eq!(crate::cli::run_hub_cli(&args(&["sync", "--bogus"])), 2);
+}
+
+#[test]
+fn the_usage_names_every_flag_and_exit_code() {
+ let usage = sync_usage();
+ for needle in [
+ "--url <node>",
+ "--install-timer",
+ "every 6 hours",
+ "exit codes: 0",
+ "3 a pack",
+ ] {
+ assert!(usage.contains(needle), "usage lacks {needle:?}");
+ }
+ // The budget it names is the one the code enforces, not a typed-in number.
+ assert!(
+ usage.contains(&format!(
+ "the {} disk budget",
+ human_bytes(DISK_BUDGET_BYTES)
+ )),
+ "{usage}"
+ );
+}
diff --git a/engine/crates/xerj-hub/tests/sync_e2e.rs b/engine/crates/xerj-hub/tests/sync_e2e.rs
new file mode 100644
index 000000000..bf770c681
--- /dev/null
+++ b/engine/crates/xerj-hub/tests/sync_e2e.rs
@@ -0,0 +1,687 @@
+//! Corpus Hub v2 — `xerj hub sync` end to end (plan §8 "Tests").
+//!
+//! Real nodes (an ephemeral TCP port each, the way `pack_roundtrip.rs` does), the
+//! real `xerj hub pull`, a real signed catalogue served over HTTP from a throwaway
+//! local "hub", and the real `sync` command — everything except the process
+//! boundary: sync normally runs `pull` as a child of the `xerj` binary, which a
+//! test binary cannot be, so the pull is driven in-process through the same
+//! `run_hub_cli(["pull", …])` entry point instead.
+//!
+//! The story, in one `#[tokio::test]` because it mutates process-global env
+//! (`XERJ_CODE_HOME`, `XERJ_HUB_HOME`, `XERJ_HUB_PUBKEY`, `XERJ_HUB_URL`):
+//!
+//! 1. two packs of one corpus (`mini-c`) — an older pin and a newer one — are built
+//! and signed on a builder node;
+//! 2. a client node pulls the older pin; `def` answers `MINI_LIMIT = 128`;
+//! 3. the hub's catalogue moves to the newer pin and `sync` runs: the pack is
+//! replaced, the OLD directory is gone, `def` now answers `256` (the restore
+//! really happened), `last_sync` is written;
+//! 4. a second sync pulls nothing;
+//! 5. the catalogue moves to a pin whose archive is corrupted: sync exits 3, the
+//! pin that works stays in place and keeps answering, `last_sync` is untouched;
+//! 6. a catalogue whose signature does not verify is refused (exit 1) and changes
+//! nothing.
+
+use std::io::{Read, Write};
+use std::net::{TcpListener, TcpStream};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, SystemTime};
+
+use serde_json::json;
+use sha2::{Digest, Sha256};
+
+use xerj_autoindex::harvest::sign::{generate_keypair, sign_bytes};
+use xerj_hub::catalogue::canonical_json;
+use xerj_hub::node::HubNode;
+use xerj_hub::pack::{self, PackFileEntry};
+use xerj_hub::sync::{run_sync_using, CliOps, Ledger, PullOutcome, SyncOps};
+use xerj_hub::{
+ Catalogue, CatalogueEntry, DefRecord, LicenceClass, PackIndex, PackManifest, PackSource,
+ Sketch, CATALOGUE_FORMAT_VERSION, PACK_FORMAT_VERSION,
+};
+
+/// One pin of the `mini-c` corpus: the older and the newer differ in pin, date,
+/// index prefix and in the value `MINI_LIMIT` carries, so a `def` lookup says
+/// which pack is live.
+#[derive(Clone, Copy)]
+struct PackSpec {
+ pin_sha: &'static str,
+ pin_date: &'static str,
+ prefix: &'static str,
+ limit: &'static str,
+}
+
+const OLD: PackSpec = PackSpec {
+ pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5",
+ pin_date: "2026-10-02",
+ prefix: "xc-mini-c-b1",
+ limit: "128",
+};
+const NEW: PackSpec = PackSpec {
+ pin_sha: "9f3c1a2b4d5e60718293a4b5c6d7e8f901234567",
+ pin_date: "2026-10-09",
+ prefix: "xc-mini-c-b2",
+ limit: "256",
+};
+/// A pin the catalogue will advertise with a corrupted archive behind it.
+const BAD_SHA: &str = "c0ffee0011223344556677889900aabbccddeeff";
+
+impl PackSpec {
+ fn pin8(&self) -> &'static str {
+ &self.pin_sha[..8]
+ }
+}
+
+/// Boot an es-compat node on an ephemeral port with `data_dir` and the given
+/// snapshot-repo allowlist, and wait until it answers.
+async fn spawn_node(data_dir: &Path, allowlist: Vec<String>) -> String {
+ std::fs::create_dir_all(data_dir).unwrap();
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = data_dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ config.limits.snapshot_repo_allowlist = allowlist;
+ let metrics = xerj_common::metrics::Metrics::new().unwrap();
+ let engine = xerj_engine::Engine::new(config.clone()).unwrap();
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ let app = xerj_api::router::build_es_compat_router(state);
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let port = listener.local_addr().unwrap().port();
+ tokio::spawn(async move {
+ let _ = axum::serve(listener, app.into_make_service()).await;
+ });
+ let url = format!("http://127.0.0.1:{port}");
+ let probe = url.clone();
+ tokio::task::spawn_blocking(move || {
+ HubNode::connect(&probe)
+ .unwrap()
+ .wait_until_reachable(Duration::from_secs(30))
+ .unwrap();
+ })
+ .await
+ .unwrap();
+ url
+}
+
+fn sample_defs(spec: &PackSpec) -> Vec<DefRecord> {
+ vec![
+ DefRecord {
+ symbol: "MINI_LIMIT".into(),
+ kind: "define".into(),
+ lang: "c".into(),
+ file: "limits.h".into(),
+ line: 5,
+ text: format!("#define MINI_LIMIT {}", spec.limit),
+ value: Some(spec.limit.into()),
+ },
+ DefRecord {
+ symbol: "mini_clamp".into(),
+ kind: "function".into(),
+ lang: "c".into(),
+ file: "clamp.c".into(),
+ line: 4,
+ text: "int mini_clamp(int value, int lo, int hi)".into(),
+ value: None,
+ },
+ ]
+}
+
+fn manifest(spec: &PackSpec) -> PackManifest {
+ PackManifest {
+ format_version: PACK_FORMAT_VERSION,
+ slug: "mini-c".into(),
+ pin_sha: spec.pin_sha.into(),
+ pin_date: spec.pin_date.into(),
+ built_at: "2026-10-14T00:00:00Z".into(),
+ builder_version: "xerj test".into(),
+ index_prefix: spec.prefix.into(),
+ indices: vec![
+ PackIndex {
+ name: format!("{}-defs", spec.prefix),
+ docs: 2,
+ bytes: 0,
+ },
+ PackIndex {
+ name: format!("{}-docs", spec.prefix),
+ docs: 1,
+ bytes: 0,
+ },
+ ],
+ snapshot_name: format!("mini-c-{}", spec.pin8()),
+ licence_spdx: "MIT".into(),
+ licence_class: LicenceClass::AdaptWithAttribution,
+ sources: vec![PackSource {
+ repo: "mini-c".into(),
+ url: "file:///fixtures/mini-c".into(),
+ sha: spec.pin_sha.into(),
+ licence: "MIT".into(),
+ }],
+ defs_count: 2,
+ files_count: 2,
+ }
+}
+
+/// Create the docs + defs indices for `spec` on the builder node and snapshot
+/// `<prefix>-*` into `location`.
+fn seed_and_snapshot(url: &str, spec: &PackSpec, location: &Path) {
+ let node = HubNode::connect(url).unwrap();
+ let prefix = spec.prefix;
+ node.request_ok(
+ "PUT",
+ &format!("/{prefix}-docs"),
+ Some(&json!({"mappings":{"properties":{"body":{"type":"text"}}}})),
+ )
+ .unwrap();
+ node.bulk_ndjson(
+ &format!("/{prefix}-docs/_bulk"),
+ "{\"index\":{}}\n{\"body\":\"mini c reference\"}\n".to_string(),
+ )
+ .unwrap();
+ node.request_ok(
+ "PUT",
+ &format!("/{prefix}-defs"),
+ Some(&json!({"mappings":{"properties":{
+ "symbol":{"type":"keyword"},
+ "symbol_text":{"type":"text"},
+ "kind":{"type":"keyword"},
+ "lang":{"type":"keyword"},
+ "file":{"type":"keyword"},
+ "line":{"type":"integer"},
+ "text":{"type":"text"},
+ "value":{"type":"keyword"}
+ }}})),
+ )
+ .unwrap();
+ let mut ndjson = String::new();
+ for d in sample_defs(spec) {
+ ndjson.push_str("{\"index\":{}}\n");
+ ndjson.push_str(
+ &json!({
+ "symbol": d.symbol, "symbol_text": d.symbol, "kind": d.kind,
+ "lang": d.lang, "file": d.file, "line": d.line, "text": d.text, "value": d.value,
+ })
+ .to_string(),
+ );
+ ndjson.push('\n');
+ }
+ node.bulk_ndjson(&format!("/{prefix}-defs/_bulk"), ndjson)
+ .unwrap();
+ node.request_ok("POST", &format!("/{prefix}-defs/_refresh"), None)
+ .ok();
+
+ let repo = format!("hub-build-{}", spec.pin8());
+ node.request_ok(
+ "PUT",
+ &format!("/_snapshot/{repo}"),
+ Some(&json!({"type":"fs","settings":{"location": location.to_string_lossy()}})),
+ )
+ .unwrap();
+ node.request_ok(
+ "PUT",
+ &format!(
+ "/_snapshot/{repo}/mini-c-{}?wait_for_completion=true",
+ spec.pin8()
+ ),
+ Some(&json!({"indices": format!("{prefix}-*")})),
+ )
+ .unwrap();
+}
+
+/// Build and sign one pack on the builder node; returns the archive path and its
+/// sha256.
+fn build_pack(builder_url: &str, spec: &PackSpec, out: &Path, seed: &str) -> (PathBuf, String) {
+ let location = out.join(format!("snapshot-{}", spec.pin8()));
+ seed_and_snapshot(builder_url, spec, &location);
+ let files = vec![PackFileEntry {
+ path: "limits.h".into(),
+ lang: "c".into(),
+ bytes: 300,
+ licence: "MIT".into(),
+ }];
+ let archive = out.join(format!("mini-c-{}.hubpack.tar.zst", spec.pin8()));
+ pack::write_pack(
+ &archive,
+ &manifest(spec),
+ &sample_defs(spec),
+ &files,
+ &location,
+ seed,
+ )
+ .unwrap();
+ let sha = sha256_hex(&archive);
+ (archive, sha)
+}
+
+fn sha256_hex(path: &Path) -> String {
+ Sha256::digest(std::fs::read(path).unwrap())
+ .iter()
+ .map(|b| format!("{b:02x}"))
+ .collect()
+}
+
+fn sig_of(archive: &Path) -> PathBuf {
+ let mut s = archive.as_os_str().to_os_string();
+ s.push(".sig");
+ PathBuf::from(s)
+}
+
+/// A catalogue entry whose pack is a `file://` archive.
+fn catalogue_entry(
+ pin_sha: &str,
+ pin_date: &str,
+ archive: &Path,
+ pack_sha256: &str,
+) -> CatalogueEntry {
+ CatalogueEntry {
+ slug: "mini-c".into(),
+ domain: "DEV".into(),
+ topics: vec!["c".into()],
+ ask_for: "a tiny C fixture corpus".into(),
+ pin_sha: pin_sha.into(),
+ pin_date: pin_date.into(),
+ refresh_cadence_days: 7,
+ licence_spdx: "MIT".into(),
+ licence_class: LicenceClass::AdaptWithAttribution,
+ pack_bytes: std::fs::metadata(archive).map(|m| m.len()).unwrap_or(0),
+ pack_url: format!("file://{}", archive.display()),
+ pack_sha256: pack_sha256.into(),
+ sig_url: format!("file://{}", sig_of(archive).display()),
+ dependency_names: vec![],
+ quality_grade: None,
+ term_sketch: Sketch {
+ m_bits: 64,
+ k_hashes: 7,
+ bits_base64: "AAAAAAAAAAA=".into(),
+ },
+ }
+}
+
+/// The signed bytes of a one-corpus catalogue: `(catalogue.json, signature)`.
+fn signed_catalogue(entry: CatalogueEntry, seed: &str) -> (Vec<u8>, String) {
+ let cat = Catalogue {
+ format_version: CATALOGUE_FORMAT_VERSION,
+ generated_at: "2026-10-14T00:00:00Z".into(),
+ corpora: vec![entry],
+ };
+ let bytes = canonical_json(&cat).unwrap();
+ let sig = sign_bytes(&bytes, seed).unwrap();
+ (bytes, sig)
+}
+
+/// A throwaway local "hub": serves whatever catalogue it currently holds at
+/// `/api/v1/catalogue.json` and its signature at `….sig`, for as many requests as
+/// come.
+struct TinyHub {
+ url: String,
+ addr: std::net::SocketAddr,
+ body: Arc<Mutex<(Vec<u8>, String)>>,
+ stop: Arc<AtomicBool>,
+ handle: Option<std::thread::JoinHandle<()>>,
+}
+
+impl TinyHub {
+ fn start(initial: (Vec<u8>, String)) -> TinyHub {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let addr = listener.local_addr().unwrap();
+ let body = Arc::new(Mutex::new(initial));
+ let stop = Arc::new(AtomicBool::new(false));
+ let (b, s) = (body.clone(), stop.clone());
+ let handle = std::thread::spawn(move || {
+ while let Ok((mut stream, _)) = listener.accept() {
+ if s.load(Ordering::SeqCst) {
+ break;
+ }
+ let mut buf = [0u8; 4096];
+ let n = stream.read(&mut buf).unwrap_or(0);
+ let req = String::from_utf8_lossy(&buf[..n]).into_owned();
+ let first = req.lines().next().unwrap_or("");
+ let (cat, sig) = b.lock().unwrap().clone();
+ let payload: Vec<u8> = if first.contains(".sig") {
+ sig.into_bytes()
+ } else {
+ cat
+ };
+ let head = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\
+ Content-Length: {}\r\nConnection: close\r\n\r\n",
+ payload.len()
+ );
+ let _ = stream.write_all(head.as_bytes());
+ let _ = stream.write_all(&payload);
+ let _ = stream.flush();
+ }
+ });
+ TinyHub {
+ url: format!("http://{addr}"),
+ addr,
+ body,
+ stop,
+ handle: Some(handle),
+ }
+ }
+
+ fn serve(&self, bytes: Vec<u8>, sig: String) {
+ *self.body.lock().unwrap() = (bytes, sig);
+ }
+
+ fn shutdown(mut self) {
+ self.stop.store(true, Ordering::SeqCst);
+ let _ = TcpStream::connect(self.addr); // wake the accept loop
+ if let Some(h) = self.handle.take() {
+ h.join().unwrap();
+ }
+ }
+}
+
+/// Drives the real `hub pull` in-process and reads the real ledger, counting the
+/// pulls and noting what was on disk around each one.
+struct InProcessOps {
+ node_url: String,
+ hub_home: PathBuf,
+ code_home: PathBuf,
+ pulls: Arc<AtomicUsize>,
+ /// For each pull: was the OLD pin's directory present before it, and were
+ /// BOTH directories present when it returned (i.e. sync had not yet removed
+ /// the old one).
+ around_pull: Arc<Mutex<Vec<(bool, bool)>>>,
+}
+
+impl SyncOps for InProcessOps {
+ fn pull(&self, slug: &str) -> PullOutcome {
+ self.pulls.fetch_add(1, Ordering::SeqCst);
+ let old = self.hub_home.join("packs").join(slug).join(OLD.pin8());
+ let new = self.hub_home.join("packs").join(slug).join(NEW.pin8());
+ let old_before = old.is_dir();
+ let rc = xerj_hub::cli::run_hub_cli(&[
+ "pull".into(),
+ slug.into(),
+ "--url".into(),
+ self.node_url.clone(),
+ "--quiet".into(),
+ ]);
+ self.around_pull
+ .lock()
+ .unwrap()
+ .push((old_before, old.is_dir() && new.is_dir()));
+ if rc == 0 {
+ PullOutcome::Done
+ } else {
+ PullOutcome::Failed {
+ code: rc,
+ message: format!("hub pull exited {rc}"),
+ }
+ }
+ }
+
+ fn ledger(&self, slug: &str) -> Ledger {
+ // The real ledger reader, pointed at this test's code root.
+ CliOps::with_program("unused", vec![], None, self.code_home.clone()).ledger(slug)
+ }
+}
+
+fn age_file(path: &Path, secs: u64) {
+ let when = SystemTime::now() - Duration::from_secs(secs);
+ std::fs::OpenOptions::new()
+ .write(true)
+ .open(path)
+ .unwrap()
+ .set_modified(when)
+ .unwrap();
+}
+
+fn pin_dirs(hub_home: &Path) -> Vec<String> {
+ let mut v: Vec<String> = std::fs::read_dir(hub_home.join("packs/mini-c"))
+ .map(|rd| {
+ rd.flatten()
+ .map(|e| e.file_name().to_string_lossy().into_owned())
+ .collect()
+ })
+ .unwrap_or_default();
+ v.sort();
+ v
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn sync_replaces_the_pack_with_a_newer_pin_and_the_old_dir_is_gone() {
+ // One canonical work root so every snapshot location is inside the nodes'
+ // allowlist (the engine refuses a repo location outside data_dir or the
+ // allowlist; we configure the root, never weaken the check).
+ let work_tmp = tempfile::tempdir().unwrap();
+ let work = std::fs::canonicalize(work_tmp.path()).unwrap();
+ let out = work.join("out");
+ std::fs::create_dir_all(&out).unwrap();
+ let allowlist = vec![work.to_string_lossy().into_owned()];
+ let (seed, public) = generate_keypair().unwrap();
+
+ // ── 1. Build + sign both pins on the builder node ─────────────────────
+ let builder = spawn_node(&work.join("data-builder"), allowlist.clone()).await;
+ let (old_archive, old_sha, new_archive, new_sha) = {
+ let (builder, out, seed) = (builder.clone(), out.clone(), seed.clone());
+ tokio::task::spawn_blocking(move || {
+ let (oa, os) = build_pack(&builder, &OLD, &out, &seed);
+ let (na, ns) = build_pack(&builder, &NEW, &out, &seed);
+ (oa, os, na, ns)
+ })
+ .await
+ .unwrap()
+ };
+
+ // A corrupted copy of the NEW archive, advertised under a third pin with the
+ // original (now mismatching) sha256 — the way a tampered download looks.
+ let bad_archive = out.join("mini-c-c0ffee00.hubpack.tar.zst");
+ std::fs::copy(&new_archive, &bad_archive).unwrap();
+ std::fs::copy(sig_of(&new_archive), sig_of(&bad_archive)).unwrap();
+ let mut bytes = std::fs::read(&bad_archive).unwrap();
+ let mid = bytes.len() / 2;
+ bytes[mid] ^= 0xff;
+ std::fs::write(&bad_archive, &bytes).unwrap();
+
+ // ── 2. The local hub starts out advertising the OLD pin ───────────────
+ let hub = TinyHub::start(signed_catalogue(
+ catalogue_entry(OLD.pin_sha, OLD.pin_date, &old_archive, &old_sha),
+ &seed,
+ ));
+ let client = spawn_node(&work.join("data-client"), allowlist).await;
+ let hub_home = work.join("hub-home");
+ let code_home = work.join("code-home");
+ std::fs::create_dir_all(&hub_home).unwrap();
+ std::env::set_var("XERJ_CODE_HOME", &code_home);
+ std::env::set_var("XERJ_HUB_HOME", &hub_home);
+ std::env::set_var("XERJ_HUB_PUBKEY", &public);
+ std::env::set_var("XERJ_HUB_URL", &hub.url);
+
+ // The first pull is the ordinary one: nothing cached yet, so sync has no
+ // part in it. Seed the catalogue cache the way a first `ask` would.
+ {
+ let (bytes, _) = signed_catalogue(
+ catalogue_entry(OLD.pin_sha, OLD.pin_date, &old_archive, &old_sha),
+ &seed,
+ );
+ std::fs::write(hub_home.join("catalogue.json"), bytes).unwrap();
+ }
+ let rc = {
+ let client = client.clone();
+ tokio::task::spawn_blocking(move || {
+ xerj_hub::cli::run_hub_cli(&[
+ "pull".into(),
+ "mini-c".into(),
+ "--url".into(),
+ client,
+ "--quiet".into(),
+ ])
+ })
+ .await
+ .unwrap()
+ };
+ assert_eq!(rc, 0, "the older pin pulls cleanly");
+ assert_eq!(pin_dirs(&hub_home), vec![OLD.pin8()]);
+ let limit = |client: String| {
+ tokio::task::spawn_blocking(move || {
+ let node = HubNode::connect(&client).unwrap();
+ let a = xerj_hub::def::lookup(&node, "mini-c", "MINI_LIMIT", 5);
+ (
+ a.hits.first().and_then(|h| h.value.clone()),
+ a.pin_date.clone(),
+ )
+ })
+ };
+ assert_eq!(
+ limit(client.clone()).await.unwrap(),
+ (Some("128".to_string()), OLD.pin_date.to_string()),
+ "the old pin answers 128"
+ );
+
+ // The shared ops factory: every sync below goes through the real command.
+ let pulls = Arc::new(AtomicUsize::new(0));
+ let around_pull = Arc::new(Mutex::new(Vec::new()));
+ let run = {
+ let (client, hub_home, code_home) = (client.clone(), hub_home.clone(), code_home.clone());
+ let (pulls, around_pull) = (pulls.clone(), around_pull.clone());
+ move || {
+ let (client, hub_home, code_home) =
+ (client.clone(), hub_home.clone(), code_home.clone());
+ let (pulls, around_pull) = (pulls.clone(), around_pull.clone());
+ tokio::task::spawn_blocking(move || {
+ run_sync_using(&[], &move |_url| {
+ Ok(Box::new(InProcessOps {
+ node_url: client.clone(),
+ hub_home: hub_home.clone(),
+ code_home: code_home.clone(),
+ pulls: pulls.clone(),
+ around_pull: around_pull.clone(),
+ }) as Box<dyn SyncOps>)
+ })
+ })
+ }
+ };
+
+ // ── 3. The hub moves to the NEW pin; sync replaces the pack ───────────
+ let (bytes, sig) = signed_catalogue(
+ catalogue_entry(NEW.pin_sha, NEW.pin_date, &new_archive, &new_sha),
+ &seed,
+ );
+ hub.serve(bytes, sig);
+ assert!(
+ !hub_home.join("last_sync").exists(),
+ "precondition: sync has never run"
+ );
+
+ assert_eq!(run().await.unwrap(), 0, "a sync with a newer pin succeeds");
+
+ assert_eq!(
+ pin_dirs(&hub_home),
+ vec![NEW.pin8()],
+ "the new pin replaced the old one and the old directory is gone"
+ );
+ assert!(hub_home
+ .join("packs/mini-c")
+ .join(NEW.pin8())
+ .join("pack.json")
+ .is_file());
+ assert_eq!(pulls.load(Ordering::SeqCst), 1, "exactly one pull");
+ assert_eq!(
+ *around_pull.lock().unwrap(),
+ vec![(true, true)],
+ "the old pin was on disk before the pull and STILL on disk when it returned: \
+ it is removed only after the new pin has restored"
+ );
+ let ledger: serde_json::Value = serde_json::from_slice(
+ &std::fs::read(code_home.join("state").join("mini-c.json")).unwrap(),
+ )
+ .unwrap();
+ assert_eq!(ledger["source"], json!("hubpack"));
+ assert_eq!(ledger["pin_sha"], json!(NEW.pin_sha));
+ assert_eq!(ledger["index_prefix"], json!(NEW.prefix));
+ assert_eq!(
+ limit(client.clone()).await.unwrap(),
+ (Some("256".to_string()), NEW.pin_date.to_string()),
+ "the new pin is the one answering: the restore really ran"
+ );
+ let stamp = std::fs::read_to_string(hub_home.join("last_sync")).unwrap();
+ chrono::DateTime::parse_from_rfc3339(stamp.trim()).expect("last_sync is RFC 3339");
+ assert!(
+ hub_home.join("catalogue.json").is_file(),
+ "the refreshed catalogue was cached"
+ );
+
+ // ── 4. Nothing moved: a second sync pulls nothing ─────────────────────
+ assert_eq!(run().await.unwrap(), 0);
+ assert_eq!(
+ pulls.load(Ordering::SeqCst),
+ 1,
+ "a pack already at the catalogue's pin costs no download"
+ );
+ assert_eq!(pin_dirs(&hub_home), vec![NEW.pin8()]);
+
+ // ── 5. The hub advertises a pin whose archive is corrupt ──────────────
+ age_file(&hub_home.join("last_sync"), 3 * 86_400);
+ let before = std::fs::metadata(hub_home.join("last_sync"))
+ .unwrap()
+ .modified()
+ .unwrap();
+ let (bytes, sig) = signed_catalogue(
+ catalogue_entry(BAD_SHA, "2026-10-16", &bad_archive, &new_sha),
+ &seed,
+ );
+ hub.serve(bytes, sig);
+
+ assert_eq!(
+ run().await.unwrap(),
+ 3,
+ "a pack that fails verification is exit 3"
+ );
+ assert_eq!(
+ pin_dirs(&hub_home),
+ vec![NEW.pin8()],
+ "nothing was extracted for the bad pin and the working pin was not touched"
+ );
+ assert!(!hub_home.join("packs/mini-c/c0ffee00.tmp").exists());
+ assert_eq!(
+ limit(client.clone()).await.unwrap().0,
+ Some("256".to_string()),
+ "the pin that works keeps answering"
+ );
+ assert_eq!(
+ std::fs::metadata(hub_home.join("last_sync"))
+ .unwrap()
+ .modified()
+ .unwrap(),
+ before,
+ "a failed sync does not claim to have synced"
+ );
+
+ // ── 6. A catalogue that does not verify is refused, and changes nothing ─
+ let (good, good_sig) = signed_catalogue(
+ catalogue_entry(OLD.pin_sha, OLD.pin_date, &old_archive, &old_sha),
+ &seed,
+ );
+ let mut tampered = good.clone();
+ let mid = tampered.len() / 2;
+ tampered[mid] ^= 0x01;
+ hub.serve(tampered, good_sig);
+ let pulls_before = pulls.load(Ordering::SeqCst);
+
+ assert_eq!(
+ run().await.unwrap(),
+ 1,
+ "a catalogue whose signature fails is a refresh failure, exit 1"
+ );
+ assert_eq!(
+ pulls.load(Ordering::SeqCst),
+ pulls_before,
+ "nothing was pulled"
+ );
+ assert_eq!(pin_dirs(&hub_home), vec![NEW.pin8()]);
+
+ // Clean up so the env does not leak and the local hub thread ends.
+ hub.shutdown();
+ for k in [
+ "XERJ_CODE_HOME",
+ "XERJ_HUB_HOME",
+ "XERJ_HUB_PUBKEY",
+ "XERJ_HUB_URL",
+ ] {
+ std::env::remove_var(k);
+ }
+}