Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(hub): refuse excluded and oversize packs in pull itself, make the pull lock the OS's, extract atomically, route a definition question on its symbol

xerj-org34aeb70c529f
diff --git a/engine/crates/xerj-hub/src/ask.rs b/engine/crates/xerj-hub/src/ask.rs
index cfe3ec407..6d2ac56bf 100644
--- a/engine/crates/xerj-hub/src/ask.rs
+++ b/engine/crates/xerj-hub/src/ask.rs
@@ -16,9 +16,11 @@
//! 4. **No pack, hosted allowed** → answer through the hosted hub now, then (unless
//! `--no-fetch`) start a detached `xerj hub pull` for next time — subject to the
//! pull policy ([`pull_policy`]).
-//! 5. **No pack, no hosted** → `--offline` (or `--no-fetch`) exits 1 naming the
-//! pack to pull; otherwise pull synchronously (same policy) and answer
-//! locally.
+//! 5. **No pack, no hosted** → the pull policy decides first ([`pull_policy`]): an
+//! excluded corpus is never pulled and a pack over the size limit is not
+//! downloaded, whatever else was asked, and neither is ever answered with a
+//! command to run. Then `--offline` (or `--no-fetch`) exits 1 naming the pack to
+//! pull; otherwise pull synchronously and answer locally.
//! 6. **Disk budget** — before any pull, least-recently-used packs are evicted
//! until the pack cache fits ([`evict_to_budget`], called from `hub pull`).
//! 7. The §1 renderer prints the [`Answer`]; `--json` prints the struct.
@@ -49,6 +51,7 @@
//! | 2 | usage error; unknown `--corpus`; a pull rejected the slug |
//! | 3 | a pulled pack failed verification (sha256 / signature / format) |
//! | 4 | ambiguous — re-run with `--corpus` (the code `xerj autoindex` uses for "needs a decision") |
+//! | 5 | this build has no hub public key, so no pack can be verified (nothing was downloaded) |
use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
@@ -67,8 +70,8 @@ use xerj_common::xccode::{self, licence, passage, state, CodeParams, XcHttp};
use crate::catalogue::load_catalogue;
use crate::node::{self, HubNode, NodeLauncher, ResolvedNode};
-use crate::router::{route, HostedSearch, Routing};
-use crate::{cli, def, lockfile};
+use crate::router::{route_ranked, HostedSearch, Routing};
+use crate::{cli, def, lockfile, sketch};
/// Answered with at least one hit.
pub const EXIT_OK: i32 = 0;
@@ -80,6 +83,9 @@ pub const EXIT_USAGE: i32 = 2;
pub const EXIT_UNVERIFIED: i32 = 3;
/// More than one corpus matches: the caller must choose (`--corpus`).
pub const EXIT_NEEDS_DECISION: i32 = 4;
+/// A pull was refused because this build has no hub public key: not a pack that
+/// failed verification ([`EXIT_UNVERIFIED`]) but one that cannot be judged at all.
+pub const EXIT_NO_KEY: i32 = 5;
/// Printed (stderr) when the router finds nothing credible.
pub const NO_CORPUS_MESSAGE: &str = "no hub corpus matches; see xerj.org/llms.txt";
@@ -333,16 +339,50 @@ pub fn megabytes(bytes: u64) -> String {
format!("{:.1}", bytes as f64 / (1024.0 * 1024.0))
}
-/// `run: xerj hub pull <slug> (<MB> MB)` — what to type for a pack too large to
-/// pull automatically.
+/// `run: xerj hub pull <slug> (<MB> MB)` — the command that fetches this pack.
+///
+/// Only ever offered for a pack the pull policy allows ([`PullPolicy::Allow`]):
+/// `xerj hub pull` refuses an excluded corpus and a pack over the size limit, so a
+/// hint naming it would send the reader to a command that fails — or, worse, one
+/// they would be tempted to get around. Callers check [`pull_policy`] first.
pub fn pull_hint(entry: &CatalogueEntry) -> String {
format!(
"run: xerj hub pull {} ({} MB)",
- entry.slug,
+ printable(&entry.slug),
megabytes(entry.pack_bytes)
)
}
+/// A catalogue- or network-supplied string made safe to print on one terminal
+/// line. The catalogue is accepted unverified until the hub key ships, and a slug
+/// or an `ask_for` is printed for an agent (and a person) to read: a control
+/// character in it is an escape sequence, a forged line, or a rewritten screen.
+/// Every control character — ESC, CR, LF, BEL, NUL, DEL, the C1 range — the
+/// Unicode line/paragraph separators and the bidirectional overrides (which
+/// reorder how a slug is *displayed*) become a space; runs of spaces collapse.
+pub fn printable(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ let mut last_space = false;
+ for c in s.chars() {
+ let hostile = c.is_control()
+ || matches!(
+ c,
+ '\u{2028}' | '\u{2029}' | '\u{200E}' | '\u{200F}' | '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}'
+ );
+ let c = if hostile { ' ' } else { c };
+ if c == ' ' {
+ if last_space {
+ continue;
+ }
+ last_space = true;
+ } else {
+ last_space = false;
+ }
+ out.push(c);
+ }
+ out.trim().to_string()
+}
+
/// `approach-only licence (<spdx>): read, do not copy`.
pub fn approach_only_warning(spdx: &str) -> String {
format!(
@@ -467,7 +507,7 @@ pub fn ask(args: &AskArgs, deps: &AskDeps<'_>) -> AskOutcome {
};
// Step 2: route.
- let Some(slug) = select_corpus(args, deps, hosted, &mut out) else {
+ let Some(slug) = select_corpus(args, deps, hosted, symbol.as_deref(), &mut out) else {
return out;
};
let entry = deps.catalogue.corpora.iter().find(|e| e.slug == slug);
@@ -489,20 +529,51 @@ pub fn ask(args: &AskArgs, deps: &AskDeps<'_>) -> AskOutcome {
// Step 4: hosted answers now; the pull happens behind it.
if let Some(h) = hosted {
match h.answer(&req) {
- Ok(answer) => return finish_hosted(args, deps, entry, answer, out),
+ Ok(answer) => return finish_hosted(args, deps, &slug, entry, answer, out),
Err(e) => out.stderr.push(format!("hosted answer failed: {e:#}")),
}
}
// Step 5: neither a pack nor a hosted answer.
+ let shown = printable(&slug);
let Some(entry) = entry else {
out.stderr.push(format!(
- "corpus '{slug}' is not in the catalogue, so it cannot be pulled, and no hosted \
+ "corpus '{shown}' is not in the catalogue, so it cannot be pulled, and no hosted \
answer is available"
));
out.exit = EXIT_NO_ANSWER;
return out;
};
+ // The pull policy is decided FIRST, before anything about how this run was asked
+ // to behave. Whether a pack may be pulled at all does not depend on `--offline`
+ // or `--no-fetch`; and what is printed below must never be a command that
+ // `xerj hub pull` refuses — or, for an excluded licence class, a way round
+ // "never pulled". (The offline branch used to run before the policy and print
+ // `run: xerj hub pull <slug>` for an excluded corpus; `hub pull` itself had no
+ // licence check, so an agent that followed the hint pulled it.) An excluded
+ // corpus and an oversize pack get a plain no.
+ match pull_policy(entry) {
+ PullPolicy::Excluded => {
+ out.stderr.push(format!(
+ "'{shown}' has an excluded licence class ({}) and is never pulled; no hosted \
+ answer is available",
+ printable(&entry.licence_spdx)
+ ));
+ out.exit = EXIT_NO_ANSWER;
+ return out;
+ }
+ PullPolicy::TooLarge => {
+ out.stderr.push(format!(
+ "'{shown}' is {} MB, over the {} MB limit for pulling a pack, so it is not \
+ downloaded; no hosted answer is available",
+ megabytes(entry.pack_bytes),
+ megabytes(AUTO_PULL_MAX_BYTES)
+ ));
+ out.exit = EXIT_NO_ANSWER;
+ return out;
+ }
+ PullPolicy::Allow => {}
+ }
if args.offline || args.no_fetch {
let flag = if args.offline {
"--offline"
@@ -510,41 +581,72 @@ pub fn ask(args: &AskArgs, deps: &AskDeps<'_>) -> AskOutcome {
"--no-fetch"
};
out.stderr.push(format!(
- "no local pack for '{slug}' and {flag} forbids downloading it; {}",
+ "no local pack for '{shown}' and {flag} forbids downloading it; {}",
pull_hint(entry)
));
out.exit = EXIT_NO_ANSWER;
return out;
}
- match pull_policy(entry) {
- PullPolicy::Excluded => {
- out.stderr.push(format!(
- "'{slug}' has an excluded licence class ({}) and is never pulled; no hosted \
- answer is available",
- entry.licence_spdx
- ));
- out.exit = EXIT_NO_ANSWER;
+ match deps.backend.pull_now(entry) {
+ Ok(()) => answer_locally(args, deps, &req, Some(entry), out),
+ Err(f) => {
+ out.stderr.push(f.message);
+ out.exit = f.code;
out
}
- PullPolicy::TooLarge => {
- out.stderr.push(format!(
- "'{slug}' is {} MB, over the {} MB automatic pull limit, and no hosted answer \
- is available; {}",
- megabytes(entry.pack_bytes),
- megabytes(AUTO_PULL_MAX_BYTES),
- pull_hint(entry)
- ));
- out.exit = EXIT_NO_ANSWER;
- out
+ }
+}
+
+/// A definition name found in the term sketches of more corpora than this names no
+/// one corpus (`main`, `init`, `NULL`): it is not evidence, and listing every
+/// corpus that defines it would only be noise.
+const MAX_SYMBOL_CANDIDATES: usize = 5;
+/// How many closest corpora a "nothing matches" answer names.
+const CLOSEST_SHOWN: usize = 3;
+
+/// The corpora whose term sketch holds `symbol` verbatim, by slug. The sketch is
+/// built from every definition's symbol in its own case, so a hit means the corpus
+/// defines exactly that name (a Bloom filter's false-positive rate at the planned
+/// sizing is around 1e-8). Empty when the name is too common to point anywhere
+/// ([`MAX_SYMBOL_CANDIDATES`]).
+fn symbol_hits(cat: &Catalogue, symbol: &str) -> Vec<String> {
+ let mut hits: Vec<String> = cat
+ .corpora
+ .iter()
+ .filter(|e| sketch::contains(&e.term_sketch, symbol))
+ .map(|e| e.slug.clone())
+ .collect();
+ if hits.len() > MAX_SYMBOL_CANDIDATES {
+ return Vec::new();
+ }
+ hits.sort();
+ hits
+}
+
+/// Fold the exact-symbol evidence into the verdict the question's words reached.
+///
+/// A confident verdict stands: the question (or the project's own dependencies)
+/// already decided, and a symbol that merely also exists elsewhere does not
+/// override that. Otherwise the symbol decides what the words could not: of an
+/// ambiguous list it keeps the candidates that define the name, and when the words
+/// found nothing credible it proposes the corpora that define it. One such corpus
+/// is a confident answer (an exact name in one corpus's definitions is strong
+/// evidence — the question's own words, diluted by every stop word around the
+/// name, never could reach that score for a `what is the value of NAME` question);
+/// several are ambiguous.
+fn with_symbol_evidence(routing: Routing, hits: &[String]) -> Routing {
+ let pick = |candidates: Vec<String>| match candidates.len() {
+ 0 => None,
+ 1 => Some(Routing::Confident(candidates[0].clone())),
+ _ => Some(Routing::Ambiguous(candidates)),
+ };
+ match routing {
+ Routing::Confident(slug) => Routing::Confident(slug),
+ Routing::Ambiguous(slugs) => {
+ let both: Vec<String> = slugs.iter().filter(|s| hits.contains(s)).cloned().collect();
+ pick(both).unwrap_or(Routing::Ambiguous(slugs))
}
- PullPolicy::Allow => match deps.backend.pull_now(entry) {
- Ok(()) => answer_locally(args, deps, &req, Some(entry), out),
- Err(f) => {
- out.stderr.push(f.message);
- out.exit = f.code;
- out
- }
- },
+ Routing::NoCorpus => pick(hits.to_vec()).unwrap_or(Routing::NoCorpus),
}
}
@@ -554,6 +656,7 @@ fn select_corpus(
args: &AskArgs,
deps: &AskDeps<'_>,
hosted: Option<&dyn HostedHub>,
+ symbol: Option<&str>,
out: &mut AskOutcome,
) -> Option<String> {
if let Some(c) = &args.corpus {
@@ -561,27 +664,37 @@ fn select_corpus(
return Some(c.clone());
}
out.stderr.push(format!(
- "unknown corpus '{c}' (catalogue generated {})",
- deps.catalogue.generated_at
+ "unknown corpus '{}' (catalogue generated {})",
+ printable(c),
+ printable(&deps.catalogue.generated_at)
));
out.exit = EXIT_USAGE;
return None;
}
let cached = cached_slugs(deps.hub_home);
+ // A definition lookup names its symbol: route on the symbol itself as well as
+ // on the sentence around it.
+ let by_symbol = symbol
+ .map(|s| symbol_hits(deps.catalogue, s.trim()))
+ .unwrap_or_default();
// Route on the catalogue alone first. The hosted node is asked only when
// that is not decisive, so a question a local pack can answer never leaves
// the machine.
- let mut routing = route(&args.question, deps.cwd, deps.catalogue, None, &cached);
+ let (routing, mut ranked) =
+ route_ranked(&args.question, deps.cwd, deps.catalogue, None, &cached);
+ let mut routing = with_symbol_evidence(routing, &by_symbol);
if !matches!(routing, Routing::Confident(_)) {
if let Some(h) = hosted {
- routing = route(
+ let (hosted_routing, hosted_ranked) = route_ranked(
&args.question,
deps.cwd,
deps.catalogue,
Some(&SearchOnly(h)),
&cached,
);
+ routing = with_symbol_evidence(hosted_routing, &by_symbol);
+ ranked = hosted_ranked;
}
}
match routing {
@@ -595,9 +708,9 @@ fn select_corpus(
.corpora
.iter()
.find(|e| &e.slug == s)
- .map(|e| e.ask_for.replace(['\n', '\r'], " "))
+ .map(|e| printable(&e.ask_for))
.unwrap_or_default();
- format!("{s} — {ask_for}")
+ format!("{} — {ask_for}", printable(s))
})
.collect();
out.stdout = lines.join("\n");
@@ -608,6 +721,21 @@ fn select_corpus(
}
Routing::NoCorpus => {
out.stderr.push(NO_CORPUS_MESSAGE.to_string());
+ // Nothing was credible, but something may have been close: name the
+ // nearest few so the asker can pick one with `--corpus` instead of
+ // starting over from "no".
+ let closest: Vec<String> = ranked
+ .iter()
+ .filter(|(slug, _)| deps.catalogue.corpora.iter().any(|e| &e.slug == slug))
+ .take(CLOSEST_SHOWN)
+ .map(|(slug, _)| printable(slug))
+ .collect();
+ if !closest.is_empty() {
+ out.stderr.push(format!(
+ "closest corpora: {}; re-run with --corpus <slug> to ask one of them",
+ closest.join(", ")
+ ));
+ }
out.exit = EXIT_NO_ANSWER;
None
}
@@ -640,10 +768,15 @@ fn answer_locally(
fn finish_hosted(
args: &AskArgs,
deps: &AskDeps<'_>,
+ slug: &str,
entry: Option<&CatalogueEntry>,
mut answer: Answer,
mut out: AskOutcome,
) -> AskOutcome {
+ // The answer is attributed to the corpus this run asked about, whatever the
+ // response says it is: "always names the corpus" must be true of what was
+ // asked, not of what a response claims. (The flow below also pulls `slug`.)
+ answer.corpus = slug.to_string();
answer.answered_from = AnsweredFrom::Hosted;
let mut notes: Vec<String> = Vec::new();
match entry {
@@ -653,24 +786,31 @@ fn finish_hosted(
match deps.backend.pull_in_background(&e.slug) {
Ok(BackgroundPull::Started { pid }) => out.stderr.push(format!(
"pulling '{}' in the background (pid {pid}); the next answer is local",
- e.slug
+ printable(&e.slug)
)),
Ok(BackgroundPull::AlreadyRunning { .. }) => {}
- Err(msg) => out
- .stderr
- .push(format!("could not start the background pull: {msg}")),
+ Err(msg) => out.stderr.push(format!(
+ "could not start the background pull: {}",
+ printable(&msg)
+ )),
}
}
}
- PullPolicy::TooLarge => notes.push(pull_hint(e)),
+ // No command to run for these: `xerj hub pull` refuses both.
+ PullPolicy::TooLarge => notes.push(format!(
+ "the pack is {} MB, over the {} MB limit for pulling a pack: this answer is \
+ from the hosted hub only",
+ megabytes(e.pack_bytes),
+ megabytes(AUTO_PULL_MAX_BYTES)
+ )),
PullPolicy::Excluded => notes.push(format!(
"excluded licence ({}): this corpus is never pulled",
- e.licence_spdx
+ printable(&e.licence_spdx)
)),
},
None => notes.push(format!(
"corpus '{}' is not in the local catalogue; it was not pulled",
- answer.corpus
+ printable(slug)
)),
}
finalize(args, entry, answer, notes, &mut out);
@@ -1239,6 +1379,14 @@ impl Backend for RealBackend {
}
fn pull_now(&self, entry: &CatalogueEntry) -> Result<(), PullFailure> {
+ // Everything that can refuse this pull — the licence policy, the size cap, a
+ // malformed slug or pin, no key to verify with — is decided BEFORE a node is
+ // started for it. A pull that is going to be refused must not first boot a
+ // whole node.
+ let plan = cli::plan_pull_entry(entry).map_err(|e| PullFailure {
+ code: e.code,
+ message: format!("pull of '{}' failed: {}", printable(&entry.slug), e.message),
+ })?;
let node = self.node().map_err(|message| PullFailure {
code: EXIT_NO_ANSWER,
message,
@@ -1248,7 +1396,7 @@ impl Backend for RealBackend {
entry.slug,
megabytes(entry.pack_bytes)
);
- match cli::pull(&entry.slug, &node.0.url, node.0.managed, false) {
+ match cli::pull_with(&plan, &node.0.url, node.0.managed, false) {
Ok(cli::Pulled::Already) => Ok(()),
Ok(cli::Pulled::Done(line)) => {
eprintln!("xerj hub: {line}");
@@ -1508,6 +1656,9 @@ mod tests {
struct StubHosted {
found: Vec<(String, f32)>,
fail: bool,
+ /// Attribute the answer to this corpus instead of the one asked about —
+ /// what a buggy or hostile hosted node could do.
+ claim: Option<String>,
searches: Cell<usize>,
answers: Cell<usize>,
}
@@ -1516,6 +1667,7 @@ mod tests {
StubHosted {
found: vec![],
fail: false,
+ claim: None,
searches: Cell::new(0),
answers: Cell::new(0),
}
@@ -1534,7 +1686,11 @@ mod tests {
anyhow::bail!("hosted node unreachable");
}
// Deliberately mislabelled: `ask` must stamp it `Hosted` itself.
- Ok(canned(req.slug, AnsweredFrom::LocalPack, req.mode))
+ let mut a = canned(req.slug, AnsweredFrom::LocalPack, req.mode);
+ if let Some(other) = &self.claim {
+ a.corpus = other.clone();
+ }
+ Ok(a)
}
}
@@ -1835,16 +1991,20 @@ mod tests {
);
assert_eq!(out.exit, EXIT_OK);
assert!(!backend.called("pull_"), "{:?}", backend.calls());
+ // The note names the size, and is in the structured answer too ...
+ assert!(out.stdout.contains("60.0 MB"), "{}", out.stdout);
+ let ans = out.answer.unwrap();
assert!(
- out.stdout.contains("run: xerj hub pull lz4 (60.0 MB)"),
- "{}",
- out.stdout
+ ans.warnings
+ .iter()
+ .any(|w| w.contains("60.0 MB") && w.contains("hosted hub only")),
+ "{:?}",
+ ans.warnings
);
- // The same hint is in the structured answer.
- let ans = out.answer.unwrap();
- assert!(ans
- .warnings
- .contains(&"run: xerj hub pull lz4 (60.0 MB)".to_string()));
+ // ... but it is not a command: `xerj hub pull` refuses an oversize pack, so
+ // a `run:` line would send the reader to a command that fails.
+ assert!(!out.stdout.contains("xerj hub pull"), "{}", out.stdout);
+ assert!(!out.stdout.contains("run:"), "{}", out.stdout);
}
#[test]
@@ -2011,8 +2171,8 @@ mod tests {
assert!(!backend.called("pull_now"), "{:?}", backend.calls());
assert!(out.stderr[0].contains("80.0 MB"), "{:?}", out.stderr);
assert!(
- out.stderr[0].contains("run: xerj hub pull lz4"),
- "{:?}",
+ !out.stderr[0].contains("xerj hub pull") && !out.stderr[0].contains("run:"),
+ "no command for a pack `hub pull` refuses: {:?}",
out.stderr
);
@@ -2044,6 +2204,435 @@ mod tests {
assert!(out.stdout.contains("answered from: hosted hub"));
}
+ // ── the pull policy comes before --offline / --no-fetch ─────────────────
+
+ /// The two refusals that must never be answered with a command.
+ fn no_pull_command(out: &AskOutcome) {
+ let all = format!("{}\n{}", out.stdout, out.stderr.join("\n"));
+ assert!(!all.contains("xerj hub pull"), "no pull hint at all: {all}");
+ assert!(!all.contains("run:"), "no command to run: {all}");
+ }
+
+ #[test]
+ fn offline_never_hints_a_pull_for_an_excluded_corpus() {
+ // The bypass: `--offline` returned before the pull policy and printed
+ // `run: xerj hub pull <slug>` for an excluded corpus; `hub pull` then had no
+ // licence check of its own.
+ let rig = Rig::new();
+ let cat = catalogue(vec![lz4(12 * 1024 * 1024, LicenceClass::Excluded)]);
+ for (flag, name) in [(true, "--offline"), (false, "--no-fetch")] {
+ let backend = FakeBackend::new(false);
+ let mut a = args("q", Some("lz4"));
+ if flag {
+ a.offline = true;
+ } else {
+ a.no_fetch = true;
+ }
+ let out = rig.run(&a, &cat, None, &backend);
+ assert_eq!(out.exit, EXIT_NO_ANSWER, "{name}");
+ assert!(
+ out.stderr[0].contains("excluded licence class")
+ && out.stderr[0].contains("never pulled"),
+ "{name}: {:?}",
+ out.stderr
+ );
+ no_pull_command(&out);
+ assert_eq!(backend.calls(), ["ready:lz4"], "{name}: nothing pulled");
+ }
+ }
+
+ #[test]
+ fn offline_never_hints_a_pull_for_a_pack_over_the_limit() {
+ let rig = Rig::new();
+ let cat = catalogue(vec![lz4(
+ 80 * 1024 * 1024,
+ LicenceClass::AdaptWithAttribution,
+ )]);
+ for offline in [true, false] {
+ let backend = FakeBackend::new(false);
+ let mut a = args("q", Some("lz4"));
+ if offline {
+ a.offline = true;
+ } else {
+ a.no_fetch = true;
+ }
+ let out = rig.run(&a, &cat, None, &backend);
+ assert_eq!(out.exit, EXIT_NO_ANSWER);
+ assert!(out.stderr[0].contains("80.0 MB"), "{:?}", out.stderr);
+ no_pull_command(&out);
+ assert_eq!(backend.calls(), ["ready:lz4"]);
+ }
+ }
+
+ #[test]
+ fn offline_still_hints_the_pull_for_a_pack_that_may_be_pulled() {
+ // The hint is for the case where it works.
+ let rig = Rig::new();
+ for class in [
+ LicenceClass::AdaptWithAttribution,
+ LicenceClass::ApproachOnly,
+ ] {
+ let cat = catalogue(vec![lz4(12 * 1024 * 1024, class)]);
+ let mut a = args("q", Some("lz4"));
+ a.offline = true;
+ let out = rig.run(&a, &cat, None, &FakeBackend::new(false));
+ assert!(
+ out.stderr
+ .join("\n")
+ .contains("run: xerj hub pull lz4 (12.0 MB)"),
+ "{:?}",
+ out.stderr
+ );
+ }
+ }
+
+ #[test]
+ fn a_hosted_excluded_answer_carries_no_pull_command_either() {
+ let rig = Rig::new();
+ let cat = catalogue(vec![lz4(80 * 1024 * 1024, LicenceClass::Excluded)]);
+ let backend = FakeBackend::new(false);
+ let out = rig.run(
+ &args("q", Some("lz4")),
+ &cat,
+ Some(&StubHosted::new()),
+ &backend,
+ );
+ assert_eq!(out.exit, EXIT_OK);
+ assert!(out.stdout.contains("never pulled"), "{}", out.stdout);
+ no_pull_command(&out);
+ assert!(!backend.called("pull_"));
+ }
+
+ // ── hosted answers are attributed to the corpus that was asked ──────────
+
+ #[test]
+ fn a_hosted_answer_is_attributed_to_the_corpus_asked_about_not_the_one_it_claims() {
+ let rig = Rig::new();
+ let cat = catalogue(vec![lz4(1024, LicenceClass::AdaptWithAttribution)]);
+ let mut hosted = StubHosted::new();
+ hosted.claim = Some("some-other-corpus".to_string());
+ let out = rig.run(
+ &args("q", Some("lz4")),
+ &cat,
+ Some(&hosted),
+ &FakeBackend::new(false),
+ );
+ assert_eq!(out.exit, EXIT_OK, "{:?}", out.stderr);
+ assert_eq!(out.answer.as_ref().unwrap().corpus, "lz4");
+ assert!(out.stdout.starts_with("lz4 pin "), "{}", out.stdout);
+ assert!(!out.stdout.contains("some-other-corpus"), "{}", out.stdout);
+ }
+
+ // ── nothing a catalogue says reaches the terminal as control codes ──────
+
+ #[test]
+ fn printable_removes_control_characters_and_collapses_what_is_left() {
+ assert_eq!(
+ printable("an agent working on lz4"),
+ "an agent working on lz4"
+ );
+ assert_eq!(printable("a\nb\r\nc\td"), "a b c d");
+ // An escape sequence cannot clear the screen or recolour the line.
+ assert_eq!(printable("x\u{1b}[2J\u{1b}[31my"), "x [2J [31my");
+ assert!(!printable("\u{1b}]0;title\u{7}z").contains('\u{1b}'));
+ assert!(!printable("nul\0byte\u{7f}del\u{85}c1")
+ .chars()
+ .any(char::is_control));
+ // Bidirectional overrides and line separators that reorder or split a line.
+ let sneaky = printable("safe\u{202E}evil\u{2028}next\u{2066}x");
+ assert!(
+ !sneaky
+ .chars()
+ .any(|c| matches!(c, '\u{202E}' | '\u{2028}' | '\u{2066}')),
+ "{sneaky:?}"
+ );
+ assert_eq!(printable(" padded "), "padded");
+ assert_eq!(printable(""), "");
+ // Ordinary non-ASCII text is untouched.
+ assert_eq!(printable("sürf — 日本語"), "sürf — 日本語");
+ }
+
+ #[test]
+ fn candidate_lines_never_carry_terminal_control_codes() {
+ let rig = Rig::new();
+ let mut a = entry(
+ "alpha",
+ 1,
+ LicenceClass::AdaptWithAttribution,
+ &["tokio", "runtime"],
+ );
+ a.ask_for = "fine\n\u{1b}[2Jfake line\r\nanother".to_string();
+ let mut b = entry(
+ "beta",
+ 1,
+ LicenceClass::AdaptWithAttribution,
+ &["tokio", "runtime"],
+ );
+ b.slug = "be\u{1b}[31mta".to_string();
+ let cat = catalogue(vec![a, b]);
+ let out = rig.run(
+ &args("tokio runtime", None),
+ &cat,
+ None,
+ &FakeBackend::new(true),
+ );
+ assert_eq!(out.exit, EXIT_NEEDS_DECISION, "{out:?}");
+ assert_eq!(
+ out.stdout.lines().count(),
+ 2,
+ "one line per candidate: {:?}",
+ out.stdout
+ );
+ assert!(
+ !out.stdout.chars().any(|c| c.is_control() && c != '\n'),
+ "{:?}",
+ out.stdout
+ );
+ }
+
+ // ── routing on the symbol, and naming the closest on a miss ─────────────
+
+ /// A catalogue where `lz4` defines `LZ4_VERSION_RELEASE` and the fillers do not.
+ fn symbol_catalogue() -> Catalogue {
+ catalogue(with_fillers(vec![
+ entry(
+ "lz4",
+ 1024,
+ LicenceClass::AdaptWithAttribution,
+ &[
+ "lz4",
+ "block",
+ "decoder",
+ "LZ4_VERSION_RELEASE",
+ "LZ4_compress_default",
+ ],
+ ),
+ entry(
+ "zstd",
+ 1024,
+ LicenceClass::AdaptWithAttribution,
+ &["zstd", "frame", "ZSTD_VERSION_RELEASE"],
+ ),
+ ]))
+ }
+
+ #[test]
+ fn a_definition_question_routes_on_the_symbol_the_words_around_it_cannot() {
+ let rig = Rig::new();
+ let cat = symbol_catalogue();
+ let q = "what is the exact value of LZ4_VERSION_RELEASE";
+ // The sentence alone: the symbol is one of six words and its lower-case
+ // twin never matches, so the score cannot reach "confident".
+ let cached: Vec<String> = vec![];
+ assert!(
+ matches!(
+ crate::router::route(q, rig.cwd.path(), &cat, None, &cached),
+ Routing::NoCorpus
+ ),
+ "the words alone do not route"
+ );
+ // The flow knows the question is about that symbol, and the symbol is
+ // defined by exactly one corpus.
+ let backend = FakeBackend::new(true);
+ let out = rig.run(&args(q, None), &cat, None, &backend);
+ assert_eq!(out.exit, EXIT_OK, "{:?}", out.stderr);
+ assert_eq!(backend.calls(), ["ready:lz4", "answer_local:lz4"]);
+
+ // `--def SYM` with an unrelated sentence routes the same way.
+ let mut a = args("tell me about it", None);
+ a.mode = ModeRequest::Def("LZ4_compress_default".to_string());
+ let backend = FakeBackend::new(true);
+ let out = rig.run(&a, &cat, None, &backend);
+ assert_eq!(out.exit, EXIT_OK, "{:?}", out.stderr);
+ assert_eq!(backend.calls(), ["ready:lz4", "answer_local:lz4"]);
+
+ // And a bare identifier, the shortest definition question.
+ let backend = FakeBackend::new(true);
+ let out = rig.run(&args("ZSTD_VERSION_RELEASE", None), &cat, None, &backend);
+ assert_eq!(out.exit, EXIT_OK, "{:?}", out.stderr);
+ assert_eq!(backend.calls(), ["ready:zstd", "answer_local:zstd"]);
+ }
+
+ #[test]
+ fn a_symbol_defined_by_several_corpora_is_ambiguous_not_a_guess() {
+ let rig = Rig::new();
+ let cat = catalogue(with_fillers(vec![
+ entry(
+ "boringssl",
+ 1,
+ LicenceClass::AdaptWithAttribution,
+ &["ssl", "SSL_CTX_new"],
+ ),
+ entry(
+ "openssl",
+ 1,
+ LicenceClass::AdaptWithAttribution,
+ &["ssl", "SSL_CTX_new"],
+ ),
+ ]));
+ let backend = FakeBackend::new(true);
+ let out = rig.run(
+ &args("what is the value of SSL_CTX_new", None),
+ &cat,
+ None,
+ &backend,
+ );
+ assert_eq!(out.exit, EXIT_NEEDS_DECISION, "{out:?}");
+ let lines: Vec<&str> = out.stdout.lines().collect();
+ assert_eq!(lines.len(), 2, "{:?}", out.stdout);
+ assert!(lines[0].starts_with("boringssl — "), "{}", lines[0]);
+ assert!(lines[1].starts_with("openssl — "), "{}", lines[1]);
+ assert!(backend.calls().is_empty());
+ }
+
+ #[test]
+ fn a_symbol_every_corpus_defines_is_no_evidence() {
+ // `main`-like names say nothing about which corpus is meant.
+ let rig = Rig::new();
+ let entries: Vec<CatalogueEntry> = (0..MAX_SYMBOL_CANDIDATES + 1)
+ .map(|i| {
+ entry(
+ &format!("c{i}"),
+ 1,
+ LicenceClass::AdaptWithAttribution,
+ &["main_loop"],
+ )
+ })
+ .collect();
+ let cat = catalogue(entries);
+ assert!(symbol_hits(&cat, "main_loop").is_empty());
+ let out = rig.run(
+ &args("main_loop", None),
+ &cat,
+ None,
+ &FakeBackend::new(true),
+ );
+ assert_ne!(out.exit, EXIT_OK, "not guessed: {out:?}");
+ }
+
+ #[test]
+ fn a_confident_question_is_not_overridden_by_a_symbol_defined_elsewhere() {
+ let rig = Rig::new();
+ let cat = symbol_catalogue();
+ // The words name lz4 outright; the symbol happens to live in zstd only.
+ let backend = FakeBackend::new(true);
+ let mut a = args("lz4 block decoder", None);
+ a.mode = ModeRequest::Def("ZSTD_VERSION_RELEASE".to_string());
+ let out = rig.run(&a, &cat, None, &backend);
+ assert_eq!(out.exit, EXIT_OK, "{:?}", out.stderr);
+ assert_eq!(backend.calls(), ["ready:lz4", "answer_local:lz4"]);
+ }
+
+ #[test]
+ fn symbol_evidence_narrows_an_ambiguous_list() {
+ let both = vec!["a".to_string(), "b".to_string(), "c".to_string()];
+ assert_eq!(
+ with_symbol_evidence(Routing::Ambiguous(both.clone()), &["b".to_string()]),
+ Routing::Confident("b".to_string())
+ );
+ assert_eq!(
+ with_symbol_evidence(
+ Routing::Ambiguous(both.clone()),
+ &["b".to_string(), "c".to_string()]
+ ),
+ Routing::Ambiguous(vec!["b".to_string(), "c".to_string()])
+ );
+ assert_eq!(
+ with_symbol_evidence(Routing::Ambiguous(both.clone()), &["z".to_string()]),
+ Routing::Ambiguous(both),
+ "a symbol defined elsewhere changes nothing"
+ );
+ assert_eq!(
+ with_symbol_evidence(Routing::NoCorpus, &[]),
+ Routing::NoCorpus
+ );
+ }
+
+ #[test]
+ fn no_corpus_names_the_three_closest_so_the_asker_can_pick_one() {
+ let rig = Rig::new();
+ // Four corpora each hold one word of the question: nobody is credible
+ // (each scores 0.2, below the 0.3 candidate bar), but there is a ranking.
+ let cat = catalogue(vec![
+ entry("c-one", 1, LicenceClass::AdaptWithAttribution, &["decoder"]),
+ entry(
+ "c-two",
+ 1,
+ LicenceClass::AdaptWithAttribution,
+ &["internals"],
+ ),
+ entry(
+ "c-three",
+ 1,
+ LicenceClass::AdaptWithAttribution,
+ &["overview"],
+ ),
+ entry("c-four", 1, LicenceClass::AdaptWithAttribution, &["please"]),
+ entry(
+ "c-five",
+ 1,
+ LicenceClass::AdaptWithAttribution,
+ &["unrelated"],
+ ),
+ ]);
+ let out = rig.run(
+ &args("decoder internals overview please", None),
+ &cat,
+ None,
+ &FakeBackend::new(true),
+ );
+ assert_eq!(out.exit, EXIT_NO_ANSWER, "{out:?}");
+ assert_eq!(out.stderr[0], NO_CORPUS_MESSAGE);
+ let closest = out.stderr.get(1).expect("a closest-corpora line");
+ assert!(closest.starts_with("closest corpora: "), "{closest}");
+ assert!(closest.contains("--corpus"), "{closest}");
+ let named = ["c-one", "c-two", "c-three", "c-four", "c-five"]
+ .iter()
+ .filter(|s| closest.contains(*s))
+ .count();
+ assert_eq!(named, 3, "exactly the top three: {closest}");
+ assert!(
+ !closest.contains("c-five"),
+ "the unrelated one is not close: {closest}"
+ );
+ }
+
+ // ── a pull that will be refused does not start a node ───────────────────
+
+ #[test]
+ fn a_refused_pull_is_refused_before_a_node_is_started() {
+ // `RealBackend::pull_now` used to start (and wait for) a node, and only
+ // then ask the pull whether it would go ahead.
+ let hub = tempfile::tempdir().unwrap();
+ let forbidden = NodeLauncher {
+ exe: PathBuf::from("/nonexistent/xerj-must-not-be-spawned"),
+ pre_args: vec![],
+ };
+ let backend = RealBackend::new(
+ hub.path().to_path_buf(),
+ Some("http://127.0.0.1:1".to_string()),
+ forbidden,
+ );
+ for (class, bytes, want) in [
+ (LicenceClass::Excluded, 1024, "excluded licence class"),
+ (
+ LicenceClass::AdaptWithAttribution,
+ 80 * 1024 * 1024,
+ "auto-pull cap",
+ ),
+ ] {
+ let failure = backend
+ .pull_now(&lz4(bytes, class))
+ .expect_err("refused by the policy");
+ assert_eq!(failure.code, 1);
+ assert!(failure.message.contains(want), "{}", failure.message);
+ }
+ assert!(
+ std::fs::read_dir(hub.path()).unwrap().next().is_none(),
+ "no node dir, no log, no record: nothing was started or written"
+ );
+ }
+
// ── the hosted switch ───────────────────────────────────────────────────
#[test]
diff --git a/engine/crates/xerj-hub/src/cli.rs b/engine/crates/xerj-hub/src/cli.rs
index a704330db..642c29e37 100644
--- a/engine/crates/xerj-hub/src/cli.rs
+++ b/engine/crates/xerj-hub/src/cli.rs
@@ -31,8 +31,8 @@ use anyhow::{anyhow, bail, Context, Result};
use serde_json::{json, Value};
use xerj_common::hub::{
- render, Answer, LicenceClass, PackIndex, PackManifest, PackSource, AUTO_PULL_MAX_BYTES,
- DEFAULT_HUB_URL, DISK_BUDGET_BYTES, HUB_PUBKEY_HEX, PACK_FORMAT_VERSION,
+ render, Answer, CatalogueEntry, LicenceClass, PackIndex, PackManifest, PackSource,
+ AUTO_PULL_MAX_BYTES, DEFAULT_HUB_URL, DISK_BUDGET_BYTES, HUB_PUBKEY_HEX, PACK_FORMAT_VERSION,
};
use xerj_common::xccode::manifest::read_hub_manifest;
use xerj_common::xccode::state;
@@ -40,6 +40,7 @@ use xerj_common::xccode::XcHttp;
use crate::ask::{self, AskArgs, HostedSwitch, ModeRequest};
use crate::catalogue::{build_catalogue, load_catalogue, CorpusBuildInput};
+use crate::flock::{held_by, FileLock, LockError};
use crate::lockfile;
use crate::node::{self, resolve_url, HubNode, NodeLauncher, StopOutcome};
use crate::pack;
@@ -50,6 +51,10 @@ const HUB_USAGE: &str = "usage: xerj hub <command>
Build a signed, pre-indexed pack from a registry manifest (CI).
pull <slug> [--quiet] [--url <node>]
Download, verify and restore a pack, then `xerj code <slug>` works.
+ Exits 2 for an unknown corpus, 3 when the pack fails verification, 5 when
+ this build has no hub public key to verify with (nothing is downloaded),
+ 1 for anything else (an excluded licence and a pack over the size limit
+ are refused here too).
status
Cached packs, the automatic node, the hosted setting, catalogue age.
stop
@@ -478,6 +483,17 @@ fn run_pull(args: &[String]) -> i32 {
eprintln!("xerj hub pull: <slug> is required\n\n{HUB_USAGE}");
return 2;
};
+ // Everything that can refuse this pull — a malformed or unknown slug, an
+ // excluded licence, a pack over the size cap, no key to verify it with — is
+ // decided BEFORE a node is looked for or started. A doomed pull used to boot a
+ // whole node first, and only then say "no".
+ let plan = match plan_pull(&slug) {
+ Ok(p) => p,
+ Err(PullError { code, message }) => {
+ eprintln!("xerj hub pull: {message}");
+ return code;
+ }
+ };
// Unlike build-pack, pull's --url is optional: `--url` > `XERJ_URL` >
// `localhost:9200`, and when none of those answers, the automatic node
// (plan §5) is started or reused.
@@ -495,7 +511,7 @@ fn run_pull(args: &[String]) -> i32 {
return 1;
}
};
- match pull(&slug, &resolved.url, resolved.managed, quiet) {
+ match pull_with(&plan, &resolved.url, resolved.managed, quiet) {
Ok(Pulled::Already) => {
if !quiet {
println!("{slug}: already present");
@@ -522,11 +538,14 @@ pub(crate) enum Pulled {
/// A pull failure carrying an exit code the caller can act on: **2** = the slug
/// is unknown or malformed; **3** = the pack failed verification (sha /
-/// signature / format, or there was no key to verify with); **1** = an
-/// infrastructure or policy failure (catalogue load, the size cap, node connect,
-/// a pull already in progress, the download, the restore, or writing the
-/// ledger). Keeping 2 and 3 exact lets a caller tell "wrong name" and "untrusted
-/// bytes" apart from "the machine or network got in the way".
+/// signature / format); **5** = this build has no hub public key, so nothing can
+/// be verified (and nothing was downloaded); **1** = an infrastructure or policy
+/// failure (catalogue load, the licence or size policy, node connect, a pull
+/// already in progress, the download, the restore, or writing the ledger).
+/// Keeping 2, 3 and 5 exact lets a caller tell "wrong name", "untrusted bytes"
+/// and "this binary cannot verify anything" apart from "the machine or network
+/// got in the way".
+#[derive(Debug)]
pub(crate) struct PullError {
pub(crate) code: i32,
pub(crate) message: String,
@@ -544,59 +563,100 @@ impl PullError {
const E_INFRA: i32 = 1;
/// The slug is unknown or malformed.
const E_SLUG: i32 = 2;
-/// The pack failed verification (sha / signature / format / missing key).
+/// The pack failed verification (sha / signature / format).
const E_VERIFY: i32 = 3;
+/// No hub public key is configured: the pack was never even fetched. Its own code,
+/// because it is not a verification FAILURE (the bytes were not judged and found
+/// wanting) and not a transient one (retrying cannot help): the build is missing
+/// the key.
+const E_NOKEY: i32 = ask::EXIT_NO_KEY;
/// Largest signature file pull will download — an ed25519 sig is 128 hex chars,
/// so this caps a hostile hub's "signature" far below the pack cap.
const SIG_MAX_BYTES: u64 = 4096;
-/// A pull lock older than this is an abandoned lock from a crashed pull and is
-/// stolen, so a hard kill cannot wedge a corpus forever.
-const LOCK_STALE: Duration = Duration::from_secs(30 * 60);
-
-/// A best-effort cross-process mutex for one `(slug, pin)`: an atomically
-/// created `<pin8>.lock` directory. A second concurrent pull of the same pin
-/// sees it and refuses rather than racing the shared tmp/extract dirs; the guard
-/// removes it on drop (every non-crash return below). A lock left by a crash is
-/// stolen once older than [`LOCK_STALE`]. The durable lockfile is plan §8; this
-/// only closes the same-pin race.
+/// How long a pull waits for another pull of the same pin to finish before it
+/// gives up. A pull is a download of at most [`AUTO_PULL_MAX_BYTES`] plus a
+/// restore, so this is generous; it exists so a wedged holder cannot hang a
+/// caller (an agent's tool call) for ever.
+const PULL_LOCK_WAIT: Duration = Duration::from_secs(60);
+
+/// Cross-process exclusion for one `(slug, pin)`: an OS lock on `<pin8>.lock`
+/// (see [`crate::flock`]). A second concurrent pull of the same pin waits for the
+/// first (bounded by [`PULL_LOCK_WAIT`]) instead of racing it over the shared
+/// tmp/extract directories — and then finds the pack present.
+///
+/// The lock belongs to the process holding it: it is released the moment that
+/// process exits, however it exits. This used to be an atomically created
+/// directory that was only ever removed by the pull's own destructor, so a pull
+/// killed with SIGKILL, an OOM or an agent's tool timeout left the corpus
+/// un-pullable for thirty minutes, with no owner recorded and a message that
+/// named no path.
struct PullLock {
- path: PathBuf,
+ _lock: FileLock,
}
impl PullLock {
- fn acquire(pack_dir: &Path, slug: &str, pin8: &str) -> Result<PullLock, PullError> {
+ fn acquire(
+ pack_dir: &Path,
+ slug: &str,
+ pin8: &str,
+ wait: Duration,
+ quiet: bool,
+ ) -> Result<PullLock, PullError> {
fs::create_dir_all(pack_dir)
.map_err(|e| PullError::new(E_INFRA, format!("create {}: {e}", pack_dir.display())))?;
let path = pack_dir.join(format!("{pin8}.lock"));
- // Steal an abandoned lock from a crashed pull.
- let stale = fs::metadata(&path)
- .ok()
- .and_then(|md| md.modified().ok())
- .and_then(|m| m.elapsed().ok())
- .map(|age| age > LOCK_STALE)
- .unwrap_or(false);
- if stale {
- let _ = fs::remove_dir(&path);
- }
- match fs::create_dir(&path) {
- Ok(()) => Ok(PullLock { path }),
- Err(_) => Err(PullError::new(
+ let contended = |holder| {
+ PullError::new(
+ E_INFRA,
+ format!(
+ "a pull of '{slug}' ({pin8}) is already in progress (lock {}{}) and was still \
+ running after {}s; wait for it to finish — the lock is released the moment \
+ that process exits, however it exits",
+ path.display(),
+ held_by(holder),
+ wait.as_secs()
+ ),
+ )
+ };
+ let infra = |e: std::io::Error| {
+ PullError::new(
E_INFRA,
- format!("a pull of '{slug}' ({pin8}) is already in progress"),
- )),
+ format!("cannot take the pull lock {}: {e}", path.display()),
+ )
+ };
+ // One immediate try, so the caller is only told it is waiting when it is.
+ match FileLock::acquire(&path, Duration::ZERO) {
+ Ok(lock) => return Ok(PullLock { _lock: lock }),
+ Err(LockError::Io(e)) => return Err(infra(e)),
+ Err(LockError::Timeout { .. }) => {}
+ }
+ if !quiet {
+ eprintln!(
+ "xerj hub: another pull of '{slug}' ({pin8}) is running; waiting up to {}s for it \
+ (lock {})",
+ wait.as_secs(),
+ path.display()
+ );
+ }
+ match FileLock::acquire(&path, wait) {
+ Ok(lock) => Ok(PullLock { _lock: lock }),
+ Err(LockError::Timeout { holder }) => Err(contended(holder)),
+ Err(LockError::Io(e)) => Err(infra(e)),
}
}
}
-impl Drop for PullLock {
- fn drop(&mut self) {
- let _ = fs::remove_dir(&self.path);
- }
+
+/// What a pull has decided before it touches a node or the disk: the catalogue
+/// entry, the pin its files are named by, and the key it will verify with.
+pub(crate) struct PullPlan {
+ entry: CatalogueEntry,
+ pin8: String,
+ pubkey: String,
}
-/// Pull `slug` into the node at `url`. `managed` is true for the automatic node
-/// (started `--insecure`, so no credential is sent); `quiet` suppresses the
-/// eviction notices.
-pub(crate) fn pull(slug: &str, url: &str, managed: bool, quiet: bool) -> Result<Pulled, PullError> {
+/// Look `slug` up in the catalogue and decide whether it may be pulled at all
+/// ([`plan_pull_entry`]).
+pub(crate) fn plan_pull(slug: &str) -> Result<PullPlan, PullError> {
// 0. Validate the slug BEFORE it is ever joined into a filesystem path. The
// catalogue is accepted UNVERIFIED when no hub pubkey is configured, so a
// hostile `slug` ("../../etc") must be rejected here, ahead of every fs op.
@@ -628,20 +688,52 @@ pub(crate) fn pull(slug: &str, url: &str, managed: bool, quiet: bool) -> Result<
),
)
})?;
- // The matched entry's own slug drives the paths too; it equals `slug` (the
- // find matched on it, already validated), but re-check defensively.
- if !valid_slug(&entry.slug) {
+ plan_pull_entry(entry)
+}
+
+/// Everything that can refuse to pull `entry`, checked in the order that costs
+/// least and before any node, download or temp dir exists:
+///
+/// 1. its slug is safe to join into a path (the matched entry's own slug drives
+/// the paths, so it is re-checked even though the lookup matched on a valid one);
+/// 2. its pin is a commit;
+/// 3. its licence class is not `excluded` — **plan §11: an excluded corpus is never
+/// pulled.** This is enforced here, in the pull itself, and not only in the
+/// `ask` flow that decides whether to *start* one: `xerj hub pull <slug>` is a
+/// command anybody (or any agent that was handed a hint) can type, and a policy
+/// that only the caller applies is a policy the next caller forgets. There is
+/// deliberately no override flag: nothing in the plan lets a command line waive
+/// a licence class, and a flag an agent can read in `--help` is not a human
+/// decision. If an override is ever wanted it needs its own decision and its own
+/// confirmation, not a switch here;
+/// 4. its advertised size is within [`AUTO_PULL_MAX_BYTES`] — a malicious hub must
+/// not OOM the client by advertising a huge pack (decision 2). The transfer is
+/// bounded again in [`fetch_to`] in case the field lies;
+/// 5. there is a key to verify it with (exit [`E_NOKEY`]). Resolved before any temp
+/// dir exists, so a pull with no key leaves nothing to re-fetch.
+pub(crate) fn plan_pull_entry(entry: &CatalogueEntry) -> Result<PullPlan, PullError> {
+ let slug = entry.slug.as_str();
+ if !valid_slug(slug) {
return Err(PullError::new(
E_SLUG,
- format!("catalogue entry has an invalid slug '{}'", entry.slug),
+ format!("catalogue entry has an invalid slug '{slug}'"),
));
}
let pin8 = pin8_of(&entry.pin_sha)
.map_err(|e| PullError::new(E_INFRA, format!("catalogue entry for '{slug}': {e:#}")))?;
-
- // 2. Size cap BEFORE any download — a malicious hub must not OOM the client
- // by advertising a huge pack (decision 2 / AUTO_PULL_MAX_BYTES). The
- // transfer is bounded again in `fetch_to` in case the field lies.
+ if entry.licence_class == LicenceClass::Excluded {
+ return Err(PullError::new(
+ E_INFRA,
+ format!(
+ "pack '{slug}' has an excluded licence class ({}) and is never pulled",
+ if entry.licence_spdx.is_empty() {
+ "unknown"
+ } else {
+ &entry.licence_spdx
+ }
+ ),
+ ));
+ }
if entry.pack_bytes > AUTO_PULL_MAX_BYTES {
return Err(PullError::new(
E_INFRA,
@@ -651,14 +743,31 @@ pub(crate) fn pull(slug: &str, url: &str, managed: bool, quiet: bool) -> Result<
),
));
}
+ let pubkey = hub_pubkey().map_err(|e| PullError::new(E_NOKEY, format!("{e:#}")))?;
+ Ok(PullPlan {
+ entry: entry.clone(),
+ pin8,
+ pubkey,
+ })
+}
- // 3. The verifying key, resolved BEFORE any temp dir exists — a pull with no
- // key is a verification failure that must not leave a `.tmp` to re-fetch.
- let pubkey = hub_pubkey().map_err(|e| PullError::new(E_VERIFY, format!("{e:#}")))?;
+/// Pull the planned corpus into the node at `url`. `managed` is true for the
+/// automatic node (started `--insecure`, so no credential is sent); `quiet`
+/// suppresses the eviction and wait notices.
+pub(crate) fn pull_with(
+ plan: &PullPlan,
+ url: &str,
+ managed: bool,
+ quiet: bool,
+) -> Result<Pulled, PullError> {
+ let entry = &plan.entry;
+ let slug = entry.slug.as_str();
+ let pin8 = plan.pin8.as_str();
+ let hub_home = hub_home();
- // 3b. Disk budget (plan §5 step 6): make room BEFORE any download, evicting
- // least-recently-used packs — never this slug's. Pack directories only;
- // the node's restored indices are not touched.
+ // Disk budget (plan §5 step 6): make room BEFORE any download, evicting
+ // least-recently-used packs — never this slug's. Pack directories only; the
+ // node's restored indices are not touched.
for c in ask::evict_to_budget(&hub_home, slug, entry.pack_bytes, DISK_BUDGET_BYTES) {
if !quiet {
eprintln!(
@@ -672,7 +781,7 @@ pub(crate) fn pull(slug: &str, url: &str, managed: bool, quiet: bool) -> Result<
}
}
- // 4. The node (infra if unreachable).
+ // The node (infra if unreachable).
let root = xerj_autoindex::xc::code_root();
let node = if managed {
HubNode::connect_insecure(url)
@@ -681,14 +790,15 @@ pub(crate) fn pull(slug: &str, url: &str, managed: bool, quiet: bool) -> Result<
}
.map_err(|e| PullError::new(E_INFRA, format!("connect {url}: {e:#}")))?;
- // 5. Take the per-pin lock so a concurrent pull does not race the tmp/extract
- // dirs; released on every return below (Drop).
+ // Take the per-pin lock so a concurrent pull does not race the tmp/extract
+ // dirs; released on every return below (Drop), and by the OS if this process
+ // dies before it gets there.
let pack_dir = hub_home.join("packs").join(slug);
- let _lock = PullLock::acquire(&pack_dir, slug, &pin8)?;
+ let _lock = PullLock::acquire(&pack_dir, slug, pin8, PULL_LOCK_WAIT, quiet)?;
- // 6. Download pack + sig into a temp dir (size-capped). NOTHING is extracted.
+ // Download pack + sig into a temp dir (size-capped). NOTHING is extracted.
let tmp = pack_dir.join(format!("{pin8}.tmp"));
- let extract = pack_dir.join(pin8.clone());
+ let extract = pack_dir.join(pin8);
let _ = fs::remove_dir_all(&tmp); // a stale temp from a crash must not poison this
fs::create_dir_all(&tmp)
.map_err(|e| PullError::new(E_INFRA, format!("create {}: {e}", tmp.display())))?;
@@ -709,32 +819,42 @@ pub(crate) fn pull(slug: &str, url: &str, managed: bool, quiet: bool) -> Result<
}
};
- // 7. Verify sha256 THEN signature THEN format_version, extracting nothing
- // until all pass (plan §11). Any failure: delete the temp, exit 3.
- let manifest =
- match pack::read_and_verify_pack(&archive, &sig, &entry.pack_sha256, &pubkey, &extract) {
- Ok(m) => m,
- Err(e) => {
- let _ = fs::remove_dir_all(&tmp);
- // read_and_verify_pack never populates `extract` on failure; clear
- // any half-written dir from an interrupted prior run too.
- let _ = fs::remove_dir_all(&extract);
- return Err(PullError::new(
- E_VERIFY,
- format!("pack verification failed: {e:#}"),
- ));
- }
- };
+ // Verify sha256 THEN signature THEN format_version, extracting nothing until
+ // all pass (plan §11). Any failure: delete the temp, exit 3.
+ //
+ // The extraction goes to a staging directory inside `tmp`, never to `extract`
+ // itself, for two reasons. A pack killed mid-extraction must not leave a
+ // directory at the final path that looks complete (`pack.json`, the marker
+ // `ask` and `status` go by, used to be written before `snapshot/`). And a pull
+ // that fails — or is killed — must not delete the pack an earlier, successful
+ // pull put at `extract`: only what THIS run created is removed on failure.
+ let staging = tmp.join("extract");
+ let manifest = match pack::read_and_verify_pack(
+ &archive,
+ &sig,
+ &entry.pack_sha256,
+ &plan.pubkey,
+ &staging,
+ ) {
+ Ok(m) => m,
+ Err(e) => {
+ let _ = fs::remove_dir_all(&tmp);
+ return Err(PullError::new(
+ E_VERIFY,
+ format!("pack verification failed: {e:#}"),
+ ));
+ }
+ };
+ install_extract(&staging, &extract, &tmp)?;
let _ = fs::remove_dir_all(&tmp);
- // 8. Restore — unless the WHOLE index set is already live on this node. A
- // partial match (a prior partial restore, or a local `corpus index` that
- // produced a different build) must NOT read as a complete pack: skipping
- // then would leave `--def` querying a missing index and stamp the ledger
- // "answered from: local pack" over data that is not the pack. So the skip
- // requires EVERY manifest index present; a list error counts as "not
- // present" and restores (it never silently skips). Idempotent on a real
- // full match.
+ // Restore — unless the WHOLE index set is already live on this node. A partial
+ // match (a prior partial restore, or a local `corpus index` that produced a
+ // different build) must NOT read as a complete pack: skipping then would leave
+ // `--def` querying a missing index and stamp the ledger "answered from: local
+ // pack" over data that is not the pack. So the skip requires EVERY manifest
+ // index present; a list error counts as "not present" and restores (it never
+ // silently skips). Idempotent on a real full match.
let prefix = manifest.index_prefix.clone();
let live = node
.cat_indices_json(&format!("{prefix}*"))
@@ -759,7 +879,7 @@ pub(crate) fn pull(slug: &str, url: &str, managed: bool, quiet: bool) -> Result<
.map_err(|e| PullError::new(E_INFRA, format!("restore snapshot: {e:#}")))?;
}
- // 9. Only now — with the full set provably present — stamp the ledger.
+ // Only now — with the full set provably present — stamp the ledger.
state::write_hubpack_state(
&root,
slug,
@@ -782,6 +902,34 @@ pub(crate) fn pull(slug: &str, url: &str, managed: bool, quiet: bool) -> Result<
)))
}
+/// Put the verified, fully extracted `staging` directory at `dest` by renaming it —
+/// one step, so `dest` is either absent or complete. A pack already at `dest`
+/// (from an earlier pull of the same pin) is moved into `scratch` first and put
+/// back if the rename fails; it is deleted with `scratch` once the new one is in
+/// place.
+fn install_extract(staging: &Path, dest: &Path, scratch: &Path) -> Result<(), PullError> {
+ let previous = scratch.join("previous");
+ let had_previous = dest.exists();
+ if had_previous {
+ fs::rename(dest, &previous).map_err(|e| {
+ PullError::new(
+ E_INFRA,
+ format!("cannot replace the pack at {}: {e}", dest.display()),
+ )
+ })?;
+ }
+ if let Err(e) = fs::rename(staging, dest) {
+ if had_previous {
+ let _ = fs::rename(&previous, dest);
+ }
+ return Err(PullError::new(
+ E_INFRA,
+ format!("cannot move the verified pack into {}: {e}", dest.display()),
+ ));
+ }
+ Ok(())
+}
+
/// `^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$` — a slug safe to join into a path: no
/// `..`, no `/`, non-empty, no leading/trailing separator. This is what stops a
/// catalogue (which may be unverified) from steering a pull's writes out of
@@ -959,7 +1107,8 @@ options:
exit codes: 0 answered; 1 no match or nothing to answer from; 2 usage error or
unknown corpus; 3 a pack failed verification; 4 more than one corpus matches
-(candidates are printed; re-run with --corpus <slug>)";
+(candidates are printed; re-run with --corpus <slug>); 5 this build has no hub
+public key, so no pack can be verified (nothing was downloaded)";
/// The first argument of every real `xerj` subcommand.
pub const KNOWN_SUBCOMMANDS: &[&str] = &[
@@ -1235,8 +1384,9 @@ fn hub_pubkey() -> Result<String> {
return Ok(HUB_PUBKEY_HEX.to_string());
}
bail!(
- "no hub public key configured (HUB_PUBKEY_HEX is empty and XERJ_HUB_PUBKEY is unset); \
- cannot verify a pack signature"
+ "no hub public key is configured in this build (HUB_PUBKEY_HEX is empty and \
+ XERJ_HUB_PUBKEY is unset), so a pack's signature cannot be verified; nothing was \
+ downloaded"
)
}
@@ -1534,6 +1684,235 @@ mod tests {
// `catalogue` test module already mutates under its own `ENV_LOCK`. A unit
// test here that set/removed that var would race those tests (lib tests share
// one process and run in parallel, and the lock is private to that module).
- // The empty-key path — no key => exit 3, with no `.tmp` leaked — is covered
- // end to end by the `nokey` scenario in `tests/pack_roundtrip.rs` instead.
+ // The empty-key path — no key => its own exit code (5), with no `.tmp` leaked
+ // and no node started — is covered end to end by the `nokey` scenario in
+ // `tests/pack_roundtrip.rs` instead. Everything below refuses BEFORE the key is
+ // looked at, so it needs no environment.
+
+ // ── the pull policy lives in the pull ────────────────────────────────────
+
+ fn entry(slug: &str, class: LicenceClass, pack_bytes: u64) -> CatalogueEntry {
+ CatalogueEntry {
+ slug: slug.to_string(),
+ domain: "TEST".to_string(),
+ topics: vec![],
+ ask_for: "x".to_string(),
+ pin_sha: "0774d05537f9762f838f7ab541b7765f1a729cb5".to_string(),
+ pin_date: "2026-10-02".to_string(),
+ refresh_cadence_days: 7,
+ licence_spdx: "GPL-3.0".to_string(),
+ licence_class: class,
+ pack_bytes,
+ pack_url: "file:///nonexistent".to_string(),
+ pack_sha256: "0".repeat(64),
+ sig_url: "file:///nonexistent.sig".to_string(),
+ dependency_names: vec![],
+ quality_grade: None,
+ term_sketch: xerj_common::hub::Sketch {
+ m_bits: 64,
+ k_hashes: 7,
+ bits_base64: "AAAAAAAAAAA=".to_string(),
+ },
+ }
+ }
+
+ #[test]
+ fn an_excluded_corpus_is_refused_by_pull_itself_not_only_by_the_ask_flow() {
+ // `xerj hub pull <slug>` is a command anyone can type — including from a
+ // hint an earlier run printed. Plan §11: an excluded licence class is never
+ // pulled. The refusal is the pull's, before any node, download or temp dir.
+ let err = plan_pull_entry(&entry("gpl-thing", LicenceClass::Excluded, 1024))
+ .err()
+ .expect("refused");
+ assert_eq!(err.code, 1, "a policy refusal, exit 1");
+ assert!(
+ err.message.contains("excluded licence class"),
+ "{}",
+ err.message
+ );
+ assert!(err.message.contains("never pulled"), "{}", err.message);
+ assert!(
+ err.message.contains("GPL-3.0"),
+ "names the licence: {}",
+ err.message
+ );
+ // Excluded wins over the size cap: the licence is the first thing said.
+ let err = plan_pull_entry(&entry("gpl-thing", LicenceClass::Excluded, u64::MAX / 2))
+ .err()
+ .expect("refused");
+ assert!(
+ err.message.contains("excluded licence class"),
+ "{}",
+ err.message
+ );
+ }
+
+ #[test]
+ fn pull_refuses_a_pack_over_the_cap_and_a_bad_slug_before_anything_else() {
+ let err = plan_pull_entry(&entry(
+ "big",
+ LicenceClass::AdaptWithAttribution,
+ AUTO_PULL_MAX_BYTES + 1,
+ ))
+ .err()
+ .expect("refused");
+ assert_eq!(err.code, 1);
+ assert!(err.message.contains("over the"), "{}", err.message);
+
+ let err = plan_pull_entry(&entry("../../etc", LicenceClass::AdaptWithAttribution, 1))
+ .err()
+ .expect("refused");
+ assert_eq!(err.code, 2, "a malformed slug is exit 2");
+
+ let mut bad_pin = entry("ok", LicenceClass::AdaptWithAttribution, 1);
+ bad_pin.pin_sha = "not-a-commit".to_string();
+ assert_eq!(plan_pull_entry(&bad_pin).err().expect("refused").code, 1);
+ }
+
+ #[test]
+ fn approach_only_is_still_pullable_the_policy_only_stops_excluded() {
+ // The key lookup is the first thing that can still fail for these; either
+ // way the answer is not a licence refusal.
+ for class in [
+ LicenceClass::AdaptWithAttribution,
+ LicenceClass::ApproachOnly,
+ ] {
+ match plan_pull_entry(&entry("fine", class, 1024)) {
+ Ok(plan) => assert_eq!(plan.pin8, "0774d055"),
+ Err(e) => assert_eq!(
+ e.code, E_NOKEY,
+ "only a missing key may stop it: {}",
+ e.message
+ ),
+ }
+ }
+ }
+
+ // ── the pull lock belongs to the process that holds it ───────────────────
+
+ #[test]
+ fn a_pull_waits_a_bounded_time_for_the_lock_and_names_it_when_it_gives_up() {
+ let packs = tempfile::tempdir().unwrap();
+ let dir = packs.path().join("lz4");
+ let held = PullLock::acquire(&dir, "lz4", "0774d055", Duration::ZERO, true).unwrap();
+
+ let t0 = std::time::Instant::now();
+ let err = PullLock::acquire(&dir, "lz4", "0774d055", Duration::from_millis(300), true)
+ .err()
+ .expect("still held");
+ let waited = t0.elapsed();
+ assert_eq!(err.code, 1);
+ let lock = dir.join("0774d055.lock");
+ assert!(
+ err.message.contains(&lock.display().to_string()),
+ "the message names the lock file: {}",
+ err.message
+ );
+ assert!(
+ err.message
+ .contains(&format!("held by pid {}", std::process::id())),
+ "and its owner: {}",
+ err.message
+ );
+ assert!(
+ waited >= Duration::from_millis(300) && waited < Duration::from_secs(5),
+ "bounded: {waited:?}"
+ );
+
+ // A different pin of the same corpus is a different lock.
+ let other = PullLock::acquire(&dir, "lz4", "deadbeef", Duration::ZERO, true).unwrap();
+ drop(other);
+
+ drop(held);
+ PullLock::acquire(&dir, "lz4", "0774d055", Duration::ZERO, true)
+ .expect("free the moment the holder lets go");
+ }
+
+ #[test]
+ fn a_waiting_pull_proceeds_as_soon_as_the_other_one_finishes() {
+ let packs = tempfile::tempdir().unwrap();
+ let dir = packs.path().join("lz4");
+ let held = PullLock::acquire(&dir, "lz4", "0774d055", Duration::ZERO, true).unwrap();
+ let waiter_dir = dir.clone();
+ let waiter = std::thread::spawn(move || {
+ let t0 = std::time::Instant::now();
+ let got = PullLock::acquire(
+ &waiter_dir,
+ "lz4",
+ "0774d055",
+ Duration::from_secs(20),
+ true,
+ );
+ (got.is_ok(), t0.elapsed())
+ });
+ std::thread::sleep(Duration::from_millis(400));
+ drop(held);
+ let (got, waited) = waiter.join().unwrap();
+ assert!(got, "the second pull got the lock once the first finished");
+ assert!(
+ waited >= Duration::from_millis(300) && waited < Duration::from_secs(10),
+ "{waited:?}"
+ );
+ }
+
+ #[test]
+ fn a_lock_directory_left_by_an_older_build_does_not_block_a_pull() {
+ // The old scheme: `<pin8>.lock` was a directory, removed only by the pull's
+ // own destructor.
+ let packs = tempfile::tempdir().unwrap();
+ let dir = packs.path().join("lz4");
+ fs::create_dir_all(dir.join("0774d055.lock")).unwrap();
+ PullLock::acquire(&dir, "lz4", "0774d055", Duration::ZERO, true)
+ .expect("the leftover directory is replaced by a lock file");
+ }
+
+ // ── installing a verified extract ────────────────────────────────────────
+
+ fn fake_extract(dir: &Path, marker: &str) {
+ fs::create_dir_all(dir.join("snapshot")).unwrap();
+ fs::write(dir.join("pack.json"), marker).unwrap();
+ fs::write(dir.join("snapshot").join("data"), marker).unwrap();
+ }
+
+ #[test]
+ fn a_verified_extract_is_renamed_into_place_and_replaces_an_older_one() {
+ let root = tempfile::tempdir().unwrap();
+ let scratch = root.path().join("0774d055.tmp");
+ let dest = root.path().join("0774d055");
+ let staging = scratch.join("extract");
+
+ // Nothing there yet.
+ fake_extract(&staging, "first");
+ install_extract(&staging, &dest, &scratch).unwrap();
+ assert_eq!(fs::read_to_string(dest.join("pack.json")).unwrap(), "first");
+ assert!(!staging.exists(), "moved, not copied");
+
+ // An older extract is replaced whole: nothing of it is mixed into the new one.
+ fs::write(dest.join("stale-file"), "x").unwrap();
+ fake_extract(&staging, "second");
+ install_extract(&staging, &dest, &scratch).unwrap();
+ assert_eq!(
+ fs::read_to_string(dest.join("pack.json")).unwrap(),
+ "second"
+ );
+ assert!(!dest.join("stale-file").exists());
+ }
+
+ #[test]
+ fn a_failed_install_puts_the_previous_extract_back() {
+ let root = tempfile::tempdir().unwrap();
+ let scratch = root.path().join("0774d055.tmp");
+ fs::create_dir_all(&scratch).unwrap();
+ let dest = root.path().join("0774d055");
+ fake_extract(&dest, "verified-earlier");
+ // The staging directory is missing, so the second rename fails.
+ let err =
+ install_extract(&scratch.join("no-such-staging"), &dest, &scratch).expect_err("fails");
+ assert_eq!(err.code, 1);
+ assert_eq!(
+ fs::read_to_string(dest.join("pack.json")).unwrap(),
+ "verified-earlier",
+ "the pack an earlier pull verified is still there"
+ );
+ }
}
diff --git a/engine/crates/xerj-hub/src/pack.rs b/engine/crates/xerj-hub/src/pack.rs
index 967d32817..b1adb517b 100644
--- a/engine/crates/xerj-hub/src/pack.rs
+++ b/engine/crates/xerj-hub/src/pack.rs
@@ -360,9 +360,17 @@ fn verify_sha256sums(members: &BTreeMap<String, Vec<u8>>) -> Result<()> {
/// Write every member under `dest`, creating parent directories. Keys are `/`-joined
/// relative paths already sanitised by [`read_members`].
+///
+/// `pack.json` is written **last**. A pack directory counts as present (cached,
+/// answerable, evictable) exactly when `pack.json` exists, and `BTreeMap` order puts
+/// it *before* `snapshot/`, so a process killed partway through the extraction used
+/// to leave a directory that looked complete with most of its snapshot missing.
+/// Written last, `pack.json` is the marker that everything else is already there.
+/// (`hub pull` extracts into a staging directory and renames it into place, so this
+/// is the second line of defence for anyone extracting straight into a final path.)
fn extract_into(members: &BTreeMap<String, Vec<u8>>, dest: &Path) -> Result<()> {
fs::create_dir_all(dest).with_context(|| format!("create {}", dest.display()))?;
- for (rel, data) in members {
+ let write_member = |rel: &str, data: &[u8]| -> Result<()> {
let mut target = dest.to_path_buf();
for part in rel.split('/') {
target.push(part);
@@ -370,7 +378,15 @@ fn extract_into(members: &BTreeMap<String, Vec<u8>>, dest: &Path) -> Result<()>
if let Some(parent) = target.parent() {
fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
}
- fs::write(&target, data).with_context(|| format!("write {}", target.display()))?;
+ fs::write(&target, data).with_context(|| format!("write {}", target.display()))
+ };
+ for (rel, data) in members {
+ if rel != PACK_JSON {
+ write_member(rel, data)?;
+ }
+ }
+ if let Some(manifest) = members.get(PACK_JSON) {
+ write_member(PACK_JSON, manifest)?;
}
Ok(())
}
@@ -844,6 +860,40 @@ mod tests {
);
}
+ #[test]
+ fn pack_json_is_written_last_so_an_interrupted_extraction_never_looks_complete() {
+ // A pack dir is "present" when pack.json exists, and BTreeMap order puts
+ // pack.json BEFORE snapshot/: extracted in map order, a process killed
+ // after pack.json looked like a complete pack with most of its snapshot
+ // missing. Fail the extraction partway (a file where the snapshot
+ // directory has to go) and check pack.json was not written.
+ let mut members: BTreeMap<String, Vec<u8>> = BTreeMap::new();
+ members.insert(PACK_JSON.into(), b"{}".to_vec());
+ members.insert(SHA256SUMS.into(), b"".to_vec());
+ members.insert("defs.jsonl".into(), b"".to_vec());
+ members.insert("snapshot/idx/segment_0.bin".into(), vec![1, 2, 3]);
+
+ let tmp = tempfile::tempdir().unwrap();
+ let dest = tmp.path().join("dest");
+ fs::create_dir_all(&dest).unwrap();
+ fs::write(dest.join("snapshot"), b"in the way").unwrap();
+ assert!(extract_into(&members, &dest).is_err());
+ assert!(
+ !dest.join(PACK_JSON).exists(),
+ "an extraction that failed on the snapshot must not have left pack.json"
+ );
+ assert!(
+ dest.join("defs.jsonl").exists(),
+ "the members before the failure were written"
+ );
+
+ // Cleared, the same members extract completely, pack.json included.
+ let ok = tmp.path().join("ok");
+ extract_into(&members, &ok).unwrap();
+ assert_eq!(fs::read(ok.join(PACK_JSON)).unwrap(), b"{}");
+ assert!(ok.join("snapshot/idx/segment_0.bin").is_file());
+ }
+
#[test]
fn build_defs_and_files_extracts_c_defines_and_inventories_files() {
let tmp = tempfile::tempdir().unwrap();
diff --git a/engine/crates/xerj-hub/src/router.rs b/engine/crates/xerj-hub/src/router.rs
index c4a317f82..bc530039f 100644
--- a/engine/crates/xerj-hub/src/router.rs
+++ b/engine/crates/xerj-hub/src/router.rs
@@ -107,6 +107,21 @@ pub fn route(
hosted: Option<&dyn HostedSearch>,
cached: &[String],
) -> Routing {
+ route_ranked(question, cwd, cat, hosted, cached).0
+}
+
+/// [`route`], plus every corpus that earned any evidence at all as
+/// `(slug, merged score)`, best first. The verdict is exactly [`route`]'s; the
+/// ranking is what a caller prints when the verdict is [`Routing::NoCorpus`] — the
+/// closest corpora, so the asker can name one with `--corpus` instead of being told
+/// only that nothing matched.
+pub fn route_ranked(
+ question: &str,
+ cwd: &Path,
+ cat: &Catalogue,
+ hosted: Option<&dyn HostedSearch>,
+ cached: &[String],
+) -> (Routing, Vec<(String, f32)>) {
let mut evidence: Vec<Evidence> = Vec::new();
// 1. Project dependencies — any corpus the project already depends on.
@@ -205,8 +220,8 @@ pub fn route(
}
/// Merge evidence by slug (max score), apply the cached bonus, and reach a
-/// verdict.
-fn decide(evidence: Vec<Evidence>, cached: &[String]) -> Routing {
+/// verdict; the ranking it was reached from comes back with it.
+fn decide(evidence: Vec<Evidence>, cached: &[String]) -> (Routing, Vec<(String, f32)>) {
let mut best: HashMap<String, f32> = HashMap::new();
for e in evidence {
let score = e.score;
@@ -215,7 +230,7 @@ fn decide(evidence: Vec<Evidence>, cached: &[String]) -> Routing {
.or_insert(score);
}
if best.is_empty() {
- return Routing::NoCorpus;
+ return (Routing::NoCorpus, Vec::new());
}
for (slug, score) in best.iter_mut() {
if cached.iter().any(|c| c == slug) {
@@ -237,7 +252,8 @@ fn decide(evidence: Vec<Evidence>, cached: &[String]) -> Routing {
Some((_, second)) => top_score >= DOMINANCE * *second,
};
if top_score >= CONFIDENT_MIN && dominant {
- return Routing::Confident(ranked[0].0.clone());
+ let slug = ranked[0].0.clone();
+ return (Routing::Confident(slug), ranked);
}
let candidates: Vec<String> = ranked
@@ -246,10 +262,10 @@ fn decide(evidence: Vec<Evidence>, cached: &[String]) -> Routing {
.map(|(slug, _)| slug.clone())
.collect();
if candidates.len() >= 2 {
- return Routing::Ambiguous(candidates);
+ return (Routing::Ambiguous(candidates), ranked);
}
- Routing::NoCorpus
+ (Routing::NoCorpus, ranked)
}
/// Tokenize a question into the terms matched against a sketch: each identifier
diff --git a/engine/crates/xerj-hub/tests/ask_flow.rs b/engine/crates/xerj-hub/tests/ask_flow.rs
index 97cf0a022..325f74c58 100644
--- a/engine/crates/xerj-hub/tests/ask_flow.rs
+++ b/engine/crates/xerj-hub/tests/ask_flow.rs
@@ -22,6 +22,13 @@
//! - **D. the disk budget:** a pull evicts a least-recently-used pack that is
//! over the 4 GiB budget (sparse files, so nothing is written).
//! - **E. offline with no pack:** exit 1, and no node is started to say so.
+//! - **G. an excluded corpus:** never pulled, and never hinted at — `--offline` and
+//! `--no-fetch` used to print `run: xerj hub pull <slug>` for it, and `hub pull`
+//! itself had no licence check.
+//! - **F. a pull killed mid-run:** a second process running `xerj hub pull` is
+//! blocked mid-download holding the pull lock, killed with SIGKILL, and the retry
+//! is not blocked (the lock was a directory that only the pull's destructor
+//! removed, so a killed pull wedged its corpus for half an hour).
//!
//! The test binary doubles as the `xerj` executable the flow spawns: a
//! `NodeLauncher` points at it with `--ignored --exact child_main --nocapture --`
@@ -922,4 +929,191 @@ async fn ask_end_to_end() {
assert!(!hub.join("node.pid").exists(), "no node was started");
assert!(!hub.join("packs").exists(), "nothing was downloaded");
}
+
+ // ── G. an EXCLUDED corpus: no pull, and no hint to pull it, offline or not ─
+ {
+ let hub = work.join("hub-g");
+ let code = work.join("code-g");
+ publish(&archive, &sig, &work.join("net-g"), &hub);
+ set_catalogue_field(&hub, "licence_class", json!("excluded"));
+ set_env(&hub, &code, &scratch_home, &public);
+ for flag in ["--offline", "--no-fetch", "(neither)"] {
+ let mut args = def_args();
+ match flag {
+ "--offline" => args.offline = true,
+ "--no-fetch" => args.no_fetch = true,
+ _ => {}
+ }
+ let out = run(Case {
+ hub: hub.clone(),
+ cwd: cwd.clone(),
+ url: Some("http://127.0.0.1:1".to_string()),
+ launcher: forbidden_launcher(),
+ hosted: HostedKind::None,
+ args,
+ })
+ .await;
+ assert_eq!(out.exit, EXIT_NO_ANSWER, "{flag}");
+ let msg = out.stderr.join("\n");
+ assert!(msg.contains("excluded licence class"), "{flag}: {msg}");
+ assert!(
+ !msg.contains("xerj hub pull") && !msg.contains("run:"),
+ "{flag}: a hint to pull an excluded corpus is a way round the policy: {msg}"
+ );
+ assert!(!hub.join("packs").exists(), "{flag}: nothing downloaded");
+ assert!(!hub.join("node.pid").exists(), "{flag}: no node started");
+ }
+ // The command the old hint named refuses it too.
+ let rc = tokio::task::spawn_blocking({
+ let url = "http://127.0.0.1:1".to_string();
+ move || {
+ xerj_hub::cli::run_hub_cli(&[
+ "pull".into(),
+ "mini-c".into(),
+ "--url".into(),
+ url,
+ "--quiet".into(),
+ ])
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(rc, 1, "`xerj hub pull` itself refuses an excluded corpus");
+ assert!(!hub.join("packs").exists());
+ }
+
+ // ── F. a pull killed mid-run does not wedge its corpus ───────────────────
+ // The old pull lock was a directory removed only by the pull's own destructor:
+ // SIGKILL, an OOM kill or an agent's tool timeout left the corpus un-pullable
+ // for half an hour, with no owner recorded. It is an OS lock now.
+ #[cfg(unix)]
+ {
+ use std::os::unix::ffi::OsStrExt;
+
+ let hub = work.join("hub-f");
+ let code = work.join("code-f");
+ publish(&archive, &sig, &work.join("net-f"), &hub);
+ let real_catalogue = std::fs::read(hub.join("catalogue.json")).unwrap();
+ set_env(&hub, &code, &scratch_home, &public);
+ let url = spawn_node(&work.join("data-f"), allowlist.clone()).await;
+
+ // A pack URL that never delivers a byte: a FIFO with no writer. The pull takes
+ // its lock, starts "downloading" and blocks in open(2).
+ let fifo = work.join("net-f").join("never.fifo");
+ let fifo_c = std::ffi::CString::new(fifo.as_os_str().as_bytes()).unwrap();
+ assert_eq!(unsafe { libc::mkfifo(fifo_c.as_ptr(), 0o600) }, 0, "mkfifo");
+ set_catalogue_field(
+ &hub,
+ "pack_url",
+ json!(format!("file://{}", fifo.display())),
+ );
+
+ // The victim is a separate process running `xerj hub pull` (this binary).
+ let mut victim = Command::new(std::env::current_exe().unwrap())
+ .args([
+ "--ignored",
+ "--exact",
+ "child_main",
+ "--nocapture",
+ "--",
+ "hub",
+ "pull",
+ "mini-c",
+ "--url",
+ &url,
+ "--quiet",
+ ])
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .spawn()
+ .unwrap();
+ let lock_path = hub
+ .join("packs")
+ .join("mini-c")
+ .join(format!("{PIN8}.lock"));
+ wait_for(
+ "the victim to take the pull lock",
+ Duration::from_secs(90),
+ || lock_is_held(&lock_path),
+ )
+ .await;
+ assert!(
+ hub.join("packs")
+ .join("mini-c")
+ .join(format!("{PIN8}.tmp"))
+ .is_dir(),
+ "it is mid-pull: past the lock, into its temp dir"
+ );
+ assert!(
+ victim.try_wait().unwrap().is_none(),
+ "still blocked in the download"
+ );
+
+ // SIGKILL: no destructor runs. (The kernel closes its descriptors.)
+ victim.kill().unwrap();
+ victim.wait().unwrap();
+ assert!(!lock_is_held(&lock_path), "the dead process holds nothing");
+
+ // The retry is NOT blocked — and, pointed at the real pack, it completes.
+ std::fs::write(hub.join("catalogue.json"), &real_catalogue).unwrap();
+ let t0 = Instant::now();
+ let rc = tokio::task::spawn_blocking({
+ let url = url.clone();
+ move || {
+ xerj_hub::cli::run_hub_cli(&[
+ "pull".into(),
+ "mini-c".into(),
+ "--url".into(),
+ url,
+ "--quiet".into(),
+ ])
+ }
+ })
+ .await
+ .unwrap();
+ assert_eq!(rc, 0, "the retry after a killed pull succeeds");
+ assert!(
+ t0.elapsed() < Duration::from_secs(45),
+ "and does not sit out a 30-minute lock: {:?}",
+ t0.elapsed()
+ );
+ assert!(hub
+ .join("packs")
+ .join("mini-c")
+ .join(PIN8)
+ .join("pack.json")
+ .is_file());
+ let _ = std::fs::remove_file(&fifo);
+ }
+}
+
+/// Set one field of the first catalogue entry in `<hub>/catalogue.json`.
+fn set_catalogue_field(hub: &Path, field: &str, value: serde_json::Value) {
+ let path = hub.join("catalogue.json");
+ let mut cat: serde_json::Value =
+ serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
+ cat["corpora"][0][field] = value;
+ std::fs::write(&path, serde_json::to_vec(&cat).unwrap()).unwrap();
+}
+
+/// Is somebody holding the lock file `path`? (Tries it, and lets go again.)
+#[cfg(unix)]
+fn lock_is_held(path: &Path) -> bool {
+ let Ok(f) = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(false)
+ .open(path)
+ else {
+ return false;
+ };
+ match f.try_lock() {
+ Ok(()) => {
+ let _ = f.unlock();
+ false
+ }
+ Err(std::fs::TryLockError::WouldBlock) => true,
+ Err(std::fs::TryLockError::Error(e)) => panic!("cannot probe {}: {e}", path.display()),
+ }
}
diff --git a/engine/crates/xerj-hub/tests/pack_roundtrip.rs b/engine/crates/xerj-hub/tests/pack_roundtrip.rs
index beea2fafa..f26d014f2 100644
--- a/engine/crates/xerj-hub/tests/pack_roundtrip.rs
+++ b/engine/crates/xerj-hub/tests/pack_roundtrip.rs
@@ -221,6 +221,34 @@ fn write_catalogue(hub_home: &Path, archive: &Path, pack_sha256: &str) {
/// `write_catalogue` with an explicit advertised `pack_bytes` — lets a test lie
/// about the size to exercise the size cap.
fn write_catalogue_bytes(hub_home: &Path, archive: &Path, pack_sha256: &str, pack_bytes: u64) {
+ write_catalogue_full(
+ hub_home,
+ archive,
+ pack_sha256,
+ pack_bytes,
+ LicenceClass::AdaptWithAttribution,
+ );
+}
+
+/// `write_catalogue` with an explicit licence class — lets a test publish an
+/// `excluded` (or `approach-only`) corpus.
+fn write_catalogue_class(
+ hub_home: &Path,
+ archive: &Path,
+ pack_sha256: &str,
+ licence_class: LicenceClass,
+) {
+ let real = std::fs::metadata(archive).map(|m| m.len()).unwrap_or(0);
+ write_catalogue_full(hub_home, archive, pack_sha256, real, licence_class);
+}
+
+fn write_catalogue_full(
+ hub_home: &Path,
+ archive: &Path,
+ pack_sha256: &str,
+ pack_bytes: u64,
+ licence_class: LicenceClass,
+) {
let sig = {
let mut s = archive.as_os_str().to_os_string();
s.push(".sig");
@@ -235,7 +263,7 @@ fn write_catalogue_bytes(hub_home: &Path, archive: &Path, pack_sha256: &str, pac
pin_date: PIN_DATE.into(),
refresh_cadence_days: 7,
licence_spdx: "MIT".into(),
- licence_class: LicenceClass::AdaptWithAttribution,
+ licence_class,
pack_bytes,
pack_url: format!("file://{}", archive.display()),
pack_sha256: pack_sha256.to_string(),
@@ -546,13 +574,17 @@ async fn pack_build_pull_def_roundtrip() {
);
}
- // ── No verifying key → verify-fail exit 3, and NO temp dir is leaked ──
+ // ── No verifying key → its OWN exit code (5), NOT "failed verification" (3) ──
{
let hh = work.join("hub-nokey");
write_catalogue(&hh, &archive, &good_sha);
// HUB_PUBKEY_HEX is "" in this build, so an unset env var leaves no key.
let rc = run_pull(&work.join("code-nokey"), &hh, None, &url_b).await;
- assert_eq!(rc, 3, "no key → verification failure exit 3");
+ assert_eq!(
+ rc, 5,
+ "no key → \"this build cannot verify anything\" (5), which is not exit 3: the \
+ bytes were never judged, let alone found wanting"
+ );
assert!(
!hh.join("packs/mini-c").join(format!("{PIN8}.tmp")).exists(),
"the empty-key path must not leak a .tmp to re-download"
@@ -563,6 +595,112 @@ async fn pack_build_pull_def_roundtrip() {
);
}
+ // ── An EXCLUDED corpus is refused by `hub pull` itself (plan §11) ──
+ {
+ let hh = work.join("hub-excluded");
+ write_catalogue_class(&hh, &archive, &good_sha, LicenceClass::Excluded);
+ let rc = run_pull(&work.join("code-excluded"), &hh, Some(&public), &url_b).await;
+ assert_eq!(rc, 1, "an excluded corpus is a policy refusal (exit 1)");
+ assert!(
+ !hh.join("packs").exists(),
+ "nothing was downloaded, extracted or even given a directory"
+ );
+ assert!(
+ !work.join("code-excluded").join("state").exists(),
+ "and no ledger was written"
+ );
+ // An approach-only corpus is a different matter: it is pulled (with a warning
+ // wherever it is answered).
+ let hh = work.join("hub-approach");
+ write_catalogue_class(&hh, &archive, &good_sha, LicenceClass::ApproachOnly);
+ let rc = run_pull(&work.join("code-approach"), &hh, Some(&public), &url_b).await;
+ assert_eq!(rc, 0, "approach-only is allowed through");
+ }
+
+ // ── Everything that can refuse a pull is decided BEFORE a node is started ──
+ {
+ // No --url, XERJ_URL on a silent port: the old order started (and waited for)
+ // a whole node first and only then looked at the slug, the licence, the size
+ // and the key.
+ let hh = work.join("hub-preflight");
+ write_catalogue_class(&hh, &archive, &good_sha, LicenceClass::Excluded);
+ let silent = std::net::TcpListener::bind("127.0.0.1:0")
+ .unwrap()
+ .local_addr()
+ .unwrap()
+ .port();
+ let silent_url = format!("http://127.0.0.1:{silent}");
+ for (slug, want_rc, why) in [
+ ("mini-c", 1, "excluded"),
+ ("no-such-corpus", 2, "unknown slug"),
+ ("../../etc", 2, "malformed slug"),
+ ] {
+ let rc = run_pull_without_url(
+ &work.join("code-preflight"),
+ &hh,
+ Some(&public),
+ &silent_url,
+ slug,
+ )
+ .await;
+ assert_eq!(rc, want_rc, "{why}");
+ for started in [
+ "node",
+ "node.pid",
+ "node.port",
+ "node.log",
+ "node.toml",
+ "node.lock",
+ ] {
+ assert!(
+ !hh.join(started).exists(),
+ "{why}: `{started}` exists, so a node was prepared or started before the \
+ pull was refused"
+ );
+ }
+ }
+ std::env::remove_var("XERJ_URL");
+ }
+
+ // ── A pull that fails verification must not delete the pack an earlier pull
+ // verified (it used to `remove_dir_all` the extract on ANY failure) ──
+ {
+ let code_home = work.join("code-keep");
+ let hh = work.join("hub-keep");
+ write_catalogue(&hh, &archive, &good_sha);
+ let rc = run_pull(&code_home, &hh, Some(&public), &url_b).await;
+ assert_eq!(rc, 0, "the first pull succeeds");
+ let extract = hh.join("packs").join("mini-c").join(PIN8);
+ assert!(extract.join("pack.json").is_file());
+ assert!(extract.join("snapshot").is_dir());
+
+ // A re-pull whose archive no longer matches the catalogue's hash.
+ let flipped = out.join("flipped.hubpack.tar.zst");
+ write_catalogue(&hh, &flipped, &good_sha);
+ let rc = run_pull(&code_home, &hh, Some(&public), &url_b).await;
+ assert_eq!(rc, 3, "the bad archive is refused");
+ assert!(
+ extract.join("pack.json").is_file() && extract.join("snapshot").is_dir(),
+ "the pack verified earlier is still there: a failed re-pull deletes only what \
+ that run created"
+ );
+ assert!(
+ !hh.join("packs/mini-c").join(format!("{PIN8}.tmp")).exists(),
+ "and the failed run's own temp is gone"
+ );
+
+ // A good re-pull replaces it whole and leaves no staging behind.
+ write_catalogue(&hh, &archive, &good_sha);
+ let rc = run_pull(&code_home, &hh, Some(&public), &url_b).await;
+ assert_eq!(rc, 0);
+ let leftovers: Vec<String> = std::fs::read_dir(hh.join("packs").join("mini-c"))
+ .unwrap()
+ .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
+ .filter(|n| n != PIN8 && n != &format!("{PIN8}.lock"))
+ .collect();
+ assert!(leftovers.is_empty(), "no temp/staging left: {leftovers:?}");
+ }
+
// ── An unreachable node is an INFRA failure (exit 1), not 2 or 3 ──
{
let hh = work.join("hub-infra");
@@ -577,15 +715,49 @@ async fn pack_build_pull_def_roundtrip() {
assert_eq!(rc, 1, "unreachable node → infra exit 1 (distinct from 2/3)");
}
- // ── A concurrent pull of the same pin is refused by the lock (exit 1) ──
+ // ── A concurrent pull of the same pin: the second WAITS for the first ──
+ // (it used to be refused outright, and a lock left by a killed pull blocked the
+ // corpus for half an hour; the lock is now the OS's, held by a live process.)
{
let hh = work.join("hub-lock");
write_catalogue(&hh, &archive, &good_sha);
let lockdir = hh.join("packs").join("mini-c");
std::fs::create_dir_all(&lockdir).unwrap();
- std::fs::create_dir(lockdir.join(format!("{PIN8}.lock"))).unwrap(); // simulate in-flight pull
+ // Another process (here: another open file description) holds the pin's lock.
+ let holder = std::fs::OpenOptions::new()
+ .create(true)
+ .read(true)
+ .write(true)
+ .truncate(false)
+ .open(lockdir.join(format!("{PIN8}.lock")))
+ .unwrap();
+ holder.try_lock().unwrap();
+ let release = std::thread::spawn(move || {
+ std::thread::sleep(Duration::from_millis(1500));
+ drop(holder); // the first pull finishes
+ });
+ let t0 = std::time::Instant::now();
let rc = run_pull(&work.join("code-lock"), &hh, Some(&public), &url_b).await;
- assert_eq!(rc, 1, "a held pull lock → exit 1");
+ release.join().unwrap();
+ assert_eq!(
+ rc, 0,
+ "the second pull waited for the first, then went ahead"
+ );
+ assert!(
+ t0.elapsed() >= Duration::from_millis(1200),
+ "and it really did wait: {:?}",
+ t0.elapsed()
+ );
+ }
+
+ // ── A lock DIRECTORY left by an older build blocks nothing ──
+ {
+ let hh = work.join("hub-oldlock");
+ write_catalogue(&hh, &archive, &good_sha);
+ let lockdir = hh.join("packs").join("mini-c");
+ std::fs::create_dir_all(lockdir.join(format!("{PIN8}.lock"))).unwrap();
+ let rc = run_pull(&work.join("code-oldlock"), &hh, Some(&public), &url_b).await;
+ assert_eq!(rc, 0, "a stale lock directory is replaced, not obeyed");
}
// ── A traversal slug is rejected before ANY filesystem write (exit 2) ──
@@ -620,6 +792,37 @@ async fn run_pull(code: &Path, hub: &Path, pubkey: Option<&str>, url: &str) -> i
run_pull_slug(code, hub, pubkey, url, "mini-c").await
}
+/// `xerj hub pull <slug> --quiet` with NO `--url`: the node comes from `XERJ_URL`
+/// (here a silent loopback port), so the automatic-node path is what would run if
+/// the pull got that far.
+async fn run_pull_without_url(
+ code: &Path,
+ hub: &Path,
+ pubkey: Option<&str>,
+ xerj_url: &str,
+ slug: &str,
+) -> i32 {
+ let (code, hub, pubkey, xerj_url, slug) = (
+ code.to_path_buf(),
+ hub.to_path_buf(),
+ pubkey.map(str::to_string),
+ xerj_url.to_string(),
+ slug.to_string(),
+ );
+ tokio::task::spawn_blocking(move || {
+ std::env::set_var("XERJ_CODE_HOME", &code);
+ std::env::set_var("XERJ_HUB_HOME", &hub);
+ std::env::set_var("XERJ_URL", &xerj_url);
+ match pubkey {
+ Some(pk) => std::env::set_var("XERJ_HUB_PUBKEY", &pk),
+ None => std::env::remove_var("XERJ_HUB_PUBKEY"),
+ }
+ xerj_hub::cli::run_hub_cli(&["pull".into(), slug, "--quiet".into()])
+ })
+ .await
+ .unwrap()
+}
+
async fn run_pull_slug(
code: &Path,
hub: &Path,