AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(hub): never route to or answer from an excluded corpus in the ask flow; cap MCP slugs
xerj-org67fdfe492c8a
diff --git a/engine/crates/xerj-hub/src/ask.rs b/engine/crates/xerj-hub/src/ask.rsindex 3b5c25591..937e4ec85 100644--- a/engine/crates/xerj-hub/src/ask.rs+++ b/engine/crates/xerj-hub/src/ask.rs@@ -518,6 +518,20 @@ pub fn ask(args: &AskArgs, deps: &AskDeps<'_>) -> AskOutcome {return out;};let entry = deps.catalogue.corpora.iter().find(|e| e.slug == slug);+ // An excluded licence class is never served — not from a pack that happens to+ // be on disk (pulled before a reclassification, or copied in by hand), and not+ // by the hosted node, whose `ask_answer` routes over the non-excluded corpora+ // only and whose answer endpoints refuse one. `xerj_hub_find` never lists one;+ // naming it explicitly gets this plain no rather than an answer.+ if let Some(e) = entry.filter(|e| e.licence_class == LicenceClass::Excluded) {+ out.stderr.push(format!(+ "'{}' has an excluded licence class ({}) and is never served or pulled",+ printable(&slug),+ printable(&e.licence_spdx)+ ));+ out.exit = EXIT_NO_ANSWER;+ return out;+ }let req = Request {slug: &slug,question: &args.question,@@ -658,6 +672,39 @@ fn with_symbol_evidence(routing: Routing, hits: &[String]) -> Routing {}}+/// The catalogue without its excluded-licence corpora: what routing, the+/// "closest corpora" hint and `find` may ever name. Mirrors the gRPC+/// `ask_answer`'s `routable` catalogue.+fn routable(cat: &Catalogue) -> Catalogue {+ Catalogue {+ format_version: cat.format_version,+ generated_at: cat.generated_at.clone(),+ corpora: cat+ .corpora+ .iter()+ .filter(|e| e.licence_class != LicenceClass::Excluded)+ .cloned()+ .collect(),+ }+}++/// A named corpus is echoed back at most this many bytes long (the hub's own+/// slugs are at most 64; anything longer is not one, and a caller-supplied+/// string is not repeated back in full).+const SLUG_ECHO_MAX: usize = 64;++/// `printable`, cut to [`SLUG_ECHO_MAX`] bytes on a character boundary.+fn printable_slug(s: &str) -> String {+ if s.len() <= SLUG_ECHO_MAX {+ return printable(s);+ }+ let mut end = SLUG_ECHO_MAX;+ while !s.is_char_boundary(end) {+ end -= 1;+ }+ format!("{}…", printable(&s[..end]))+}+/// Step 2. `None` means `out` already holds the verdict (an exit code and its/// message / candidate lines).fn select_corpus(@@ -673,31 +720,33 @@ fn select_corpus(}out.stderr.push(format!("unknown corpus '{}' (catalogue generated {})",- printable(c),+ printable_slug(c),printable(&deps.catalogue.generated_at)));out.exit = EXIT_USAGE;return None;}+ // Route over the corpora that can be served at all: an excluded corpus is+ // never picked, never offered as ambiguous, never named as "closest".+ let catalogue = &routable(deps.catalogue);let cached = cached_slugs(deps.hub_home);// A definition lookup names its symbol: route on the symbol itself as well as// on the sentence around it.let by_symbol = symbol- .map(|s| symbol_hits(deps.catalogue, s.trim()))+ .map(|s| symbol_hits(catalogue, s.trim())).unwrap_or_default();// Route on the catalogue alone first. The hosted node is asked only when// that is not decisive, so a question a local pack can answer never leaves// the machine.- let (routing, mut ranked) =- route_ranked(&args.question, deps.cwd, deps.catalogue, None, &cached);+ let (routing, mut ranked) = route_ranked(&args.question, deps.cwd, catalogue, None, &cached);let mut routing = with_symbol_evidence(routing, &by_symbol);if !matches!(routing, Routing::Confident(_)) {if let Some(h) = hosted {let (hosted_routing, hosted_ranked) = route_ranked(&args.question,deps.cwd,- deps.catalogue,+ catalogue,Some(&SearchOnly(h)),&cached,);@@ -705,14 +754,37 @@ fn select_corpus(ranked = hosted_ranked;}}+ // Hosted search evidence can still name an excluded corpus (the local+ // catalogue knows its class even when the hosted node's ranking does not+ // say): drop it from the verdict. A slug the local catalogue does not list at+ // all is left alone — the hosted node may know a newer catalogue.+ let excluded = |s: &String| {+ deps.catalogue+ .corpora+ .iter()+ .any(|e| &e.slug == s && e.licence_class == LicenceClass::Excluded)+ };+ let known = |s: &String| catalogue.corpora.iter().any(|e| &e.slug == s);+ let routing = match routing {+ Routing::Confident(slug) if excluded(&slug) => Routing::NoCorpus,+ Routing::Ambiguous(slugs) => {+ let kept: Vec<String> = slugs.into_iter().filter(|s| !excluded(s)).collect();+ // One left is still not a confident pick: the asker decides.+ if kept.is_empty() {+ Routing::NoCorpus+ } else {+ Routing::Ambiguous(kept)+ }+ }+ other => other,+ };match routing {Routing::Confident(slug) => Some(slug),Routing::Ambiguous(slugs) => {let lines: Vec<String> = slugs.iter().map(|s| {- let ask_for = deps- .catalogue+ let ask_for = catalogue.corpora.iter().find(|e| &e.slug == s)@@ -734,7 +806,7 @@ fn select_corpus(// starting over from "no".let closest: Vec<String> = ranked.iter()- .filter(|(slug, _)| deps.catalogue.corpora.iter().any(|e| &e.slug == slug))+ .filter(|(slug, _)| known(slug)).take(CLOSEST_SHOWN).map(|(slug, _)| printable(slug)).collect();@@ -1556,6 +1628,7 @@ pub fn find(catalogue: &Catalogue,) -> FindOutcome {let k = k.clamp(1, FIND_MAX_K);+ let catalogue = &routable(catalogue);let cached = cached_slugs(hub_home);let (_, symbol) = detect_mode(question, &ModeRequest::Auto);let by_symbol = symbol@@ -2175,28 +2248,28 @@ mod tests {fn an_excluded_corpus_is_never_pulled() {let rig = Rig::new();let cat = catalogue(vec![lz4(1024, LicenceClass::Excluded)]);- // Hosted answers, but no pull is started.- let backend = FakeBackend::new(false);- let out = rig.run(- &args("q", Some("lz4")),- &cat,- Some(&StubHosted::new()),- &backend,- );- assert_eq!(out.exit, EXIT_OK);- assert!(!backend.called("pull_"), "{:?}", backend.calls());- assert!(- out.stdout.contains("excluded licence (MIT)"),- "{}",- out.stdout- );-- // With no hosted answer to fall back on, there is nothing to say but no.- let backend = FakeBackend::new(false);- let out = rig.run(&args("q", Some("lz4")), &cat, None, &backend);- assert_eq!(out.exit, EXIT_NO_ANSWER);- assert!(!backend.called("pull_"), "{:?}", backend.calls());- assert!(out.stderr[0].contains("never pulled"), "{:?}", out.stderr);+ // Neither the hosted node (it never serves an excluded corpus) nor a pull:+ // with or without a hosted hub, there is nothing to say but no.+ for hosted in [None, Some(StubHosted::new())] {+ let backend = FakeBackend::new(false);+ let out = rig.run(+ &args("q", Some("lz4")),+ &cat,+ hosted.as_ref().map(|h| h as &dyn HostedHub),+ &backend,+ );+ assert_eq!(out.exit, EXIT_NO_ANSWER);+ assert!(!backend.called("pull_"), "{:?}", backend.calls());+ assert!(+ out.stderr[0].contains("excluded licence class (MIT)")+ && out.stderr[0].contains("never served or pulled"),+ "{:?}",+ out.stderr+ );+ if let Some(h) = &hosted {+ assert_eq!(h.answers.get(), 0);+ }+ }}#[test]@@ -2396,12 +2469,12 @@ mod tests {assert_eq!(out.exit, EXIT_NO_ANSWER, "{name}");assert!(out.stderr[0].contains("excluded licence class")- && out.stderr[0].contains("never pulled"),+ && out.stderr[0].contains("never served or pulled"),"{name}: {:?}",out.stderr);no_pull_command(&out);- assert_eq!(backend.calls(), ["ready:lz4"], "{name}: nothing pulled");+ assert!(backend.calls().is_empty(), "{name}: nothing read or pulled");}}@@ -2451,20 +2524,22 @@ mod tests {}#[test]- fn a_hosted_excluded_answer_carries_no_pull_command_either() {+ fn a_named_excluded_corpus_is_refused_with_no_pull_command_either() {+ // The hosted node is not asked: it never serves an excluded corpus.let rig = Rig::new();let cat = catalogue(vec![lz4(80 * 1024 * 1024, LicenceClass::Excluded)]);let backend = FakeBackend::new(false);- let out = rig.run(- &args("q", Some("lz4")),- &cat,- Some(&StubHosted::new()),- &backend,+ let hosted = StubHosted::new();+ let out = rig.run(&args("q", Some("lz4")), &cat, Some(&hosted), &backend);+ assert_eq!(out.exit, EXIT_NO_ANSWER);+ assert!(+ out.stderr[0].contains("never served or pulled"),+ "{:?}",+ out.stderr);- assert_eq!(out.exit, EXIT_OK);- assert!(out.stdout.contains("never pulled"), "{}", out.stdout);no_pull_command(&out);- assert!(!backend.called("pull_"));+ assert!(backend.calls().is_empty(), "{:?}", backend.calls());+ assert_eq!(hosted.answers.get(), 0);}// ── hosted answers are attributed to the corpus that was asked ──────────@@ -3275,6 +3350,94 @@ mod tests {);}+ // ── an excluded corpus is never routed to or answered ───────────────────++ /// Review finding (rev7): `xerj_hub_find` hid an excluded corpus, but the ask+ /// flow behind `xerj_hub_ask` / `xerj hub ask` still routed to it, named it as+ /// a "closest corpus" to re-run with, and — given a pack on disk — answered+ /// from it at step 3 without any licence check. The gRPC `ask_answer` already+ /// routes over the non-excluded corpora only; the local flow now does the same.+ #[test]+ fn rev7_mcp_flow_routes_to_and_serves_an_excluded_corpus_find_hides() {+ let rig = Rig::new();+ let cat = catalogue(with_fillers(vec![+ lz4(1024, LicenceClass::Excluded),+ entry(+ "zstd",+ 1024,+ LicenceClass::AdaptWithAttribution,+ &["zstd", "frame"],+ ),+ ]));+ // A pack of the excluded corpus is on disk (pulled before it was+ // reclassified, or copied in by hand).+ let pack = rig.hub.path().join("packs").join("lz4").join("00000000");+ fs::create_dir_all(&pack).unwrap();+ fs::write(pack.join("pack.json"), "{}").unwrap();++ // find hides it ...+ let found = find("lz4 block decoder", 5, rig.cwd.path(), rig.hub.path(), &cat);+ assert!(found.corpora.iter().all(|c| c.slug != "lz4"), "{found:?}");++ // ... so ask, named or routed, must neither answer from it nor point at it.+ for (question, corpus) in [+ ("q", Some("lz4")),+ ("lz4 block decoder", None),+ ("how does the lz4 block decoder do compression", None),+ ("lz4", None),+ ] {+ for hosted in [None, Some(StubHosted::new())] {+ let backend = FakeBackend::new(true);+ let out = rig.run(+ &args(question, corpus),+ &cat,+ hosted.as_ref().map(|h| h as &dyn HostedHub),+ &backend,+ );+ let ctx = format!("{question:?} corpus={corpus:?} hosted={}", hosted.is_some());+ assert!(out.answer.is_none(), "{ctx}: answered {:?}", out.answer);+ assert_ne!(out.exit, EXIT_OK, "{ctx}");+ assert!(+ !backend.called("answer_local") && !backend.called("ready:lz4"),+ "{ctx}: {:?}",+ backend.calls()+ );+ assert!(!backend.called("pull_"), "{ctx}: {:?}", backend.calls());+ if let Some(h) = &hosted {+ assert_eq!(h.answers.get(), 0, "{ctx}: hosted asked about lz4");+ }+ let all = format!("{}\n{}", out.stdout, out.stderr.join("\n"));+ if corpus.is_some() {+ // Named explicitly: a plain refusal, not "unknown corpus".+ assert!(all.contains("excluded licence class"), "{ctx}: {all}");+ assert!(!all.contains("xerj hub pull"), "{ctx}: {all}");+ } else {+ assert!(!all.contains("lz4"), "{ctx}: points at it: {all}");+ }+ }+ }+ }++ #[test]+ fn an_unknown_named_corpus_is_echoed_back_cut_to_the_slug_cap() {+ let rig = Rig::new();+ let cat = catalogue(vec![lz4(1024, LicenceClass::AdaptWithAttribution)]);+ let backend = FakeBackend::new(false);+ let long = "z".repeat(200 * 1024);+ let out = rig.run(&args("q", Some(&long)), &cat, None, &backend);+ assert_eq!(out.exit, EXIT_USAGE);+ let msg = out.stderr.join("\n");+ assert!(msg.contains("unknown corpus"), "{msg}");+ assert!(msg.len() < 200, "{} bytes", msg.len());+ assert!(+ msg.contains(&format!("{}…", "z".repeat(SLUG_ECHO_MAX))),+ "{msg}"+ );+ // A short unknown slug is shown whole.+ let out = rig.run(&args("q", Some("nope")), &cat, None, &backend);+ assert!(out.stderr[0].contains("'nope'"), "{:?}", out.stderr);+ }+// ── find (MCP `xerj_hub_find`) ──────────────────────────────────────────#[test]diff --git a/engine/crates/xerj-mcp/src/hub.rs b/engine/crates/xerj-mcp/src/hub.rsindex 48a65d649..f237d4299 100644--- a/engine/crates/xerj-mcp/src/hub.rs+++ b/engine/crates/xerj-mcp/src/hub.rs@@ -161,15 +161,21 @@ fn text_arg(args: &Value, key: &str) -> Result<String, String> {Ok(t.to_string())}+/// The longest corpus slug a tool accepts — the server's `valid_slug` cap.+const MAX_SLUG_BYTES: usize = 64;+/// A corpus slug, checked before anything is looked up or built from it.fn slug_arg(args: &Value, key: &str, required: bool) -> Result<Option<String>, String> {match args.get(key) {None | Some(Value::Null) if !required => Ok(None),None | Some(Value::Null) => Err(format!("`{key}` is required")),- Some(Value::String(s)) if xerj_hub::cli::valid_slug(s) => Ok(Some(s.clone())),+ Some(Value::String(s)) if s.len() <= MAX_SLUG_BYTES && xerj_hub::cli::valid_slug(s) => {+ Ok(Some(s.clone()))+ }Some(Value::String(s)) if s.is_empty() && !required => Ok(None),Some(_) => Err(format!(- "`{key}` must be a corpus slug (lower-case letters, digits, '.', '_', '-')"+ "`{key}` must be a corpus slug (lower-case letters, digits, '.', '_', '-'; at most \+ {MAX_SLUG_BYTES} bytes)")),}}@@ -332,6 +338,30 @@ mod tests {assert_eq!(int_arg(&json!({ "k": 50 }), "k", 50).unwrap(), Some(50));}+ /// Review finding (rev7): `slug_arg` took a slug of any length (a 200 KB+ /// one was accepted and echoed back in "unknown corpus '…'"). The server's+ /// own `valid_slug` caps a slug at 64 bytes; the tool arguments do too.+ #[test]+ fn a_slug_longer_than_the_servers_cap_is_refused_and_never_echoed() {+ let at_cap = "a".repeat(MAX_SLUG_BYTES);+ assert_eq!(+ slug_arg(&json!({ "slug": at_cap }), "slug", true).unwrap(),+ Some(at_cap.clone())+ );+ for len in [MAX_SLUG_BYTES + 1, 200 * 1024] {+ let long = "a".repeat(len);+ for (key, required) in [("slug", true), ("corpus", false)] {+ let err = slug_arg(&json!({ key: long }), key, required)+ .expect_err("over the cap is refused");+ assert!(+ err.len() < 200,+ "the slug is not echoed back: {}",+ err.len()+ );+ }+ }+ }+#[test]fn an_ambiguous_route_is_not_an_error_and_a_failure_is() {let ambiguous = AskOutcome {