Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(hub): never route to or answer from an excluded corpus in the ask flow; cap MCP slugs

xerj-org67fdfe492c8a
diff --git a/engine/crates/xerj-hub/src/ask.rs b/engine/crates/xerj-hub/src/ask.rs
index 3b5c25591..937e4ec85 100644
--- a/engine/crates/xerj-hub/src/ask.rs
+++ b/engine/crates/xerj-hub/src/ask.rs
@@ -518,6 +518,20 @@ pub fn ask(args: &AskArgs, deps: &AskDeps<'_>) -> AskOutcome {
return out;
};
let entry = deps.catalogue.corpora.iter().find(|e| e.slug == slug);
+ // An excluded licence class is never served — not from a pack that happens to
+ // be on disk (pulled before a reclassification, or copied in by hand), and not
+ // by the hosted node, whose `ask_answer` routes over the non-excluded corpora
+ // only and whose answer endpoints refuse one. `xerj_hub_find` never lists one;
+ // naming it explicitly gets this plain no rather than an answer.
+ if let Some(e) = entry.filter(|e| e.licence_class == LicenceClass::Excluded) {
+ out.stderr.push(format!(
+ "'{}' has an excluded licence class ({}) and is never served or pulled",
+ printable(&slug),
+ printable(&e.licence_spdx)
+ ));
+ out.exit = EXIT_NO_ANSWER;
+ return out;
+ }
let req = Request {
slug: &slug,
question: &args.question,
@@ -658,6 +672,39 @@ fn with_symbol_evidence(routing: Routing, hits: &[String]) -> Routing {
}
}
+/// The catalogue without its excluded-licence corpora: what routing, the
+/// "closest corpora" hint and `find` may ever name. Mirrors the gRPC
+/// `ask_answer`'s `routable` catalogue.
+fn routable(cat: &Catalogue) -> Catalogue {
+ Catalogue {
+ format_version: cat.format_version,
+ generated_at: cat.generated_at.clone(),
+ corpora: cat
+ .corpora
+ .iter()
+ .filter(|e| e.licence_class != LicenceClass::Excluded)
+ .cloned()
+ .collect(),
+ }
+}
+
+/// A named corpus is echoed back at most this many bytes long (the hub's own
+/// slugs are at most 64; anything longer is not one, and a caller-supplied
+/// string is not repeated back in full).
+const SLUG_ECHO_MAX: usize = 64;
+
+/// `printable`, cut to [`SLUG_ECHO_MAX`] bytes on a character boundary.
+fn printable_slug(s: &str) -> String {
+ if s.len() <= SLUG_ECHO_MAX {
+ return printable(s);
+ }
+ let mut end = SLUG_ECHO_MAX;
+ while !s.is_char_boundary(end) {
+ end -= 1;
+ }
+ format!("{}…", printable(&s[..end]))
+}
+
/// Step 2. `None` means `out` already holds the verdict (an exit code and its
/// message / candidate lines).
fn select_corpus(
@@ -673,31 +720,33 @@ fn select_corpus(
}
out.stderr.push(format!(
"unknown corpus '{}' (catalogue generated {})",
- printable(c),
+ printable_slug(c),
printable(&deps.catalogue.generated_at)
));
out.exit = EXIT_USAGE;
return None;
}
+ // Route over the corpora that can be served at all: an excluded corpus is
+ // never picked, never offered as ambiguous, never named as "closest".
+ let catalogue = &routable(deps.catalogue);
let cached = cached_slugs(deps.hub_home);
// A definition lookup names its symbol: route on the symbol itself as well as
// on the sentence around it.
let by_symbol = symbol
- .map(|s| symbol_hits(deps.catalogue, s.trim()))
+ .map(|s| symbol_hits(catalogue, s.trim()))
.unwrap_or_default();
// Route on the catalogue alone first. The hosted node is asked only when
// that is not decisive, so a question a local pack can answer never leaves
// the machine.
- let (routing, mut ranked) =
- route_ranked(&args.question, deps.cwd, deps.catalogue, None, &cached);
+ let (routing, mut ranked) = route_ranked(&args.question, deps.cwd, catalogue, None, &cached);
let mut routing = with_symbol_evidence(routing, &by_symbol);
if !matches!(routing, Routing::Confident(_)) {
if let Some(h) = hosted {
let (hosted_routing, hosted_ranked) = route_ranked(
&args.question,
deps.cwd,
- deps.catalogue,
+ catalogue,
Some(&SearchOnly(h)),
&cached,
);
@@ -705,14 +754,37 @@ fn select_corpus(
ranked = hosted_ranked;
}
}
+ // Hosted search evidence can still name an excluded corpus (the local
+ // catalogue knows its class even when the hosted node's ranking does not
+ // say): drop it from the verdict. A slug the local catalogue does not list at
+ // all is left alone — the hosted node may know a newer catalogue.
+ let excluded = |s: &String| {
+ deps.catalogue
+ .corpora
+ .iter()
+ .any(|e| &e.slug == s && e.licence_class == LicenceClass::Excluded)
+ };
+ let known = |s: &String| catalogue.corpora.iter().any(|e| &e.slug == s);
+ let routing = match routing {
+ Routing::Confident(slug) if excluded(&slug) => Routing::NoCorpus,
+ Routing::Ambiguous(slugs) => {
+ let kept: Vec<String> = slugs.into_iter().filter(|s| !excluded(s)).collect();
+ // One left is still not a confident pick: the asker decides.
+ if kept.is_empty() {
+ Routing::NoCorpus
+ } else {
+ Routing::Ambiguous(kept)
+ }
+ }
+ other => other,
+ };
match routing {
Routing::Confident(slug) => Some(slug),
Routing::Ambiguous(slugs) => {
let lines: Vec<String> = slugs
.iter()
.map(|s| {
- let ask_for = deps
- .catalogue
+ let ask_for = catalogue
.corpora
.iter()
.find(|e| &e.slug == s)
@@ -734,7 +806,7 @@ fn select_corpus(
// starting over from "no".
let closest: Vec<String> = ranked
.iter()
- .filter(|(slug, _)| deps.catalogue.corpora.iter().any(|e| &e.slug == slug))
+ .filter(|(slug, _)| known(slug))
.take(CLOSEST_SHOWN)
.map(|(slug, _)| printable(slug))
.collect();
@@ -1556,6 +1628,7 @@ pub fn find(
catalogue: &Catalogue,
) -> FindOutcome {
let k = k.clamp(1, FIND_MAX_K);
+ let catalogue = &routable(catalogue);
let cached = cached_slugs(hub_home);
let (_, symbol) = detect_mode(question, &ModeRequest::Auto);
let by_symbol = symbol
@@ -2175,28 +2248,28 @@ mod tests {
fn an_excluded_corpus_is_never_pulled() {
let rig = Rig::new();
let cat = catalogue(vec![lz4(1024, LicenceClass::Excluded)]);
- // Hosted answers, but no pull is started.
- let backend = FakeBackend::new(false);
- let out = rig.run(
- &args("q", Some("lz4")),
- &cat,
- Some(&StubHosted::new()),
- &backend,
- );
- assert_eq!(out.exit, EXIT_OK);
- assert!(!backend.called("pull_"), "{:?}", backend.calls());
- assert!(
- out.stdout.contains("excluded licence (MIT)"),
- "{}",
- out.stdout
- );
-
- // With no hosted answer to fall back on, there is nothing to say but no.
- let backend = FakeBackend::new(false);
- let out = rig.run(&args("q", Some("lz4")), &cat, None, &backend);
- assert_eq!(out.exit, EXIT_NO_ANSWER);
- assert!(!backend.called("pull_"), "{:?}", backend.calls());
- assert!(out.stderr[0].contains("never pulled"), "{:?}", out.stderr);
+ // Neither the hosted node (it never serves an excluded corpus) nor a pull:
+ // with or without a hosted hub, there is nothing to say but no.
+ for hosted in [None, Some(StubHosted::new())] {
+ let backend = FakeBackend::new(false);
+ let out = rig.run(
+ &args("q", Some("lz4")),
+ &cat,
+ hosted.as_ref().map(|h| h as &dyn HostedHub),
+ &backend,
+ );
+ assert_eq!(out.exit, EXIT_NO_ANSWER);
+ assert!(!backend.called("pull_"), "{:?}", backend.calls());
+ assert!(
+ out.stderr[0].contains("excluded licence class (MIT)")
+ && out.stderr[0].contains("never served or pulled"),
+ "{:?}",
+ out.stderr
+ );
+ if let Some(h) = &hosted {
+ assert_eq!(h.answers.get(), 0);
+ }
+ }
}
#[test]
@@ -2396,12 +2469,12 @@ mod tests {
assert_eq!(out.exit, EXIT_NO_ANSWER, "{name}");
assert!(
out.stderr[0].contains("excluded licence class")
- && out.stderr[0].contains("never pulled"),
+ && out.stderr[0].contains("never served or pulled"),
"{name}: {:?}",
out.stderr
);
no_pull_command(&out);
- assert_eq!(backend.calls(), ["ready:lz4"], "{name}: nothing pulled");
+ assert!(backend.calls().is_empty(), "{name}: nothing read or pulled");
}
}
@@ -2451,20 +2524,22 @@ mod tests {
}
#[test]
- fn a_hosted_excluded_answer_carries_no_pull_command_either() {
+ fn a_named_excluded_corpus_is_refused_with_no_pull_command_either() {
+ // The hosted node is not asked: it never serves an excluded corpus.
let rig = Rig::new();
let cat = catalogue(vec![lz4(80 * 1024 * 1024, LicenceClass::Excluded)]);
let backend = FakeBackend::new(false);
- let out = rig.run(
- &args("q", Some("lz4")),
- &cat,
- Some(&StubHosted::new()),
- &backend,
+ let hosted = StubHosted::new();
+ let out = rig.run(&args("q", Some("lz4")), &cat, Some(&hosted), &backend);
+ assert_eq!(out.exit, EXIT_NO_ANSWER);
+ assert!(
+ out.stderr[0].contains("never served or pulled"),
+ "{:?}",
+ out.stderr
);
- assert_eq!(out.exit, EXIT_OK);
- assert!(out.stdout.contains("never pulled"), "{}", out.stdout);
no_pull_command(&out);
- assert!(!backend.called("pull_"));
+ assert!(backend.calls().is_empty(), "{:?}", backend.calls());
+ assert_eq!(hosted.answers.get(), 0);
}
// ── hosted answers are attributed to the corpus that was asked ──────────
@@ -3275,6 +3350,94 @@ mod tests {
);
}
+ // ── an excluded corpus is never routed to or answered ───────────────────
+
+ /// Review finding (rev7): `xerj_hub_find` hid an excluded corpus, but the ask
+ /// flow behind `xerj_hub_ask` / `xerj hub ask` still routed to it, named it as
+ /// a "closest corpus" to re-run with, and — given a pack on disk — answered
+ /// from it at step 3 without any licence check. The gRPC `ask_answer` already
+ /// routes over the non-excluded corpora only; the local flow now does the same.
+ #[test]
+ fn rev7_mcp_flow_routes_to_and_serves_an_excluded_corpus_find_hides() {
+ let rig = Rig::new();
+ let cat = catalogue(with_fillers(vec![
+ lz4(1024, LicenceClass::Excluded),
+ entry(
+ "zstd",
+ 1024,
+ LicenceClass::AdaptWithAttribution,
+ &["zstd", "frame"],
+ ),
+ ]));
+ // A pack of the excluded corpus is on disk (pulled before it was
+ // reclassified, or copied in by hand).
+ let pack = rig.hub.path().join("packs").join("lz4").join("00000000");
+ fs::create_dir_all(&pack).unwrap();
+ fs::write(pack.join("pack.json"), "{}").unwrap();
+
+ // find hides it ...
+ let found = find("lz4 block decoder", 5, rig.cwd.path(), rig.hub.path(), &cat);
+ assert!(found.corpora.iter().all(|c| c.slug != "lz4"), "{found:?}");
+
+ // ... so ask, named or routed, must neither answer from it nor point at it.
+ for (question, corpus) in [
+ ("q", Some("lz4")),
+ ("lz4 block decoder", None),
+ ("how does the lz4 block decoder do compression", None),
+ ("lz4", None),
+ ] {
+ for hosted in [None, Some(StubHosted::new())] {
+ let backend = FakeBackend::new(true);
+ let out = rig.run(
+ &args(question, corpus),
+ &cat,
+ hosted.as_ref().map(|h| h as &dyn HostedHub),
+ &backend,
+ );
+ let ctx = format!("{question:?} corpus={corpus:?} hosted={}", hosted.is_some());
+ assert!(out.answer.is_none(), "{ctx}: answered {:?}", out.answer);
+ assert_ne!(out.exit, EXIT_OK, "{ctx}");
+ assert!(
+ !backend.called("answer_local") && !backend.called("ready:lz4"),
+ "{ctx}: {:?}",
+ backend.calls()
+ );
+ assert!(!backend.called("pull_"), "{ctx}: {:?}", backend.calls());
+ if let Some(h) = &hosted {
+ assert_eq!(h.answers.get(), 0, "{ctx}: hosted asked about lz4");
+ }
+ let all = format!("{}\n{}", out.stdout, out.stderr.join("\n"));
+ if corpus.is_some() {
+ // Named explicitly: a plain refusal, not "unknown corpus".
+ assert!(all.contains("excluded licence class"), "{ctx}: {all}");
+ assert!(!all.contains("xerj hub pull"), "{ctx}: {all}");
+ } else {
+ assert!(!all.contains("lz4"), "{ctx}: points at it: {all}");
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn an_unknown_named_corpus_is_echoed_back_cut_to_the_slug_cap() {
+ let rig = Rig::new();
+ let cat = catalogue(vec![lz4(1024, LicenceClass::AdaptWithAttribution)]);
+ let backend = FakeBackend::new(false);
+ let long = "z".repeat(200 * 1024);
+ let out = rig.run(&args("q", Some(&long)), &cat, None, &backend);
+ assert_eq!(out.exit, EXIT_USAGE);
+ let msg = out.stderr.join("\n");
+ assert!(msg.contains("unknown corpus"), "{msg}");
+ assert!(msg.len() < 200, "{} bytes", msg.len());
+ assert!(
+ msg.contains(&format!("{}…", "z".repeat(SLUG_ECHO_MAX))),
+ "{msg}"
+ );
+ // A short unknown slug is shown whole.
+ let out = rig.run(&args("q", Some("nope")), &cat, None, &backend);
+ assert!(out.stderr[0].contains("'nope'"), "{:?}", out.stderr);
+ }
+
// ── find (MCP `xerj_hub_find`) ──────────────────────────────────────────
#[test]
diff --git a/engine/crates/xerj-mcp/src/hub.rs b/engine/crates/xerj-mcp/src/hub.rs
index 48a65d649..f237d4299 100644
--- a/engine/crates/xerj-mcp/src/hub.rs
+++ b/engine/crates/xerj-mcp/src/hub.rs
@@ -161,15 +161,21 @@ fn text_arg(args: &Value, key: &str) -> Result<String, String> {
Ok(t.to_string())
}
+/// The longest corpus slug a tool accepts — the server's `valid_slug` cap.
+const MAX_SLUG_BYTES: usize = 64;
+
/// A corpus slug, checked before anything is looked up or built from it.
fn slug_arg(args: &Value, key: &str, required: bool) -> Result<Option<String>, String> {
match args.get(key) {
None | Some(Value::Null) if !required => Ok(None),
None | Some(Value::Null) => Err(format!("`{key}` is required")),
- Some(Value::String(s)) if xerj_hub::cli::valid_slug(s) => Ok(Some(s.clone())),
+ Some(Value::String(s)) if s.len() <= MAX_SLUG_BYTES && xerj_hub::cli::valid_slug(s) => {
+ Ok(Some(s.clone()))
+ }
Some(Value::String(s)) if s.is_empty() && !required => Ok(None),
Some(_) => Err(format!(
- "`{key}` must be a corpus slug (lower-case letters, digits, '.', '_', '-')"
+ "`{key}` must be a corpus slug (lower-case letters, digits, '.', '_', '-'; at most \
+ {MAX_SLUG_BYTES} bytes)"
)),
}
}
@@ -332,6 +338,30 @@ mod tests {
assert_eq!(int_arg(&json!({ "k": 50 }), "k", 50).unwrap(), Some(50));
}
+ /// Review finding (rev7): `slug_arg` took a slug of any length (a 200 KB
+ /// one was accepted and echoed back in "unknown corpus '…'"). The server's
+ /// own `valid_slug` caps a slug at 64 bytes; the tool arguments do too.
+ #[test]
+ fn a_slug_longer_than_the_servers_cap_is_refused_and_never_echoed() {
+ let at_cap = "a".repeat(MAX_SLUG_BYTES);
+ assert_eq!(
+ slug_arg(&json!({ "slug": at_cap }), "slug", true).unwrap(),
+ Some(at_cap.clone())
+ );
+ for len in [MAX_SLUG_BYTES + 1, 200 * 1024] {
+ let long = "a".repeat(len);
+ for (key, required) in [("slug", true), ("corpus", false)] {
+ let err = slug_arg(&json!({ key: long }), key, required)
+ .expect_err("over the cap is refused");
+ assert!(
+ err.len() < 200,
+ "the slug is not echoed back: {}",
+ err.len()
+ );
+ }
+ }
+ }
+
#[test]
fn an_ambiguous_route_is_not_an_error_and_a_failure_is() {
let ambiguous = AskOutcome {