Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

mcp: token-budgeted responses — max_tokens on every search tool (#1058)

xerj-org732806534717
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 136345494..3852e2d1f 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -7567,6 +7567,7 @@ dependencies = [
"serde_json",
"tempfile",
"tokio",
+ "unicode-segmentation",
"xerj-common",
"xerj-rerank",
]
diff --git a/engine/crates/xerj-mcp/Cargo.toml b/engine/crates/xerj-mcp/Cargo.toml
index 3bbc980ee..18575f4ea 100644
--- a/engine/crates/xerj-mcp/Cargo.toml
+++ b/engine/crates/xerj-mcp/Cargo.toml
@@ -38,6 +38,13 @@ xerj-common = { path = "../xerj-common" }
# ceilings and the builder refuses exactly what the node would refuse — the
# two cannot drift. A light leaf: it adds no dependency this crate lacks.
xerj-rerank = { path = "../xerj-rerank" }
+# The `max_tokens` budget (#1058) counts with XERJ's StandardTokenizer —
+# UAX #29 word boundaries via `unicode_words()`. Depended on directly (the
+# same crate and the same call xerj-fts's StandardTokenizer makes,
+# engine/crates/xerj-fts/src/analyzer.rs) rather than pulling the whole FTS
+# stack into this thin proxy; the count is identical by construction.
+# See src/token_budget.rs.
+unicode-segmentation = "1"
[dev-dependencies]
# The loopback admin-key fallback tests sandbox cwd + HOME so discovery sees
diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rs
index 5cf81ef30..7f4c37ba5 100644
--- a/engine/crates/xerj-mcp/src/lib.rs
+++ b/engine/crates/xerj-mcp/src/lib.rs
@@ -76,6 +76,16 @@
//!
//! Diagnostics go to **stderr**; stdout is reserved exclusively for the
//! JSON-RPC stream.
+//!
+//! ## Token budgets (#1058)
+//!
+//! The five search tools accept `max_tokens` and are held to it: the response
+//! text NEVER exceeds the budget. Enforcement lives in [`token_budget`],
+//! counted with one named tokenizer (XERJ's `StandardTokenizer`, UAX #29
+//! words), with overlapping file:line passages deduped before budgeting and
+//! over-budget hits reduced to locators.
+
+mod token_budget;
use std::time::Duration;
@@ -509,6 +519,37 @@ fn rerank_arg_schema(query_required: bool) -> Value {
json!({ "type": ["object", "boolean"], "description": description })
}
+/// JSON-schema fragment for the optional `max_tokens` argument, shared by all
+/// five search tools (`xerj_search`, `xerj_hybrid_search`,
+/// `xerj_semantic_search`, `xerj_vector_search`, `xerj_code_search`) so the
+/// budget contract cannot drift between them.
+///
+/// The description answers the three questions an agent deciding to spend a
+/// budget has: what a token IS here (one named tokenizer, honestly scoped as
+/// an approximation), what happens when the budget binds (locators, not
+/// silent loss — and an accounting block that says exactly what was cut), and
+/// the floor below which the tool refuses rather than lie.
+fn max_tokens_arg_schema() -> Value {
+ json!({
+ "type": "integer",
+ "minimum": token_budget::MIN_MAX_TOKENS,
+ "description": format!(
+ "Token budget for this response: the returned text NEVER exceeds it. Tokens are \
+ counted with {} — the same word split the engine's own analyzer uses for BM25. \
+ Honest scope: that under-counts model-context subwords (snake_case is ONE token \
+ here), so treat it as a deterministic proxy, not a BPE count. The budget covers \
+ the whole response text (JSON keys and syntax included). Overlapping file:line \
+ passages are deduped first (higher-ranked wins). Hits that no longer fit are \
+ listed as LOCATORS ONLY (file:line, index/id) rather than dropped, and the \
+ response's `_token_budget` block accounts for every hit: kept, locator-only, \
+ deduped, dropped. Errors are never truncated — a refusal you cannot read \
+ end-to-end is worse than a long response. Minimum {}.",
+ token_budget::TOKENIZER_NAME,
+ token_budget::MIN_MAX_TOKENS,
+ )
+ })
+}
+
/// The ten tool specifications advertised via `tools/list`. Input schemas are
/// plain JSON Schema; every property maps onto a field the engine accepts.
///
@@ -531,7 +572,9 @@ pub fn tool_specs() -> Value {
Optional `rerank` adds a second stage that re-judges the top hits with \
an external relevance model — read that argument's description before \
using it: it needs a provider the operator configured, and it sends \
- document text off the machine.",
+ document text off the machine. \
+ Pass `max_tokens` to cap the response at a token budget (never exceeded; \
+ see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -549,7 +592,8 @@ pub fn tool_specs() -> Value {
"from": { "type": "integer", "description": "Offset for pagination." },
"sort": { "description": "ES sort clause (array or object)." },
"_source": { "description": "Source filtering (bool, field, or {includes,excludes})." },
- "rerank": rerank_arg_schema(false)
+ "rerank": rerank_arg_schema(false),
+ "max_tokens": max_tokens_arg_schema()
}
}
},
@@ -559,7 +603,9 @@ pub fn tool_specs() -> Value {
"Meaning-based search over a `semantic_text` field. The query text is \
embedded SERVER-SIDE by XERJ's built-in lexical embedder (no external \
API key), then matched by vector similarity. Proxies POST /{index}/_search \
- with {\"query\":{\"semantic\":{...}}}.",
+ with {\"query\":{\"semantic\":{...}}}. \
+ Pass `max_tokens` to cap the response at a token budget (never exceeded; \
+ see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -567,7 +613,8 @@ pub fn tool_specs() -> Value {
"field": { "type": "string", "description": "Name of the semantic_text field." },
"query": { "type": "string", "description": "Natural-language query text to embed and match." },
"k": { "type": "integer", "description": "Number of nearest results (default 10)." },
- "filter": { "type": "object", "description": "Optional ES query clause applied as a pre-filter." }
+ "filter": { "type": "object", "description": "Optional ES query clause applied as a pre-filter." },
+ "max_tokens": max_tokens_arg_schema()
},
"required": ["index", "field", "query"]
}
@@ -581,7 +628,9 @@ pub fn tool_specs() -> Value {
official bench query; num_candidates sets the beam width (floored at \
800). Filtered kNN, non-cosine metrics, SQ8 fields, and small indexes \
run an exact brute-force scan. \
- Proxies POST /{index}/_search with a top-level {\"knn\":{...}}.",
+ Proxies POST /{index}/_search with a top-level {\"knn\":{...}}. \
+ Pass `max_tokens` to cap the response at a token budget (never exceeded; \
+ see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -594,7 +643,8 @@ pub fn tool_specs() -> Value {
},
"k": { "type": "integer", "description": "Number of nearest neighbours (default 10)." },
"num_candidates": { "type": "integer", "description": "Optional candidate pool size." },
- "filter": { "type": "object", "description": "Optional ES query clause applied as a pre-filter." }
+ "filter": { "type": "object", "description": "Optional ES query clause applied as a pre-filter." },
+ "max_tokens": max_tokens_arg_schema()
},
"required": ["index", "field", "query_vector"]
}
@@ -608,7 +658,9 @@ pub fn tool_specs() -> Value {
{\"query\":{\"hybrid\":{\"queries\":[...],\"fusion\":...}}}. \
Optional `rerank` re-judges the fused top hits with an external \
relevance model; here `rerank.query` is REQUIRED, because a hybrid \
- search has several sub-queries and no single question to read.",
+ search has several sub-queries and no single question to read. \
+ Pass `max_tokens` to cap the response at a token budget (never exceeded; \
+ see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -633,7 +685,8 @@ pub fn tool_specs() -> Value {
"description": "Fusion strategy (default rrf)."
},
"size": { "type": "integer", "description": "Max fused hits to return." },
- "rerank": rerank_arg_schema(true)
+ "rerank": rerank_arg_schema(true),
+ "max_tokens": max_tokens_arg_schema()
},
"required": ["index", "queries"]
}
@@ -837,6 +890,8 @@ pub fn tool_specs() -> Value {
"warnings — AGPL/SSPL/Elastic/BUSL/GPL/LGPL/MPL sources are approach-only. ",
"Refuses indices older than 30 days (override with stale_ok). ",
"A no-match is NOT an error: it says the corpus is wrong for the task. ",
+ "Pass `max_tokens` to cap the response at a token budget (never exceeded; ",
+ "see that argument for the tokenizer and the locator-only rule). ",
"Corpora: `xerj corpus list`."),
"inputSchema": {
"type": "object",
@@ -849,7 +904,8 @@ pub fn tool_specs() -> Value {
"full": { "type": "integer", "description": "Max chars per passage (default 800; 0 = file head only)." },
"no_symbol": { "type": "boolean", "description": "Window selection instead of the matching definition." },
"stale_ok": { "type": "boolean", "description": "Override the 30-day staleness refusal." },
- "licence_policy": { "type": "string", "enum": ["warn", "strict"], "description": "strict strips passage text from restricted-licence hits, keeping locator + warning (default warn)." }
+ "licence_policy": { "type": "string", "enum": ["warn", "strict"], "description": "strict strips passage text from restricted-licence hits, keeping locator + warning (default warn)." },
+ "max_tokens": max_tokens_arg_schema()
},
"required": ["corpus", "query"]
}
@@ -874,6 +930,23 @@ async fn call_tool(ctx: &Ctx, msg: &Value) -> Value {
.cloned()
.unwrap_or_else(|| json!({}));
+ // #1058: `max_tokens` is enforced PROXY-SIDE on the response — the engine
+ // never sees it (the builders copy only their own known keys), so the
+ // request bodies are bit-identical with and without a budget. Only the
+ // four `_search`-proxying tools take it here; `xerj_code_search` reads it
+ // itself and budgets its rendered text.
+ let budget = if matches!(
+ name,
+ "xerj_search" | "xerj_hybrid_search" | "xerj_semantic_search" | "xerj_vector_search"
+ ) {
+ match token_budget::opt_max_tokens(&args) {
+ Ok(b) => b,
+ Err(msg) => return tool_text(msg, true),
+ }
+ } else {
+ None
+ };
+
let result = match name {
"xerj_search" => build_search(&args),
"xerj_semantic_search" => build_semantic(&args),
@@ -890,7 +963,7 @@ async fn call_tool(ctx: &Ctx, msg: &Value) -> Value {
};
match result {
- Ok((method, path, body)) => engine_request(ctx, method, &path, body).await,
+ Ok((method, path, body)) => engine_request(ctx, method, &path, body, budget).await,
Err(msg) => tool_text(msg, true),
}
}
@@ -1441,6 +1514,13 @@ async fn run_code_search(ctx: &Ctx, args: &Value) -> Value {
Some(q) if !q.is_empty() => q.to_string(),
_ => return tool_text("xerj_code_search: `query` is required", true),
};
+ // #1058: the budget is enforced on the RENDERED text (this tool's payload
+ // is the shared renderer's output, not the engine JSON). Refused here so
+ // a mistyped budget never silently means "no budget".
+ let budget = match token_budget::opt_max_tokens(args) {
+ Ok(b) => b,
+ Err(msg) => return tool_text(format!("xerj_code_search: {msg}"), true),
+ };
let mut p = xerj_common::xccode::CodeParams::new(&corpus, &query);
if let Some(k) = args.get("k") {
match k.as_u64().filter(|n| (1..=50).contains(n)) {
@@ -1500,6 +1580,14 @@ async fn run_code_search(ctx: &Ctx, args: &Value) -> Value {
text.push('\n');
}
text.push_str(&out.text);
+ // Budget only successful retrievals — errors pass through in full. A
+ // no-match is not an error, but its text has no hit blocks, so the text
+ // budgeter passes it through unchanged unless it alone exceeds the budget.
+ if !out.is_error {
+ if let Some(max_tokens) = budget {
+ text = token_budget::budget_code_text(&text, max_tokens);
+ }
+ }
tool_text(text, out.is_error)
}
@@ -1521,7 +1609,18 @@ fn code_root() -> std::path::PathBuf {
/// an MCP tool result. Non-2xx responses (and transport errors) come back as
/// `isError:true` with the engine's text verbatim, so the agent sees exactly
/// what the engine said (including its "not a graph database" refusals).
-async fn engine_request(ctx: &Ctx, method: Method, path: &str, body: Option<Value>) -> Value {
+///
+/// `budget` (#1058): when `Some(max_tokens)`, a SUCCESSFUL response is held
+/// to that token budget by [`token_budget::budget_json_response`]. Errors are
+/// never budgeted — a refusal the agent cannot read end-to-end is worse than
+/// a long response, and refusals are small anyway.
+async fn engine_request(
+ ctx: &Ctx,
+ method: Method,
+ path: &str,
+ body: Option<Value>,
+ budget: Option<usize>,
+) -> Value {
let url = format!("{}{}", ctx.base_url, path);
let mut req = match method {
Method::Get => ctx.client.get(&url),
@@ -1540,7 +1639,12 @@ async fn engine_request(ctx: &Ctx, method: Method, path: &str, body: Option<Valu
let status = resp.status();
let text = resp.text().await.unwrap_or_default();
if status.is_success() {
- tool_text(text, false)
+ match budget {
+ Some(max_tokens) => {
+ tool_text(token_budget::budget_json_response(&text, max_tokens), false)
+ }
+ None => tool_text(text, false),
+ }
} else {
tool_text(format!("XERJ returned HTTP {status}: {text}"), true)
}
@@ -1795,6 +1899,180 @@ mod tests {
}
}
+ // ── `max_tokens` argument (#1058) ─────────────────────────────────────
+
+ #[test]
+ fn the_argument_never_reaches_the_engine() {
+ // The budget is enforced proxy-side on the RESPONSE; a request body
+ // that carried it would be forwarded to a node that never heard of
+ // it (and might 400 as an unknown key). Builders copy only their own
+ // known keys, so `max_tokens` cannot leak — pinned per builder.
+ let with_budget = json!({ "max_tokens": 200 });
+ let (_, _, body) = built(build_search(&with_budget));
+ assert!(body.get("max_tokens").is_none(), "{body}");
+ let (_, _, body) = built(build_semantic(&json!({
+ "index": "kb", "field": "f", "query": "q", "max_tokens": 200
+ })));
+ assert!(body.get("max_tokens").is_none(), "{body}");
+ let (_, _, body) = built(build_vector(&json!({
+ "index": "e", "field": "v", "query_vector": [0.1], "max_tokens": 200
+ })));
+ assert!(body.get("max_tokens").is_none(), "{body}");
+ let (_, _, body) = built(build_hybrid(&json!({
+ "index": "h", "queries": [{ "query": { "match": { "b": "x" } } }], "max_tokens": 200
+ })));
+ assert!(body.get("max_tokens").is_none(), "{body}");
+ }
+
+ #[test]
+ fn exactly_the_five_search_tools_advertise_max_tokens() {
+ let specs = tool_specs();
+ let search_tools = [
+ "xerj_search",
+ "xerj_hybrid_search",
+ "xerj_semantic_search",
+ "xerj_vector_search",
+ "xerj_code_search",
+ ];
+ for tool in specs.as_array().unwrap() {
+ let name = tool["name"].as_str().unwrap();
+ let prop = &tool["inputSchema"]["properties"]["max_tokens"];
+ if search_tools.contains(&name) {
+ assert!(prop.is_object(), "{name} must advertise `max_tokens`");
+ let d = prop["description"].as_str().expect("described");
+ // One NAMED tokenizer, stated in the description, plus the
+ // never-exceed promise and the floor.
+ assert!(
+ d.contains(token_budget::TOKENIZER_NAME),
+ "{name}: the tokenizer must be named: {d}"
+ );
+ assert!(d.contains("NEVER exceeds"), "{name}: {d}");
+ assert!(d.contains("LOCATORS ONLY"), "{name}: {d}");
+ assert!(
+ d.contains(&token_budget::MIN_MAX_TOKENS.to_string()),
+ "{name}: the floor must be stated: {d}"
+ );
+ assert_eq!(
+ prop["minimum"],
+ token_budget::MIN_MAX_TOKENS,
+ "{name}: schema minimum matches the parsed floor"
+ );
+ assert!(
+ !tool["inputSchema"]["required"]
+ .as_array()
+ .map(|r| r.iter().any(|r| r == "max_tokens"))
+ .unwrap_or(false),
+ "{name}: `max_tokens` is optional"
+ );
+ } else {
+ assert!(
+ prop.is_null(),
+ "{name} is not a search tool; no `max_tokens` there (#{name})"
+ );
+ }
+ }
+ }
+
+ /// The dispatch seam: a mistyped or under-floor budget is an `isError`
+ /// tool result BEFORE any request is made (no node needed for this test).
+ #[tokio::test]
+ async fn a_bad_budget_is_refused_at_dispatch() {
+ let ctx = Ctx {
+ client: reqwest::Client::new(),
+ base_url: "http://127.0.0.1:1".to_string(),
+ auth: None,
+ };
+ for tool in [
+ "xerj_search",
+ "xerj_hybrid_search",
+ "xerj_semantic_search",
+ "xerj_vector_search",
+ ] {
+ for bad in [json!("500"), json!(31)] {
+ let msg = json!({
+ "params": { "name": tool, "arguments": {
+ // Minimal valid args per tool, so the ONLY failure is
+ // the budget.
+ "index": "i",
+ "field": "f",
+ "query": "q",
+ "query_vector": [0.1],
+ "queries": [{ "query": { "match": { "b": "x" } } }],
+ "max_tokens": bad
+ } }
+ });
+ let res = call_tool(&ctx, &msg).await;
+ assert_eq!(res["isError"], true, "{tool} {bad}: {res}");
+ let text = res["content"][0]["text"].as_str().unwrap();
+ assert!(text.contains(">= 32"), "{tool} {bad}: {text}");
+ }
+ }
+ // And the code-search surface refuses the same way, naming the tool.
+ let msg = json!({
+ "params": { "name": "xerj_code_search", "arguments": {
+ "corpus": "c", "query": "q", "max_tokens": 31
+ } }
+ });
+ let res = call_tool(&ctx, &msg).await;
+ assert_eq!(res["isError"], true);
+ assert!(res["content"][0]["text"]
+ .as_str()
+ .unwrap()
+ .contains("xerj_code_search"));
+ }
+
+ /// #1058's hard rule, per tool: run a tool-shaped engine response through
+ /// the exact budgeter its dispatch path applies, at a binding budget, and
+ /// the result NEVER exceeds it. (The full budget sweep across many budgets
+ /// lives in token_budget's own tests; this pins which surface each tool
+ /// uses, so a future rewiring that drops one fails here.)
+ #[test]
+ fn every_search_tool_response_path_enforces_the_budget() {
+ let code_corpus_response = serde_json::to_string(&json!({
+ "took": 3,
+ "hits": { "total": { "value": 3, "relation": "eq" }, "max_score": 9.0,
+ "hits": [
+ { "_index": "ax-rust", "_id": "a", "_score": 9.0,
+ "_source": { "ax_path": "repo/a.rs", "line": 10,
+ "body": "pub fn a() { ".to_string().repeat(20) } },
+ { "_index": "ax-rust", "_id": "b", "_score": 8.0,
+ "_source": { "ax_path": "repo/b.rs", "line": 20,
+ "body": "pub fn b() { ".to_string().repeat(20) } },
+ { "_index": "ax-rust", "_id": "c", "_score": 7.0,
+ "_source": { "ax_path": "repo/c.rs", "line": 30,
+ "body": "pub fn c() { ".to_string().repeat(20) } }
+ ] }
+ }))
+ .unwrap();
+ for (tool, engine_text) in [
+ ("xerj_search", code_corpus_response.as_str()),
+ ("xerj_hybrid_search", code_corpus_response.as_str()),
+ ("xerj_semantic_search", code_corpus_response.as_str()),
+ ("xerj_vector_search", code_corpus_response.as_str()),
+ ] {
+ for max_tokens in [48usize, 96, 160] {
+ let out = token_budget::budget_json_response(engine_text, max_tokens);
+ assert!(
+ token_budget::count_tokens(&out) <= max_tokens,
+ "{tool} @ {max_tokens}: used {}",
+ token_budget::count_tokens(&out)
+ );
+ let v: serde_json::Value =
+ serde_json::from_str(&out).expect("still one JSON document");
+ assert!(v["_token_budget"].is_object(), "{tool}: accounting present");
+ }
+ }
+ // The rendered-text surface (xerj_code_search) uses the text budgeter.
+ let rendered = "\n─── repo/a.rs:10 (score 9.00, Apache-2.0)\n [fn a — 300 of 12,000 chars]\nfn a() { long body here }\n─── repo/b.rs:20 (score 8.00)\nfn b() { long body here }\n\n2 passages from 'x'\nCite file:line for anything you rely on.\n";
+ for max_tokens in [48usize, 96, 160] {
+ let out = token_budget::budget_code_text(rendered, max_tokens);
+ assert!(
+ token_budget::count_tokens(&out) <= max_tokens,
+ "xerj_code_search @ {max_tokens}"
+ );
+ }
+ }
+
#[test]
fn memory_store_requires_text_or_vector() {
assert!(build_memory_store(&json!({ "namespace": "n" })).is_err());
diff --git a/engine/crates/xerj-mcp/src/token_budget.rs b/engine/crates/xerj-mcp/src/token_budget.rs
new file mode 100644
index 000000000..cba0c0579
--- /dev/null
+++ b/engine/crates/xerj-mcp/src/token_budget.rs
@@ -0,0 +1,871 @@
+//! Token budgets for MCP search-tool responses (#1058).
+//!
+//! The five search tools (`xerj_search`, `xerj_hybrid_search`,
+//! `xerj_semantic_search`, `xerj_vector_search`, `xerj_code_search`) are thin
+//! proxies: until now they returned whatever the engine sent, so an agent
+//! that asked for 50 hits got all 50 verbatim, context cost unknown and
+//! unbounded. `max_tokens` generalises the char-level cap `xerj_code_search`
+//! already had (its `full` argument — max chars *per passage*) to a token
+//! budget over the **whole response**.
+//!
+//! ## The one tokenizer
+//!
+//! Tokens are counted with **XERJ's `StandardTokenizer`** — UAX #29 word
+//! boundaries, `unicode_words()` — the same word split the engine's own
+//! analyzer uses for BM25 analysis
+//! (`engine/crates/xerj-fts/src/analyzer.rs`, `StandardTokenizer::tokenize`).
+//! This crate calls `unicode-segmentation` directly rather than depending on
+//! the whole FTS stack: `StandardTokenizer` *is* `input.unicode_words()`, so
+//! the count is identical by construction (same crate, same function)
+//! without dragging FST/roaring/zstd into a thin proxy. Honesty, stated in
+//! the tool description too: UAX #29 words approximate model context tokens
+//! and under-count subword splits (`snake_case_name` is ONE word here,
+//! several BPE pieces); the budget is enforced in *these* tokens, the only
+//! count that is deterministic and available offline.
+//!
+//! ## Enforcement model (both surfaces)
+//!
+//! 1. **Dedup before budgeting**: hits that resolve to an identifiable
+//! `file:line` span and *overlap* a higher-ranked hit whose passage is
+//! actually shown are dropped entirely (counted in the accounting block,
+//! never re-shown).
+//! 2. **Greedy keep in rank order** until the budget would break.
+//! 3. **Over-budget hits become locators only** — the citation `file:line` /
+//! `_index`+`_id`, not the passage — while they still fit.
+//! 4. Anything that cannot fit even as a locator is counted as dropped.
+//!
+//! Nothing binding (budget not reached, nothing to dedup) returns the engine
+//! text byte-identical. The invariant the issue demands — *the response
+//! never exceeds the budget* — is guaranteed by construction (a note reserve
+//! is subtracted before any content is admitted) and re-checked as a final
+//! guard; the tests pin it for every tool across a range of budgets. Errors
+//! are NOT budgeted: a refusal the agent cannot read end-to-end is worse
+//! than a long response.
+
+use serde_json::{json, Value};
+use unicode_segmentation::UnicodeSegmentation;
+
+/// How the budget counts tokens, named in tool descriptions so an agent knows
+/// what a "token" is before spending one. Kept as a constant so the five
+/// descriptions cannot drift.
+pub const TOKENIZER_NAME: &str = "XERJ's StandardTokenizer (UAX #29 word boundaries)";
+
+/// The short name for accounting blocks (`standard-uax29-words` is what
+/// `_token_budget.tokenizer` reports).
+pub const TOKENIZER_ID: &str = "standard-uax29-words";
+
+/// Smallest accepted `max_tokens`. Below this not even the response envelope
+/// (header + accounting note) fits, so the tool refuses rather than return a
+/// budget it cannot honor.
+pub const MIN_MAX_TOKENS: usize = 32;
+
+/// Tokens reserved for the `_token_budget` accounting block before any hit is
+/// admitted. The block's fixed fields (keys + counts) measure ~14 word tokens;
+/// locator stubs inside it are charged against the content budget separately,
+/// so 32 leaves margin without padding the response.
+const NOTE_RESERVE: usize = 32;
+
+/// What inserting `"tokens_used":<n>` adds to the serialized JSON, in word
+/// tokens: the key is one word, the number is one. (serde_json's default map
+/// is sorted, so insertion order cannot change the count.) Lets the
+/// accounting report its own final size without a circular measurement.
+const TOKENS_USED_INSERT: usize = 2;
+
+/// Marker appended by the hard truncation fallback.
+const TRUNCATION_MARKER: &str = "\n[truncated at the max_tokens budget]";
+
+// ────────────────────────────── argument ────────────────────────────────────
+
+/// Read the optional `max_tokens` argument. Present-but-mistyped or below the
+/// floor is an error, never a silent drop — the `opt_typed` policy: an agent
+/// that asked for a budget and silently got none would trust a response whose
+/// size it cannot predict.
+pub fn opt_max_tokens(args: &Value) -> Result<Option<usize>, String> {
+ match args.get("max_tokens") {
+ None | Some(Value::Null) => Ok(None),
+ Some(v) => match v.as_u64() {
+ Some(n) if n as usize >= MIN_MAX_TOKENS => Ok(Some(n as usize)),
+ _ => Err(format!(
+ "`max_tokens` must be an integer >= {MIN_MAX_TOKENS} (tokens are counted as \
+ UAX #29 words by XERJ's StandardTokenizer — see the argument's description)"
+ )),
+ },
+ }
+}
+
+// ────────────────────────────── counting ────────────────────────────────────
+
+/// Count tokens the way the budget counts them: UAX #29 words, punctuation
+/// and whitespace dropped — `StandardTokenizer::tokenize` without the Token
+/// structs.
+pub fn count_tokens(text: &str) -> usize {
+ text.unicode_words().count()
+}
+
+/// Hard floor: cut at a word boundary so the result fits `max_tokens`,
+/// marking the cut when there is room for the marker. Every path in this
+/// module funnels through either a budgeted assembly (which reserves
+/// headroom) or this function, so the never-exceed invariant holds even for
+/// shapes the budgeter was not built for (non-JSON bodies,
+/// aggregation-only responses).
+pub fn truncate_to_budget(text: &str, max_tokens: usize) -> String {
+ if count_tokens(text) <= max_tokens {
+ return text.to_string();
+ }
+ let marker_tokens = count_tokens(TRUNCATION_MARKER);
+ let (allowed, with_marker) = if max_tokens > marker_tokens {
+ (max_tokens - marker_tokens, true)
+ } else {
+ (max_tokens, false)
+ };
+ // Byte end of the last word that still fits (same offset trick
+ // StandardTokenizer uses: unicode_words yields subslices of `text`).
+ let mut kept_end = 0usize;
+ for (i, word) in text.unicode_words().enumerate() {
+ if i >= allowed {
+ break;
+ }
+ kept_end = word.as_ptr() as usize + word.len() - text.as_ptr() as usize;
+ }
+ let mut out = text[..kept_end].to_string();
+ if with_marker {
+ out.push_str(TRUNCATION_MARKER);
+ }
+ out
+}
+
+// ───────────────────── JSON surface (the four `_search` tools) ─────────────
+
+/// A hit's `file:line` span, when the hit carries one: `(file, start, end)`
+/// inclusive. Mirrors `xccode::passage::provenance`'s field order (`ax_path`,
+/// `ax_file`, `path`, `file`; `start_line`/`line`), then falls back to the
+/// engine's `_passage` block, whose `ordinal` is the zero-based start line of
+/// the winning passage. `end` is `end_line` when present, else the start plus
+/// the shown passage's own line count — an estimate, and labelled as one
+/// wherever it is described: it exists to catch *overlap*, not to cite.
+fn hit_span(hit: &Value) -> Option<(String, u64, u64)> {
+ let src = hit.get("_source")?;
+ let file = ["ax_path", "ax_file", "path", "file"]
+ .iter()
+ .find_map(|k| src.get(*k).and_then(Value::as_str))?
+ .to_string();
+
+ let start = src
+ .get("start_line")
+ .or_else(|| src.get("line"))
+ .and_then(Value::as_u64)
+ .filter(|n| *n > 0)
+ .or_else(|| {
+ hit.pointer("/fields/_passage/0")?
+ .get("ordinal")
+ .and_then(Value::as_u64)
+ .map(|o| o + 1)
+ })?;
+ let mut end = src
+ .get("end_line")
+ .and_then(Value::as_u64)
+ .unwrap_or(start)
+ .max(start);
+ if let Some(lines) = passage_line_count(hit) {
+ end = end.max(start + lines);
+ }
+ Some((file, start, end))
+}
+
+/// Lines in the hit's shown `_passage` text, when it has one.
+fn passage_line_count(hit: &Value) -> Option<u64> {
+ Some(
+ hit.pointer("/fields/_passage/0")?
+ .get("text")
+ .and_then(Value::as_str)
+ .map(|t| t.matches('\n').count() as u64)
+ .unwrap_or(0),
+ )
+}
+
+fn spans_overlap(a: &(String, u64, u64), b: &(String, u64, u64)) -> bool {
+ a.0 == b.0 && a.1 <= b.2 && b.1 <= a.2
+}
+
+/// The locator-only form of a hit: index/id plus file:line when present.
+/// Deliberately terse — locators are what survives a tight budget, so they
+/// are the cheapest thing we emit.
+fn locator_stub(hit: &Value) -> Value {
+ let mut stub = serde_json::Map::new();
+ for k in ["_index", "_id"] {
+ if let Some(v) = hit.get(k).filter(|v| !v.is_null()) {
+ stub.insert(k.to_string(), v.clone());
+ }
+ }
+ if let Some(src) = hit.get("_source") {
+ for k in ["ax_path", "path", "start_line", "line"] {
+ if let Some(v) = src.get(k).filter(|v| !v.is_null()) {
+ if v.is_string() || v.is_number() {
+ stub.insert(k.to_string(), v.clone());
+ }
+ }
+ }
+ }
+ Value::Object(stub)
+}
+
+/// Budget an engine `_search` JSON response to `max_tokens` tokens. Returns
+/// text (never a JSON error): parses, dedupes overlapping file:line passages
+/// against passages actually shown, keeps hits greedily in rank order,
+/// demotes the rest to locator stubs, and appends a `_token_budget`
+/// accounting block. Nothing binding → the engine text verbatim. Non-JSON
+/// bodies, JSON without a `hits.hits` array (aggregation-only), or an
+/// envelope that alone cannot fit fall back to [`truncate_to_budget`] — the
+/// invariant outranks pretty truncation.
+pub fn budget_json_response(engine_text: &str, max_tokens: usize) -> String {
+ let Ok(resp) = serde_json::from_str::<Value>(engine_text) else {
+ return truncate_to_budget(engine_text, max_tokens);
+ };
+ let Some(hits) = resp
+ .get("hits")
+ .and_then(|h| h.get("hits"))
+ .and_then(Value::as_array)
+ .cloned()
+ else {
+ // Aggregation-only / ack shapes: nothing to demote by hit, so cut at
+ // a word boundary instead.
+ if count_tokens(engine_text) <= max_tokens {
+ return engine_text.to_string();
+ }
+ return truncate_to_budget(engine_text, max_tokens);
+ };
+ if count_tokens(engine_text) <= max_tokens && !any_overlaps(&hits) {
+ return engine_text.to_string();
+ }
+
+ // The envelope with an empty hit list is the unavoidable cost; it is
+ // admitted first, then hits fill what is left.
+ let mut header = resp.clone();
+ if let Some(arr) = header
+ .get_mut("hits")
+ .and_then(|h| h.get_mut("hits"))
+ .and_then(Value::as_array_mut)
+ {
+ arr.clear();
+ }
+ let header_tokens = count_tokens(&compact(&header));
+ if header_tokens.saturating_add(NOTE_RESERVE) >= max_tokens {
+ // Even an empty response cannot fit. Drop the hit list entirely; if
+ // that is still not enough, word-truncate.
+ let mut bare = header;
+ if let Some(hits_obj) = bare.get_mut("hits").and_then(Value::as_object_mut) {
+ hits_obj.remove("hits");
+ }
+ return truncate_to_budget(&compact(&bare), max_tokens);
+ }
+ let mut remaining = max_tokens - header_tokens - NOTE_RESERVE;
+
+ // 1. Dedup: a hit overlapping the span of an already-SHOWN passage is
+ // skipped entirely. Locator-only hits show no passage, so they claim
+ // no span — a later overlap through them is not a duplicate read.
+ let mut kept_spans: Vec<(String, u64, u64)> = Vec::new();
+ let mut deduped = 0usize;
+ let mut full: Vec<Value> = Vec::new();
+ let mut locators: Vec<Value> = Vec::new();
+ let mut dropped = 0usize;
+ for hit in &hits {
+ let span = hit_span(hit);
+ if span
+ .as_ref()
+ .is_some_and(|s| kept_spans.iter().any(|k| spans_overlap(k, s)))
+ {
+ deduped += 1;
+ continue;
+ }
+ let hit_tokens = count_tokens(&compact(hit));
+ if hit_tokens <= remaining {
+ full.push(hit.clone());
+ if let Some(s) = span {
+ kept_spans.push(s);
+ }
+ remaining -= hit_tokens;
+ } else {
+ let stub = locator_stub(hit);
+ let stub_tokens = count_tokens(&compact(&stub));
+ if stub_tokens <= remaining {
+ locators.push(stub);
+ remaining -= stub_tokens;
+ } else {
+ dropped += 1;
+ }
+ }
+ }
+
+ // 2. Assemble, measure once, then report our own final size (the insert
+ // costs a fixed TOKENS_USED_INSERT, covered by NOTE_RESERVE).
+ let mut out = header;
+ if let Some(arr) = out
+ .get_mut("hits")
+ .and_then(|h| h.get_mut("hits"))
+ .and_then(Value::as_array_mut)
+ {
+ *arr = full;
+ }
+ let accounting = json!({
+ "tokenizer": TOKENIZER_ID,
+ "max_tokens": max_tokens,
+ "hits_full": out.pointer("/hits/hits").and_then(Value::as_array).map(|a| a.len()).unwrap_or(0),
+ "hits_locator_only": locators,
+ "deduped_overlapping": deduped,
+ "hits_dropped": dropped,
+ });
+ if let Some(obj) = out.as_object_mut() {
+ obj.insert("_token_budget".to_string(), accounting);
+ }
+ let measured = count_tokens(&compact(&out));
+ if let Some(obj) = out.as_object_mut() {
+ obj.get_mut("_token_budget")
+ .and_then(Value::as_object_mut)
+ .expect("just inserted")
+ .insert(
+ "tokens_used".to_string(),
+ json!(measured + TOKENS_USED_INSERT),
+ );
+ }
+ let final_text = compact(&out);
+ // 3. The guarantee. Reserve accounting should make this unreachable; if
+ // a future edit breaks that, the marker is still honest.
+ if count_tokens(&final_text) > max_tokens {
+ return truncate_to_budget(&final_text, max_tokens);
+ }
+ final_text
+}
+
+/// Would any pair of hits in this list overlap? Used only for the
+/// nothing-binding early exit, where every hit is shown, so pairwise overlap
+/// among all of them is exactly the dedup condition.
+fn any_overlaps(hits: &[Value]) -> bool {
+ let mut spans: Vec<(String, u64, u64)> = Vec::new();
+ for hit in hits {
+ if let Some(s) = hit_span(hit) {
+ if spans.iter().any(|k| spans_overlap(k, &s)) {
+ return true;
+ }
+ spans.push(s);
+ }
+ }
+ false
+}
+
+fn compact(v: &Value) -> String {
+ serde_json::to_string(v).unwrap_or_default()
+}
+
+// ──────────────────── rendered-text surface (`xerj_code_search`) ────────────
+
+/// One rendered hit block from the shared xccode renderer: the `─── ` header
+/// line plus its passage body, bounded by the next header or the footer.
+struct CodeBlock<'a> {
+ lines: &'a [&'a str],
+}
+
+impl<'a> CodeBlock<'a> {
+ fn header(&self) -> &'a str {
+ self.lines[0]
+ }
+
+ /// `(file, start_line)` from the header's locator (`─── path:line (...)`
+ /// — the renderer's provenance format). No line number → `(file, None)`.
+ fn locator(&self) -> (String, Option<u64>) {
+ let h = self.header().strip_prefix("─── ").unwrap_or(self.header());
+ let loc = h.split(" (").next().unwrap_or(h).trim();
+ match loc.rsplit_once(':') {
+ Some((path, digits))
+ if !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()) =>
+ {
+ (path.to_string(), digits.parse().ok())
+ }
+ _ => (loc.to_string(), None),
+ }
+ }
+
+ /// Estimated passage span `[start, start + body lines]` — the body shown
+ /// IS the passage, so its line count bounds it. Coarse on purpose:
+ /// overlap detection, not citation.
+ fn span(&self) -> Option<(String, u64, u64)> {
+ let (file, start) = self.locator();
+ let start = start?;
+ let body_lines = self.lines.len().saturating_sub(1) as u64;
+ Some((file, start, start + body_lines))
+ }
+}
+
+/// Budget the shared renderer's text (`xerj_code_search`'s payload) to
+/// `max_tokens` tokens. Same model as the JSON surface, adapted to the
+/// renderer's shape: blocks start at `─── ` lines; the footer is the
+/// `N passages from '…'` line and everything after it. Deduped blocks are
+/// removed entirely; over-budget blocks keep only their `─── file:line
+/// (score …)` header — the locator the footer tells the agent to cite — with
+/// a visible omission marker; the trailing `[token budget …]` line accounts
+/// for every hit. Nothing to dedup and under budget → byte-identical text.
+pub fn budget_code_text(rendered: &str, max_tokens: usize) -> String {
+ let lines: Vec<&str> = rendered.lines().collect();
+ let Some(first_hit) = lines.iter().position(|l| l.starts_with("─── ")) else {
+ // No hit blocks (no-match text, warnings only) — still never exceed.
+ return truncate_to_budget(rendered, max_tokens);
+ };
+ let footer_start = (first_hit..lines.len())
+ .find(|&i| lines[i].contains(" passages from '"))
+ .unwrap_or(lines.len());
+
+ // Segment into blocks.
+ let mut blocks: Vec<CodeBlock> = Vec::new();
+ let mut i = first_hit;
+ while i < footer_start {
+ if lines[i].starts_with("─── ") {
+ let start = i;
+ i += 1;
+ while i < footer_start && !lines[i].starts_with("─── ") {
+ i += 1;
+ }
+ blocks.push(CodeBlock {
+ lines: &lines[start..i],
+ });
+ } else {
+ i += 1;
+ }
+ }
+
+ let preamble = lines[..first_hit].join("\n");
+ let footer = lines[footer_start..].join("\n");
+ // Nothing binding → byte-identical passthrough (the leading blank line
+ // lives in the preamble, the footer is untouched).
+ let has_overlap = {
+ let mut spans: Vec<(String, u64, u64)> = Vec::new();
+ let mut hit = false;
+ for b in &blocks {
+ if let Some(s) = b.span() {
+ if spans.iter().any(|k| spans_overlap(k, &s)) {
+ hit = true;
+ break;
+ }
+ spans.push(s);
+ }
+ }
+ hit
+ };
+ if !has_overlap && count_tokens(rendered) <= max_tokens {
+ return rendered.to_string();
+ }
+
+ let fixed_tokens = count_tokens(&preamble) + count_tokens(&footer);
+ // The trailing accounting line rides after the footer.
+ if fixed_tokens.saturating_add(NOTE_RESERVE) >= max_tokens {
+ return truncate_to_budget(rendered, max_tokens);
+ }
+ let mut remaining = max_tokens - fixed_tokens - NOTE_RESERVE;
+
+ let mut out_lines: Vec<String> = vec![preamble];
+ let mut deduped = 0usize;
+ let mut locator_only = 0usize;
+ let mut dropped = 0usize;
+ let mut kept_spans: Vec<(String, u64, u64)> = Vec::new();
+ for block in &blocks {
+ let span = block.span();
+ if span
+ .as_ref()
+ .is_some_and(|s| kept_spans.iter().any(|k| spans_overlap(k, s)))
+ {
+ deduped += 1;
+ continue;
+ }
+ let block_text = block.lines.join("\n");
+ let block_tokens = count_tokens(&block_text);
+ if block_tokens <= remaining {
+ out_lines.push(block_text);
+ if let Some(s) = span {
+ kept_spans.push(s);
+ }
+ remaining -= block_tokens;
+ } else {
+ let omission = format!(
+ " [passage omitted: over the max_tokens {max_tokens} budget — locator only]"
+ );
+ let loc_tokens = count_tokens(block.header()) + count_tokens(&omission);
+ if loc_tokens <= remaining {
+ out_lines.push(block.header().to_string());
+ out_lines.push(omission);
+ locator_only += 1;
+ remaining -= loc_tokens;
+ } else {
+ dropped += 1;
+ }
+ }
+ }
+ let shown_full = blocks.len() - deduped - locator_only - dropped;
+ if !footer.is_empty() {
+ out_lines.push(footer);
+ }
+ out_lines.push(format!(
+ "[token budget: {shown_full}/{} passages full, {locator_only} locator-only, \
+ {deduped} deduped (overlapping file:line), {dropped} dropped; counted as \
+ {TOKENIZER_ID}, max_tokens {max_tokens}]",
+ blocks.len(),
+ ));
+ let final_text = format!("{}\n", out_lines.join("\n"));
+ if count_tokens(&final_text) > max_tokens {
+ return truncate_to_budget(&final_text, max_tokens);
+ }
+ final_text
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// The issue's hard rule: the response NEVER exceeds the budget. Run a
+ /// payload through a budgeter at many budgets — including ones that bind
+ /// mid-hit — and pin the invariant each time, not once.
+ fn assert_never_exceeds(text: &str, budgets: &[usize], json_surface: bool) {
+ for &n in budgets {
+ let out = if json_surface {
+ budget_json_response(text, n)
+ } else {
+ budget_code_text(text, n)
+ };
+ assert!(
+ count_tokens(&out) <= n,
+ "budget {n}: response used {} tokens\n---\n{out}\n---",
+ count_tokens(&out)
+ );
+ }
+ }
+
+ fn search_response(hits: Vec<Value>) -> String {
+ compact(&json!({
+ "took": 12,
+ "timed_out": false,
+ "_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
+ "hits": {
+ "total": { "value": hits.len(), "relation": "eq" },
+ "max_score": 12.34,
+ "hits": hits
+ }
+ }))
+ }
+
+ fn code_hit(path: &str, line: u64, body_lines: usize) -> Value {
+ let body: Vec<String> = (0..body_lines)
+ .map(|i| format!(" let x{i} = compute_{i}();"))
+ .collect();
+ json!({
+ "_index": "ax-code",
+ "_id": format!("hit-{path}-{line}"),
+ "_score": 10.0 - line as f64 / 100.0,
+ "_source": {
+ "ax_path": path,
+ "line": line,
+ "language": "rust",
+ "kind": "function",
+ "body": body.join("\n")
+ },
+ "fields": { "_passage": [ { "field": "body", "ordinal": line - 1, "text": body.join("\n") } ] }
+ })
+ }
+
+ // ── the tokenizer ──────────────────────────────────────────────────────
+
+ #[test]
+ fn count_tokens_is_uax29_words() {
+ // Same function StandardTokenizer uses (unicode_words): punctuation
+ // and whitespace dropped, snake_case is one word.
+ assert_eq!(count_tokens("hello, world!"), 2);
+ assert_eq!(count_tokens("snake_case_name"), 1);
+ assert_eq!(count_tokens(""), 0);
+ assert_eq!(count_tokens("max_tokens\": 12345, \"hits_full"), 3);
+ }
+
+ #[test]
+ fn truncate_to_budget_never_exceeds_and_marks_the_cut() {
+ let text = "word ".repeat(500);
+ for n in [1usize, 5, 6, 7, 32, 100] {
+ let out = truncate_to_budget(&text, n);
+ assert!(count_tokens(&out) <= n, "budget {n}: {out}");
+ if n > count_tokens(TRUNCATION_MARKER) {
+ assert!(out.contains("truncated at the max_tokens budget"));
+ }
+ }
+ // Already fitting: byte-identical, no marker.
+ let small = "only a few words";
+ assert_eq!(truncate_to_budget(small, 100), small);
+ }
+
+ // ── argument parsing ───────────────────────────────────────────────────
+
+ #[test]
+ fn opt_max_tokens_validates() {
+ assert_eq!(opt_max_tokens(&json!({})), Ok(None));
+ assert_eq!(opt_max_tokens(&json!({ "max_tokens": null })), Ok(None));
+ assert_eq!(opt_max_tokens(&json!({ "max_tokens": 500 })), Ok(Some(500)));
+ for bad in [
+ json!({ "max_tokens": 0 }),
+ json!({ "max_tokens": 31 }),
+ json!({ "max_tokens": "500" }),
+ json!({ "max_tokens": 4.5 }),
+ json!({ "max_tokens": true }),
+ ] {
+ assert!(opt_max_tokens(&bad).is_err(), "should reject {bad}");
+ let msg = opt_max_tokens(&bad).unwrap_err();
+ assert!(msg.contains(">= 32"), "the floor must be named: {msg}");
+ }
+ }
+
+ // ── JSON surface: per-tool budget enforcement ──────────────────────────
+
+ /// xerj_search shape: definition-first hits over an autoindex corpus,
+ /// `_source` carrying ax_path/line and a `fields._passage` projection.
+ #[test]
+ fn search_budget_holds_and_demotes_to_locators() {
+ let resp = search_response(
+ (1..=12)
+ .map(|i| code_hit(&format!("repo/src/f{i}.rs"), 10 * i, 8))
+ .collect(),
+ );
+ assert_never_exceeds(&resp, &[32, 48, 64, 120, 200, 400, 100_000], true);
+
+ let out = budget_json_response(&resp, 300);
+ let v: Value = serde_json::from_str(&out).expect("still one JSON document");
+ let tb = &v["_token_budget"];
+ assert_eq!(tb["max_tokens"], 300, "the budget is self-describing");
+ // The tight regime: a budget too small for ANY full passage still
+ // returns locators rather than nothing.
+ let tight: Value = serde_json::from_str(&budget_json_response(&resp, 96)).unwrap();
+ assert_eq!(tight["_token_budget"]["hits_full"], 0);
+ assert!(
+ !tight["_token_budget"]["hits_locator_only"]
+ .as_array()
+ .unwrap()
+ .is_empty(),
+ "tight budget → locator-only, not silence: {}",
+ tight["_token_budget"]
+ );
+ assert_eq!(tb["tokenizer"], TOKENIZER_ID);
+ let full = tb["hits_full"].as_u64().unwrap();
+ assert!(
+ full > 0 && full < 12,
+ "binding budget keeps some, not all: {full}"
+ );
+ assert!(
+ !tb["hits_locator_only"].as_array().unwrap().is_empty(),
+ "the first over-budget hits must still be locators, not dropped"
+ );
+ for stub in tb["hits_locator_only"].as_array().unwrap() {
+ assert!(
+ stub.get("_index").is_some() && stub.get("_id").is_some(),
+ "{stub}"
+ );
+ assert!(stub.get("body").is_none(), "a locator carries no passage");
+ }
+ // The kept hits are the TOP of the ranking, in order.
+ let kept: Vec<&str> = v["hits"]["hits"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|h| h["_id"].as_str().unwrap())
+ .collect();
+ assert_eq!(kept.len() as u64, full);
+ let all: Vec<String> = serde_json::from_str::<Value>(&resp).unwrap()["hits"]["hits"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|h| h["_id"].as_str().unwrap().to_string())
+ .collect();
+ assert_eq!(
+ kept,
+ all[..kept.len()]
+ .iter()
+ .map(|s| s.as_str())
+ .collect::<Vec<_>>(),
+ "hits are admitted in rank order"
+ );
+ assert_eq!(
+ v["_token_budget"]["deduped_overlapping"], 0,
+ "distinct files"
+ );
+ }
+
+ /// xerj_semantic_search / xerj_vector_search shape: hits whose `_source`
+ /// is a prose chunk (no file/line), so no span, no dedup — pure budget.
+ #[test]
+ fn prose_budget_holds_without_spans() {
+ let hits: Vec<Value> = (0..10)
+ .map(|i| {
+ json!({
+ "_index": "kb", "_id": format!("doc-{i}"),
+ "_score": 0.9 - i as f64 / 100.0,
+ "_source": { "title": format!("doc {i}"), "text": "answer ".repeat(60) }
+ })
+ })
+ .collect();
+ let resp = search_response(hits);
+ assert_never_exceeds(&resp, &[32, 64, 128, 512, 100_000], true);
+
+ let out = budget_json_response(&resp, 128);
+ let v: Value = serde_json::from_str(&out).unwrap();
+ assert_eq!(
+ v["_token_budget"]["deduped_overlapping"], 0,
+ "no spans → no dedup"
+ );
+ assert!(v["_token_budget"]["hits_full"].as_u64().unwrap() < 10);
+ }
+
+ /// xerj_hybrid_search shape: fused hits from several sub-queries can
+ /// return the SAME passage twice via different legs — the dedup case.
+ #[test]
+ fn overlapping_passages_are_deduped_before_budgeting() {
+ let dup = code_hit("repo/src/same.rs", 100, 6);
+ let mut hits = vec![dup, code_hit("repo/src/other.rs", 55, 6)];
+ // Same file, start line INSIDE the first hit's shown passage.
+ hits.push(json!({
+ "_index": "ax-code", "_id": "hit-overlap", "_score": 9.0,
+ "_source": { "ax_path": "repo/src/same.rs", "start_line": 102, "end_line": 104, "body": "overlap" }
+ }));
+ let resp = search_response(hits);
+
+ // Budget deliberately NOT binding: dedup happens regardless.
+ let out = budget_json_response(&resp, 100_000);
+ let v: Value = serde_json::from_str(&out).unwrap();
+ assert_eq!(v["_token_budget"]["deduped_overlapping"], 1);
+ assert_eq!(
+ v["hits"]["hits"].as_array().unwrap().len(),
+ 2,
+ "the duplicate is not re-shown"
+ );
+ assert_never_exceeds(&resp, &[32, 64, 128, 100_000], true);
+ }
+
+ /// The `_passage`-only shape (no explicit line field): ordinal + text
+ /// line count still yields a span, so the same passage returned twice
+ /// dedupes.
+ #[test]
+ fn passage_ordinal_spans_dedupe_too() {
+ let mk = |id: &str| {
+ json!({
+ "_index": "ax-code", "_id": id, "_score": 5.0,
+ "_source": { "ax_path": "repo/lib.rs" },
+ "fields": { "_passage": [ { "field": "body", "ordinal": 41,
+ "text": "line one\nline two\nline three" } ] }
+ })
+ };
+ let resp = search_response(vec![mk("a"), mk("b")]);
+ let out = budget_json_response(&resp, 100_000);
+ let v: Value = serde_json::from_str(&out).unwrap();
+ assert_eq!(
+ v["_token_budget"]["deduped_overlapping"], 1,
+ "same ordinal = same passage"
+ );
+ }
+
+ #[test]
+ fn non_json_and_hitless_responses_still_respect_the_budget() {
+ let out = budget_json_response("not json at all, just a long stream of words ", 8);
+ assert!(count_tokens(&out) <= 8, "{out}");
+ assert!(out.contains("truncated at the max_tokens"));
+
+ let agg = compact(&json!({
+ "took": 1,
+ "aggregations": { "by_status": { "buckets": [
+ { "key": "open", "doc_count": 1 }, { "key": "closed", "doc_count": 2 }
+ ] } }
+ }));
+ let out = budget_json_response(&agg, 8);
+ assert!(count_tokens(&out) <= 8, "{out}");
+ // And a hitless response that fits passes through byte-identical.
+ assert_eq!(budget_json_response(&agg, 10_000), agg);
+ }
+
+ #[test]
+ fn a_binding_response_that_already_fits_is_verbatim() {
+ let resp = search_response(vec![code_hit("a.rs", 3, 2)]);
+ assert_eq!(budget_json_response(&resp, 100_000), resp);
+ }
+
+ #[test]
+ fn a_budget_at_the_floor_still_never_exceeds() {
+ // MIN_MAX_TOKENS exists so the envelope always fits; the inner guard
+ // is pinned anyway — a forced-tiny budget truncates, never overflows.
+ let resp = search_response(vec![code_hit("a.rs", 1, 4)]);
+ let out = budget_json_response(&resp, 32);
+ assert!(count_tokens(&out) <= 32, "{out}");
+ }
+
+ // ── rendered-text surface (xerj_code_search) ───────────────────────────
+
+ fn rendered_code_search() -> String {
+ let mut text = String::new();
+ for (path, line) in [
+ ("tantivy/src/reader.rs", 30u64),
+ ("quickwit/src/search.rs", 210),
+ // Same file, start line inside the first block's shown passage
+ // (span 30..35: header + label + 4 code lines).
+ ("tantivy/src/reader.rs", 33),
+ ("qdrant/src/hnsw.rs", 500),
+ ] {
+ text.push_str(&format!(
+ "\n─── {path}:{line} (score 9.{line}, Apache-2.0)\n"
+ ));
+ text.push_str(" [method run — 300 of 12,000 chars]\n");
+ for i in 0..4 {
+ text.push_str(&format!(
+ "pub fn run_{i}(query: &str) -> Result<{{}}> {{ warmup_{i}(); }}\n"
+ ));
+ }
+ }
+ text.push_str("\n4 passages from 'peer-engines' (index 2d old)\n");
+ text.push_str("Cite file:line for anything you rely on.\n");
+ text
+ }
+
+ #[test]
+ fn code_text_budget_holds_and_keeps_locators() {
+ let rendered = rendered_code_search();
+ assert_never_exceeds(&rendered, &[32, 64, 100, 150, 200, 400, 100_000], false);
+
+ let out = budget_code_text(&rendered, 120);
+ assert!(
+ out.contains("─── tantivy/src/reader.rs:30"),
+ "the first locator stays"
+ );
+ assert!(out.contains("passage omitted: over the max_tokens 120 budget"));
+ assert!(out.contains("[token budget:"));
+ assert!(
+ out.contains("Cite file:line for anything you rely on."),
+ "footer survives"
+ );
+ }
+
+ #[test]
+ fn code_text_overlapping_passages_are_deduped() {
+ let rendered = rendered_code_search();
+ // reader.rs:33 falls inside reader.rs:30's shown passage → deduped,
+ // even with no budget pressure at all.
+ let out = budget_code_text(&rendered, 100_000);
+ assert!(
+ !out.contains("─── tantivy/src/reader.rs:33"),
+ "the overlap is gone"
+ );
+ assert!(out.contains("1 deduped (overlapping file:line)"), "{out}");
+ assert_never_exceeds(&rendered, &[32, 64, 100, 100_000], false);
+ }
+
+ #[test]
+ fn code_text_with_nothing_to_do_is_byte_identical() {
+ let no_dups =
+ rendered_code_search().replace("tantivy/src/reader.rs:33", "valkey/src/listpack.c:88");
+ let out = budget_code_text(&no_dups, 100_000);
+ assert_eq!(out, no_dups, "no overlap, under budget → untouched");
+ }
+
+ #[test]
+ fn code_text_without_blocks_still_respects_the_budget() {
+ let no_match =
+ "No passage in 'kv' matches: two-way substring\nThe corpus is likely wrong.\n";
+ let out = budget_code_text(no_match, 8);
+ assert!(count_tokens(&out) <= 8, "{out}");
+ }
+}
diff --git a/landing/docs/agents/schemas/mcp-tools.json b/landing/docs/agents/schemas/mcp-tools.json
index 016cdf6de..018a18a87 100644
--- a/landing/docs/agents/schemas/mcp-tools.json
+++ b/landing/docs/agents/schemas/mcp-tools.json
@@ -2,7 +2,7 @@
"tools": [
{
"name": "xerj_search",
- "description": "Full-text / keyword / structured search over a XERJ index. Proxies POST /{index}/_search. Pass `query` as a PLAIN STRING for the definition-first code search (ranks the file that defines a symbol above files that merely mention it — the right default for code), or as an ES query-DSL object (e.g. {\"term\":{\"status\":\"open\"}}, or a bool clause) for structured queries. Omit `query` for match_all. Optional `rerank` adds a second stage that re-judges the top hits with an external relevance model — read that argument's description before using it: it needs a provider the operator configured, and it sends document text off the machine.",
+ "description": "Full-text / keyword / structured search over a XERJ index. Proxies POST /{index}/_search. Pass `query` as a PLAIN STRING for the definition-first code search (ranks the file that defines a symbol above files that merely mention it — the right default for code), or as an ES query-DSL object (e.g. {\"term\":{\"status\":\"open\"}}, or a bool clause) for structured queries. Omit `query` for match_all. Optional `rerank` adds a second stage that re-judges the top hits with an external relevance model — read that argument's description before using it: it needs a provider the operator configured, and it sends document text off the machine. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -37,13 +37,18 @@
"boolean"
],
"description": "Optional second-stage reranking. XERJ sends the text of the top `window` hits plus the question to an external relevance model, which returns a calibrated probability (0..1) per document. Hits are reordered by it, that probability REPLACES `_score`, and `min_score` prunes by it — an absolute relevance cut, which a BM25 score cannot give you. WORTH IT WHEN: a natural-language question over prose where the right document is probably in the top 30 but not at the top; or you need a yes/no relevance threshold rather than a relative rank. NOT WORTH IT FOR: exact identifier or symbol lookups (the first stage already ranks the definition first), filters, sorted or aggregation-only requests — it adds a paid network round-trip per search. NEEDS A CONFIGURED PROVIDER: the node's operator must have set a provider key (`[rerank] api_key` or TYPESAFE_API_KEY). You cannot supply one in this call. Without it the search fails with HTTP 503 `rerank_exception`; do not retry, repeat the search without `rerank`. HTTP 403 means the operator disabled it. PRIVACY: this is the only search-time feature that sends document text off the node. Proxy embeddings (`[embedding] default_endpoint`) and the WAL tap also send text off the node when an operator configures them, so never tell a user that nothing else leaves the machine — do not use rerank on data that must stay local unless the user has agreed. Only fields the response returns are sent, so `_source` filtering also limits what leaves. Pass true or {} for defaults, or an object with: `query` (string): the natural-language question to judge documents against. Filled in for you when the tool's `query` is a plain string (that mode also returns the matching passage, which leads the text the judge reads); REQUIRED when the tool's `query` is a `bool`, a `knn`, or anything else with no single question in it (the node refuses rather than guess). `window` (int, default 30, max 300): how many top hits are judged — every one is a paid judgement; `from`+`size` must fit inside it. `min_score` (0..1): drop hits judged below this probability. `fields` (string[]): which returned fields to send; default is every returned string field. `instructions` (string): override what \"relevant\" means for this corpus. `timeout_ms` (int, default 10000). `model` (string). Cannot be combined with `sort`. READ THE RESPONSE'S `_rerank` BLOCK: `applied: true` means the order is the judge's and every `_score` is a probability or null (a hit with no verdict, counted in `unjudged`, sorts last); `applied: false` means the provider missed the deadline or answered nothing usable, and you are looking at the engine's own order with engine scores, stated in `reason`."
+ },
+ "max_tokens": {
+ "type": "integer",
+ "minimum": 32,
+ "description": "Token budget for this response: the returned text NEVER exceeds it. Tokens are counted with XERJ's StandardTokenizer (UAX #29 word boundaries) — the same word split the engine's own analyzer uses for BM25. Honest scope: that under-counts model-context subwords (snake_case is ONE token here), so treat it as a deterministic proxy, not a BPE count. The budget covers the whole response text (JSON keys and syntax included). Overlapping file:line passages are deduped first (higher-ranked wins). Hits that no longer fit are listed as LOCATORS ONLY (file:line, index/id) rather than dropped, and the response's `_token_budget` block accounts for every hit: kept, locator-only, deduped, dropped. Errors are never truncated — a refusal you cannot read end-to-end is worse than a long response. Minimum 32."
}
}
}
},
{
"name": "xerj_semantic_search",
- "description": "Meaning-based search over a `semantic_text` field. The query text is embedded SERVER-SIDE by XERJ's built-in lexical embedder (no external API key), then matched by vector similarity. Proxies POST /{index}/_search with {\"query\":{\"semantic\":{...}}}.",
+ "description": "Meaning-based search over a `semantic_text` field. The query text is embedded SERVER-SIDE by XERJ's built-in lexical embedder (no external API key), then matched by vector similarity. Proxies POST /{index}/_search with {\"query\":{\"semantic\":{...}}}. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -65,6 +70,11 @@
"filter": {
"type": "object",
"description": "Optional ES query clause applied as a pre-filter."
+ },
+ "max_tokens": {
+ "type": "integer",
+ "minimum": 32,
+ "description": "Token budget for this response: the returned text NEVER exceeds it. Tokens are counted with XERJ's StandardTokenizer (UAX #29 word boundaries) — the same word split the engine's own analyzer uses for BM25. Honest scope: that under-counts model-context subwords (snake_case is ONE token here), so treat it as a deterministic proxy, not a BPE count. The budget covers the whole response text (JSON keys and syntax included). Overlapping file:line passages are deduped first (higher-ranked wins). Hits that no longer fit are listed as LOCATORS ONLY (file:line, index/id) rather than dropped, and the response's `_token_budget` block accounts for every hit: kept, locator-only, deduped, dropped. Errors are never truncated — a refusal you cannot read end-to-end is worse than a long response. Minimum 32."
}
},
"required": [
@@ -76,7 +86,7 @@
},
{
"name": "xerj_vector_search",
- "description": "K-nearest-neighbour search over a `dense_vector` field, given a caller-supplied query vector. NOTE: unfiltered kNN is HNSW-served (approximate) with exact rescoring — measured recall@10 1.00 on the official bench query; num_candidates sets the beam width (floored at 800). Filtered kNN, non-cosine metrics, SQ8 fields, and small indexes run an exact brute-force scan. Proxies POST /{index}/_search with a top-level {\"knn\":{...}}.",
+ "description": "K-nearest-neighbour search over a `dense_vector` field, given a caller-supplied query vector. NOTE: unfiltered kNN is HNSW-served (approximate) with exact rescoring — measured recall@10 1.00 on the official bench query; num_candidates sets the beam width (floored at 800). Filtered kNN, non-cosine metrics, SQ8 fields, and small indexes run an exact brute-force scan. Proxies POST /{index}/_search with a top-level {\"knn\":{...}}. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -105,6 +115,11 @@
"filter": {
"type": "object",
"description": "Optional ES query clause applied as a pre-filter."
+ },
+ "max_tokens": {
+ "type": "integer",
+ "minimum": 32,
+ "description": "Token budget for this response: the returned text NEVER exceeds it. Tokens are counted with XERJ's StandardTokenizer (UAX #29 word boundaries) — the same word split the engine's own analyzer uses for BM25. Honest scope: that under-counts model-context subwords (snake_case is ONE token here), so treat it as a deterministic proxy, not a BPE count. The budget covers the whole response text (JSON keys and syntax included). Overlapping file:line passages are deduped first (higher-ranked wins). Hits that no longer fit are listed as LOCATORS ONLY (file:line, index/id) rather than dropped, and the response's `_token_budget` block accounts for every hit: kept, locator-only, deduped, dropped. Errors are never truncated — a refusal you cannot read end-to-end is worse than a long response. Minimum 32."
}
},
"required": [
@@ -116,7 +131,7 @@
},
{
"name": "xerj_hybrid_search",
- "description": "Hybrid search: fuse several sub-queries (e.g. a lexical `match` plus a vector `knn`) into one ranked list. Fusion is `rrf` (reciprocal-rank) or `linear` (weighted). Proxies POST /{index}/_search with {\"query\":{\"hybrid\":{\"queries\":[...],\"fusion\":...}}}. Optional `rerank` re-judges the fused top hits with an external relevance model; here `rerank.query` is REQUIRED, because a hybrid search has several sub-queries and no single question to read.",
+ "description": "Hybrid search: fuse several sub-queries (e.g. a lexical `match` plus a vector `knn`) into one ranked list. Fusion is `rrf` (reciprocal-rank) or `linear` (weighted). Proxies POST /{index}/_search with {\"query\":{\"hybrid\":{\"queries\":[...],\"fusion\":...}}}. Optional `rerank` re-judges the fused top hits with an external relevance model; here `rerank.query` is REQUIRED, because a hybrid search has several sub-queries and no single question to read. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -159,6 +174,11 @@
"boolean"
],
"description": "Optional second-stage reranking. XERJ sends the text of the top `window` hits plus the question to an external relevance model, which returns a calibrated probability (0..1) per document. Hits are reordered by it, that probability REPLACES `_score`, and `min_score` prunes by it — an absolute relevance cut, which a BM25 score cannot give you. WORTH IT WHEN: a natural-language question over prose where the right document is probably in the top 30 but not at the top; or you need a yes/no relevance threshold rather than a relative rank. NOT WORTH IT FOR: exact identifier or symbol lookups (the first stage already ranks the definition first), filters, sorted or aggregation-only requests — it adds a paid network round-trip per search. NEEDS A CONFIGURED PROVIDER: the node's operator must have set a provider key (`[rerank] api_key` or TYPESAFE_API_KEY). You cannot supply one in this call. Without it the search fails with HTTP 503 `rerank_exception`; do not retry, repeat the search without `rerank`. HTTP 403 means the operator disabled it. PRIVACY: this is the only search-time feature that sends document text off the node. Proxy embeddings (`[embedding] default_endpoint`) and the WAL tap also send text off the node when an operator configures them, so never tell a user that nothing else leaves the machine — do not use rerank on data that must stay local unless the user has agreed. Only fields the response returns are sent, so `_source` filtering also limits what leaves. Pass true or {} for defaults, or an object with: `query` (string, REQUIRED for this tool): the natural-language question to judge documents against. `window` (int, default 30, max 300): how many top hits are judged — every one is a paid judgement; `from`+`size` must fit inside it. `min_score` (0..1): drop hits judged below this probability. `fields` (string[]): which returned fields to send; default is every returned string field. `instructions` (string): override what \"relevant\" means for this corpus. `timeout_ms` (int, default 10000). `model` (string). Cannot be combined with `sort`. READ THE RESPONSE'S `_rerank` BLOCK: `applied: true` means the order is the judge's and every `_score` is a probability or null (a hit with no verdict, counted in `unjudged`, sorts last); `applied: false` means the provider missed the deadline or answered nothing usable, and you are looking at the engine's own order with engine scores, stated in `reason`."
+ },
+ "max_tokens": {
+ "type": "integer",
+ "minimum": 32,
+ "description": "Token budget for this response: the returned text NEVER exceeds it. Tokens are counted with XERJ's StandardTokenizer (UAX #29 word boundaries) — the same word split the engine's own analyzer uses for BM25. Honest scope: that under-counts model-context subwords (snake_case is ONE token here), so treat it as a deterministic proxy, not a BPE count. The budget covers the whole response text (JSON keys and syntax included). Overlapping file:line passages are deduped first (higher-ranked wins). Hits that no longer fit are listed as LOCATORS ONLY (file:line, index/id) rather than dropped, and the response's `_token_budget` block accounts for every hit: kept, locator-only, deduped, dropped. Errors are never truncated — a refusal you cannot read end-to-end is worse than a long response. Minimum 32."
}
},
"required": [
@@ -472,7 +492,7 @@
},
{
"name": "xerj_code_search",
- "description": "Reference-coding retrieval over a corpus of peer projects: how did real codebases solve this problem? Returns the matching definition (not a byte window), the file:line to cite, and per-hit licence warnings — AGPL/SSPL/Elastic/BUSL/GPL/LGPL/MPL sources are approach-only. Refuses indices older than 30 days (override with stale_ok). A no-match is NOT an error: it says the corpus is wrong for the task. Corpora: `xerj corpus list`.",
+ "description": "Reference-coding retrieval over a corpus of peer projects: how did real codebases solve this problem? Returns the matching definition (not a byte window), the file:line to cite, and per-hit licence warnings — AGPL/SSPL/Elastic/BUSL/GPL/LGPL/MPL sources are approach-only. Refuses indices older than 30 days (override with stale_ok). A no-match is NOT an error: it says the corpus is wrong for the task. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule). Corpora: `xerj corpus list`.",
"inputSchema": {
"type": "object",
"properties": {
@@ -520,6 +540,11 @@
"strict"
],
"description": "strict strips passage text from restricted-licence hits, keeping locator + warning (default warn)."
+ },
+ "max_tokens": {
+ "type": "integer",
+ "minimum": 32,
+ "description": "Token budget for this response: the returned text NEVER exceeds it. Tokens are counted with XERJ's StandardTokenizer (UAX #29 word boundaries) — the same word split the engine's own analyzer uses for BM25. Honest scope: that under-counts model-context subwords (snake_case is ONE token here), so treat it as a deterministic proxy, not a BPE count. The budget covers the whole response text (JSON keys and syntax included). Overlapping file:line passages are deduped first (higher-ranked wins). Hits that no longer fit are listed as LOCATORS ONLY (file:line, index/id) rather than dropped, and the response's `_token_budget` block accounts for every hit: kept, locator-only, deduped, dropped. Errors are never truncated — a refusal you cannot read end-to-end is worse than a long response. Minimum 32."
}
},
"required": [