Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(mcp): preserve distinct records in JSON token budgets (#2255)

Jarvisa888f4dd78e4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 396ddb42e..9b5978cb2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
instead of refusing. `XERJ_SKIP_CPU_CHECK` is gone
([#2161](https://github.com/xerj-org/xerj/issues/2161)).
+- MCP JSON search responses with `max_tokens` no longer deduplicate rows
+ with distinct `ax_locator` values or hits from different indices merely
+ because their relative paths and passage line numbers match. Deduplication
+ includes the available source index, file identity and record locator;
+ overlapping file and code-symbol passages in the same source still
+ deduplicate. Plain-string `xerj_search` requests the
+ provenance fields needed for this distinction by default.
+
- `xerj corpus index` killed mid-index resumes that build's journal instead of
starting a legacy index under a new prefix. When the journal still has a
generation in progress, indexing continues at the uncommitted operations and
diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rs
index 387f19a82..ef86fdef2 100644
--- a/engine/crates/xerj-mcp/src/lib.rs
+++ b/engine/crates/xerj-mcp/src/lib.rs
@@ -2389,6 +2389,8 @@ fn build_search_with_mapping(
body.entry("_source".to_string()).or_insert_with(|| {
json!([
"ax_path",
+ "ax_file",
+ "ax_locator",
"path",
"line",
"start_line",
diff --git a/engine/crates/xerj-mcp/src/token_budget.rs b/engine/crates/xerj-mcp/src/token_budget.rs
index 662ff633a..4d481b742 100644
--- a/engine/crates/xerj-mcp/src/token_budget.rs
+++ b/engine/crates/xerj-mcp/src/token_budget.rs
@@ -136,19 +136,48 @@ pub fn truncate_to_budget(text: &str, max_tokens: usize) -> String {
// ───────────────────── JSON surface (the four `_search` tools) ─────────────
-/// A hit's `file:line` span, when the hit carries one: `(file, start, end)`
-/// inclusive. Mirrors `xccode::passage::provenance`'s field order (`ax_path`,
-/// `ax_file`, `path`, `file`; `start_line`/`line`), then falls back to the
-/// engine's `_passage` block, whose `ordinal` is the zero-based start line of
-/// the winning passage. `end` is `end_line` when present, else the start plus
+/// A relative path is meaningful only within its source index. Content IDs
+/// distinguish versions at the same path; record locators distinguish rows
+/// whose passage ordinals are local to each record, not the containing file.
+#[derive(PartialEq)]
+struct HitSource {
+ index: Option<String>,
+ file: String,
+ content_id: Option<String>,
+ record: Option<String>,
+}
+
+type HitSpan = (HitSource, u64, u64);
+
+/// A hit's inclusive line span within one source. Mirrors the provenance
+/// path order (`ax_path`, `ax_file`, `path`, `file`; `start_line`/`line`), then
+/// falls back to the engine's `_passage` block. Its `ordinal` is relative to
+/// the record, so those spans are compared only within the same locator.
+/// `end` is `end_line` when present, else the start plus
/// the shown passage's own line count — an estimate, and labelled as one
/// wherever it is described: it exists to catch *overlap*, not to cite.
-fn hit_span(hit: &Value) -> Option<(String, u64, u64)> {
+fn hit_span(hit: &Value) -> Option<HitSpan> {
let src = hit.get("_source")?;
let file = ["ax_path", "ax_file", "path", "file"]
.iter()
.find_map(|k| src.get(*k).and_then(Value::as_str))?
.to_string();
+ let text = |v: Option<&Value>| v.and_then(Value::as_str).map(str::to_owned);
+ let record = src
+ .get("ax_locator")
+ .and_then(Value::as_str)
+ // Whole-file code and its symbol records share file-global lines.
+ // Other locators (rows, pages, sections, ...) are independent spaces.
+ .filter(|loc| {
+ !loc.is_empty() && *loc != "file" && *loc != "code" && !loc.starts_with("code:")
+ })
+ .map(str::to_owned);
+ let source = HitSource {
+ index: text(hit.get("_index")),
+ file,
+ content_id: text(src.get("ax_file")),
+ record,
+ };
let start = src
.get("start_line")
@@ -169,7 +198,7 @@ fn hit_span(hit: &Value) -> Option<(String, u64, u64)> {
if let Some(lines) = passage_line_count(hit) {
end = end.max(start + lines);
}
- Some((file, start, end))
+ Some((source, start, end))
}
/// Lines in the hit's shown `_passage` text, when it has one.
@@ -183,7 +212,7 @@ fn passage_line_count(hit: &Value) -> Option<u64> {
)
}
-fn spans_overlap(a: &(String, u64, u64), b: &(String, u64, u64)) -> bool {
+fn spans_overlap<T: PartialEq>(a: &(T, u64, u64), b: &(T, u64, u64)) -> bool {
a.0 == b.0 && a.1 <= b.2 && b.1 <= a.2
}
@@ -198,7 +227,14 @@ fn locator_stub(hit: &Value) -> Value {
}
}
if let Some(src) = hit.get("_source") {
- for k in ["ax_path", "path", "start_line", "line"] {
+ for k in [
+ "ax_path",
+ "ax_file",
+ "ax_locator",
+ "path",
+ "start_line",
+ "line",
+ ] {
if let Some(v) = src.get(k).filter(|v| !v.is_null()) {
if v.is_string() || v.is_number() {
stub.insert(k.to_string(), v.clone());
@@ -271,7 +307,7 @@ pub fn budget_json_response(engine_text: &str, max_tokens: usize) -> String {
// 1. Dedup: a hit overlapping the span of an already-SHOWN passage is
// skipped entirely. Locator-only hits show no passage, so they claim
// no span — a later overlap through them is not a duplicate read.
- let mut kept_spans: Vec<(String, u64, u64)> = Vec::new();
+ let mut kept_spans: Vec<HitSpan> = Vec::new();
let mut deduped = 0usize;
let mut full: Vec<Value> = Vec::new();
let mut locators: Vec<Value> = Vec::new();
@@ -348,7 +384,7 @@ pub fn budget_json_response(engine_text: &str, max_tokens: usize) -> String {
/// nothing-binding early exit, where every hit is shown, so pairwise overlap
/// among all of them is exactly the dedup condition.
fn any_overlaps(hits: &[Value]) -> bool {
- let mut spans: Vec<(String, u64, u64)> = Vec::new();
+ let mut spans: Vec<HitSpan> = Vec::new();
for hit in hits {
if let Some(s) = hit_span(hit) {
if spans.iter().any(|k| spans_overlap(k, &s)) {
@@ -814,6 +850,69 @@ mod tests {
);
}
+ #[test]
+ fn json_budget_keeps_distinct_record_locators() {
+ for (first, second) in [
+ ("b0", "b100"),
+ ("r1", "r2"),
+ ("Sheet1!r1", "Sheet1!r2"),
+ ("items:e0", "items:e1"),
+ ("d0", "d1"),
+ ] {
+ let row = |id: &str, locator: &str| {
+ json!({
+ "_index": "ax-data", "_id": id,
+ "_source": {"ax_path": "records.jsonl", "ax_file": "file-hash", "ax_locator": locator},
+ "fields": {"_passage": [{"field": "body", "ordinal": 0, "text": "independent record"}]}
+ })
+ };
+ let resp = search_response(vec![row("1", first), row("2", second)]);
+ assert_eq!(
+ budget_json_response(&resp, 10_000),
+ resp,
+ "{first}, {second}"
+ );
+ assert_never_exceeds(&resp, &[32, 64, 128, 256, 10_000], true);
+ let duplicate = search_response(vec![row("1", first), row("2", first)]);
+ let out: Value =
+ serde_json::from_str(&budget_json_response(&duplicate, 10_000)).unwrap();
+ assert_eq!(out["_token_budget"]["deduped_overlapping"], 1);
+ }
+ }
+
+ #[test]
+ fn json_budget_scopes_relative_paths_to_their_source() {
+ let first = code_hit("src/lib.rs", 10, 4);
+ for field in ["_index", "ax_file"] {
+ let mut a = first.clone();
+ let mut b = first.clone();
+ b["_id"] = json!("second");
+ if field == "_index" {
+ b[field] = json!("other-code");
+ } else {
+ a["_source"][field] = json!("content-a");
+ b["_source"][field] = json!("content-b");
+ }
+ let resp = search_response(vec![a, b]);
+ assert_eq!(budget_json_response(&resp, 10_000), resp, "{field}");
+ assert_never_exceeds(&resp, &[32, 64, 128, 256, 10_000], true);
+ }
+ }
+
+ #[test]
+ fn json_budget_dedupes_file_and_symbol_records_in_the_same_line_space() {
+ let mut file = code_hit("src/lib.rs", 10, 4);
+ file["_source"]["ax_file"] = json!("content-a");
+ file["_source"]["ax_locator"] = json!("code");
+ let mut symbol = file.clone();
+ symbol["_id"] = json!("symbol");
+ symbol["_source"]["ax_locator"] = json!("code:10:example");
+ let resp = search_response(vec![file, symbol]);
+ let out: Value = serde_json::from_str(&budget_json_response(&resp, 10_000)).unwrap();
+ assert_eq!(out["_token_budget"]["deduped_overlapping"], 1);
+ assert_eq!(out["hits"]["hits"].as_array().unwrap().len(), 1);
+ }
+
#[test]
fn non_json_and_hitless_responses_still_respect_the_budget() {
let out = budget_json_response("not json at all, just a long stream of words ", 8);
diff --git a/engine/crates/xerj-mcp/tests/token_budget_identity.rs b/engine/crates/xerj-mcp/tests/token_budget_identity.rs
new file mode 100644
index 000000000..786b46ec4
--- /dev/null
+++ b/engine/crates/xerj-mcp/tests/token_budget_identity.rs
@@ -0,0 +1,120 @@
+//! Budgeting must not collapse independent records that share a relative path.
+use std::sync::{Arc, Mutex};
+
+use axum::{routing::get, Json, Router};
+use serde_json::{json, Value};
+
+#[tokio::test]
+async fn json_budget_identity_survives_http_dispatch_and_source_projection() {
+ for different_indices in [false, true] {
+ let hits: Vec<Value> = (0..2)
+ .map(|i| {
+ json!({
+ "_index": if different_indices { format!("repo-{i}-code") } else { "ax-data".into() },
+ "_id": i.to_string(),
+ "_source": {
+ "ax_path": "shared.txt", "ax_file": "content-hash",
+ "ax_locator": if different_indices { "file".into() } else { format!("b{}", i * 100) }
+ },
+ "fields": {"_passage": [{"field":"body", "ordinal":0, "text":"independent evidence"}]}
+ })
+ })
+ .collect();
+ let requests = Arc::new(Mutex::new(Vec::<Value>::new()));
+ let seen = requests.clone();
+ let app = Router::new()
+ .route(
+ "/:index/_mapping",
+ get(|| async {
+ Json(json!({"ax-data":{"mappings":{"properties":{"body":{"type":"text"}}}}}))
+ }),
+ )
+ .route(
+ "/:index/_search",
+ axum::routing::post(move |Json(body): Json<Value>| {
+ let mut hits = hits.clone();
+ seen.lock().unwrap().push(body.clone());
+ async move {
+ // Model the engine's projection: the plain-query default
+ // must actually request identity fields, not just consume
+ // fixtures that always return them.
+ if let Some(fields) = body.get("_source").and_then(Value::as_array) {
+ for hit in &mut hits {
+ hit["_source"].as_object_mut().unwrap().retain(|key, _| {
+ fields.iter().any(|v| v.as_str() == Some(key.as_str()))
+ });
+ }
+ }
+ Json(json!({"hits":{"total":{"value":2,"relation":"eq"},"hits":hits}}))
+ }
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ for (name, arguments) in [
+ ("xerj_search", json!({"index":"ax-*","query":"independent"})),
+ (
+ "xerj_search",
+ json!({"index":"ax-*","query":{"match_all":{}}}),
+ ),
+ (
+ "xerj_semantic_search",
+ json!({"index":"ax-*","field":"body","query":"independent"}),
+ ),
+ (
+ "xerj_vector_search",
+ json!({"index":"ax-*","field":"embedding","query_vector":[0.1]}),
+ ),
+ (
+ "xerj_hybrid_search",
+ json!({"index":"ax-*","queries":[{"query":{"match_all":{}}}]}),
+ ),
+ ] {
+ let mut baseline = None;
+ for budget in [None, Some(10_000)] {
+ let mut args = arguments.clone();
+ if let Some(budget) = budget {
+ args["max_tokens"] = json!(budget);
+ }
+ let response = xerj_mcp::handle_message_for_test(
+ &url,
+ None,
+ json!({
+ "jsonrpc":"2.0", "id":1, "method":"tools/call",
+ "params":{"name":name,"arguments":args}
+ }),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response["result"]["isError"], false, "{response}");
+ let text = response["result"]["content"][0]["text"].as_str().unwrap();
+ let body: Value = serde_json::from_str(text).unwrap();
+ assert_eq!(
+ body["hits"]["hits"].as_array().unwrap().len(),
+ 2,
+ "{name}, different_indices={different_indices}, budget={budget:?}: {body}"
+ );
+ if let Some(baseline) = &baseline {
+ assert_eq!(
+ text, baseline,
+ "nonbinding budget preserves the exact response"
+ );
+ } else {
+ baseline = Some(text.to_string());
+ }
+ }
+ }
+ {
+ let requests = requests.lock().unwrap();
+ for pair in requests.chunks_exact(2) {
+ assert_eq!(
+ pair[0], pair[1],
+ "the budget must not change the engine request"
+ );
+ }
+ }
+ server.abort();
+ let _ = server.await;
+ }
+}