Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(xccode): cap one record per source index and demote text to a recall leg (#1238)

xerj-org25ad6206943f
diff --git a/engine/crates/xerj-common/src/xccode/fields.rs b/engine/crates/xerj-common/src/xccode/fields.rs
index b236757da..9059b1a94 100644
--- a/engine/crates/xerj-common/src/xccode/fields.rs
+++ b/engine/crates/xerj-common/src/xccode/fields.rs
@@ -71,8 +71,16 @@ pub fn resolve_fields(mapping: Option<&Value>) -> Vec<String> {
// above never grows a field an index may not map (the MULTIMATCH_DEFECT
// silent-zero). Found via the zalando G7 1/5: all 25 renamed `.txt`
// chapters were invisible to `xerj code` while README.md matched.
+ //
+ // #1238: `text` joins BELOW 1.0, the same recall-leg posture as
+ // `defs_expanded^0.5`. Measured on the live exploit group (5,644
+ // indices, query "MCPJam inspector 23744", 36 needle PoC repos): at
+ // full weight one plain-text sibling-CVE demo index outscored every
+ // code-family hit and took the whole top-10; at ^0.5 the family stays
+ // searchable (same 39 total hits) while the needle docs keep their
+ // code-family scores.
if present.contains("text") {
- out.push("text".to_string());
+ out.push("text^0.5".to_string());
}
if out.is_empty() {
// #1158: a raw-JSON corpus (ghsa-db's advisory mirrors, OSV) maps
@@ -208,12 +216,13 @@ mod tests {
/// #1139: the `.txt` family's content field is `text` — it joins the
/// multi_match list ONLY when an index actually maps it, so the
/// unreadable-mapping fallback list above stays exactly FIELDS.
+ /// #1238: it joins at ^0.5 (recall leg), never at full weight.
#[test]
fn text_family_content_joins_only_when_mapped() {
let with_text = mapping(&[("i", &["body", "text"], false)]);
assert_eq!(
resolve_fields(Some(&with_text)),
- vec!["body".to_string(), "text".to_string()]
+ vec!["body".to_string(), "text^0.5".to_string()]
);
// No FIELDS member AND no text-typed field -> the ["body"] floor
// still holds (keyword placeholders — text-typed ones are #1158's
diff --git a/engine/crates/xerj-common/src/xccode/mod.rs b/engine/crates/xerj-common/src/xccode/mod.rs
index 192b0484e..ec1b82936 100644
--- a/engine/crates/xerj-common/src/xccode/mod.rs
+++ b/engine/crates/xerj-common/src/xccode/mod.rs
@@ -37,7 +37,7 @@ pub mod pathgate;
pub mod render;
pub mod state;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
use std::path::Path;
use serde_json::Value;
@@ -411,15 +411,24 @@ pub fn run_code_query(
};
// 5. #1137: cut the overfetched window back to the page, capping records
- // per source file, and patch the diversified page into the response so
- // `--json` consumers see the SAME list the prose renderer does (the
- // G7 grader is one). A page that shrank says so in the note.
- let (hits, dropped) = diversify(hits, params.k);
- if dropped > 0 {
- let cap = format!(
- "top-k diversified: ≤{MAX_PER_FILE} records per source file \
- ({dropped} same-file neighbours skipped)"
- );
+ // per source file (#1137) and per source index (#1238), and patch the
+ // diversified page into the response so `--json` consumers see the
+ // SAME list the prose renderer does (the G7 grader is one). A page
+ // that shrank says so in the note, naming only the caps that fired.
+ let (hits, file_dropped, index_dropped) = diversify(hits, params.k);
+ if file_dropped > 0 || index_dropped > 0 {
+ let mut caps = Vec::new();
+ if file_dropped > 0 {
+ caps.push(format!(
+ "≤{MAX_PER_FILE} records per source file ({file_dropped} same-file neighbours)"
+ ));
+ }
+ if index_dropped > 0 {
+ caps.push(format!(
+ "≤{MAX_PER_INDEX} per source index ({index_dropped} same-index neighbours)"
+ ));
+ }
+ let cap = format!("top-k diversified: {}", caps.join(", "));
note = Some(match note.take() {
Some(n) => format!("{n}; {cap}"),
None => cap,
@@ -498,41 +507,87 @@ pub(crate) fn hit_list(resp: &Value) -> Vec<Value> {
/// Reference coding wants five FILES more than five passages of one.
pub(crate) const MAX_PER_FILE: usize = 2;
+/// Top-k diversity cap (#1238): at most this many records per `_index` (per
+/// source REPO in a corpus group) once the fan-out is wide enough to fill the
+/// page without any one index. Measured on the live exploit group (query
+/// "MCPJam inspector 23744", 36 needle PoC repos across 5,644 indices): one
+/// sibling-CVE demo repo took 6 of the top 10 slots — every file a DIFFERENT
+/// file, so the #1137 per-file cap could not see the wall — and the client's
+/// first page held 1 needle record where the per-index cap holds 5.
+pub(crate) const MAX_PER_INDEX: usize = 1;
+
/// Fetch depth that gives the cap candidates to draw on: 10×k, bounded so a
/// k=50 page does not become a 500-hit fetch, and never below k itself.
pub(crate) fn diversity_fetch(k: usize) -> usize {
k.saturating_mul(10).min(200).max(k)
}
-/// Cap records per `_source.ax_file` at [`MAX_PER_FILE`], truncate to `want`,
-/// preserving rank order. Returns the page and how many same-file neighbours
-/// the cap skipped while filling it. A hit with no `ax_file` is never grouped
-/// (missing provenance must not cost a slot).
-pub(crate) fn diversify(hits: Vec<Value>, want: usize) -> (Vec<Value>, usize) {
+/// Cap records per `_source.ax_file` at [`MAX_PER_FILE`], and — when the hit
+/// set spans at least `want` distinct indices — per `_index` at
+/// [`MAX_PER_INDEX`], truncate to `want`, preserving rank order. Returns the
+/// page plus how many neighbours EACH cap skipped while filling it (a cap
+/// that never engaged reports 0, so the reader-facing note names only the
+/// caps that fired). A hit with no `ax_file` (or no `_index`) is never
+/// grouped on that key: missing provenance must not cost a slot.
+///
+/// The distinct-index capacity guard keeps homogeneous corpora exactly as
+/// they are: a corpus whose whole fan-out is a handful of shard indices
+/// (the blogposts pack is 4) has fewer indices than slots, the per-index
+/// cap would only shrink the page, and so it never engages.
+pub(crate) fn diversify(hits: Vec<Value>, want: usize) -> (Vec<Value>, usize, usize) {
// Keys are owned: a borrowed &str would pin every hit it came from and
// forbid the move into `out` below.
+ let distinct_indices = hits
+ .iter()
+ .filter_map(|h| h.pointer("/_index").and_then(Value::as_str))
+ .collect::<HashSet<&str>>()
+ .len();
+ let cap_index = distinct_indices >= want;
let mut per_file: HashMap<String, usize> = HashMap::new();
+ let mut per_index: HashMap<String, usize> = HashMap::new();
let mut out = Vec::with_capacity(want.min(hits.len()));
- let mut dropped = 0usize;
+ let mut file_dropped = 0usize;
+ let mut index_dropped = 0usize;
for h in hits {
- let key = h
+ // Peek before consuming: a hit one cap skips must not burn the other
+ // cap's slot.
+ let file = h
.pointer("/_source/ax_file")
.and_then(Value::as_str)
.map(str::to_owned);
- if let Some(f) = key {
- let n = per_file.entry(f).or_insert(0);
- if *n >= MAX_PER_FILE {
- dropped += 1;
- continue;
+ let index = h
+ .pointer("/_index")
+ .and_then(Value::as_str)
+ .map(str::to_owned);
+ let file_full = file
+ .as_ref()
+ .is_some_and(|f| per_file.get(f).is_some_and(|n| *n >= MAX_PER_FILE));
+ let index_full = cap_index
+ && index
+ .as_ref()
+ .is_some_and(|i| per_index.get(i).is_some_and(|n| *n >= MAX_PER_INDEX));
+ if file_full || index_full {
+ if file_full {
+ file_dropped += 1;
+ } else {
+ index_dropped += 1;
+ }
+ continue;
+ }
+ if let Some(f) = file {
+ *per_file.entry(f).or_insert(0) += 1;
+ }
+ if cap_index {
+ if let Some(i) = index {
+ *per_index.entry(i).or_insert(0) += 1;
}
- *n += 1;
}
out.push(h);
if out.len() == want {
break;
}
}
- (out, dropped)
+ (out, file_dropped, index_dropped)
}
/// The measured BM25 body: flat `multi_match` over mapping-resolved fields
@@ -1250,7 +1305,7 @@ mod tests {
hits.push(mk("security", 9));
hits.push(mk("http-headers", 10));
- let (page, dropped) = diversify(hits, 5);
+ let (page, file_dropped, index_dropped) = diversify(hits, 5);
let files: Vec<&str> = page
.iter()
.map(|h| {
@@ -1270,9 +1325,13 @@ mod tests {
]
);
assert_eq!(
- dropped, 6,
+ file_dropped, 6,
"8 compatibility hits -> 2 kept, 6 skipped, rank order kept"
);
+ assert_eq!(
+ index_dropped, 0,
+ "no _index on these hits: cap never engages"
+ );
}
/// A page that cannot fill is returned SHORT, not padded back up with the
@@ -1287,9 +1346,10 @@ mod tests {
})
})
.collect();
- let (page, dropped) = diversify(hits, 5);
+ let (page, file_dropped, index_dropped) = diversify(hits, 5);
assert_eq!(page.len(), 2);
- assert_eq!(dropped, 8);
+ assert_eq!(file_dropped, 8);
+ assert_eq!(index_dropped, 0);
}
/// Missing provenance is not a reason to drop a hit: hits without
@@ -1299,9 +1359,10 @@ mod tests {
let hits: Vec<Value> = (0..4)
.map(|i| serde_json::json!({ "_score": 4.0 - i as f64, "_source": { "body": "b" } }))
.collect();
- let (page, dropped) = diversify(hits, 5);
+ let (page, file_dropped, index_dropped) = diversify(hits, 5);
assert_eq!(page.len(), 4, "no ax_file -> no cap applies");
- assert_eq!(dropped, 0);
+ assert_eq!(file_dropped, 0);
+ assert_eq!(index_dropped, 0);
}
/// 10x k, capped at 200, never below k.
@@ -1311,4 +1372,108 @@ mod tests {
assert_eq!(diversity_fetch(50), 200);
assert_eq!(diversity_fetch(1), 10);
}
+
+ /// #1238: one REPO (one `_index` in a corpus group) may not fill the page
+ /// when the fan-out has enough indices to fill it without any one of
+ /// them — the per-index wall the per-FILE cap cannot see, because every
+ /// hit is a different file. Measured shape: the sibling-CVE demo repo
+ /// took 6 of the top 10 slots on the exploit group while 36 needle
+ /// repos waited below.
+ #[test]
+ fn one_repo_may_not_fill_the_page_when_the_fanout_is_wide() {
+ let mk = |index: &str, i: usize| {
+ serde_json::json!({
+ "_index": index,
+ "_score": 10.0 - i as f64,
+ "_source": { "ax_file": format!("{index}/file{i}.py"),
+ "ax_path": format!("{index}/file{i}.py"),
+ "body": format!("passage {i} of {index}") }
+ })
+ };
+ // 6 wall hits from the demo repo, then one hit from each of 5
+ // distinct repos — distinct indices (6) >= want (5), so the cap
+ // engages and the page is one-per-repo.
+ let mut hits: Vec<Value> = (0..6).map(|i| mk("demo-repo", i)).collect();
+ for repo in ["needle-a", "needle-b", "needle-c", "needle-d", "needle-e"] {
+ hits.push(mk(repo, 6));
+ }
+ let (page, file_dropped, index_dropped) = diversify(hits, 5);
+ let indexes: Vec<&str> = page
+ .iter()
+ .map(|h| h.pointer("/_index").and_then(Value::as_str).unwrap())
+ .collect();
+ assert_eq!(
+ indexes,
+ vec!["demo-repo", "needle-a", "needle-b", "needle-c", "needle-d"],
+ "rank order kept, one slot per repo"
+ );
+ assert_eq!(index_dropped, 5, "6 demo-repo hits -> 1 kept, 5 skipped");
+ assert_eq!(file_dropped, 0, "every hit is a different file");
+ }
+
+ /// The capacity guard: a fan-out with FEWER indices than slots must keep
+ /// the pre-#1238 behaviour — the per-index cap would only shrink the
+ /// page (a homogeneous corpus's handful of shard indices is not a wall).
+ #[test]
+ fn a_narrow_fanout_keeps_the_per_file_cap_only() {
+ let hit = |index: &str, file: &str, i: usize| {
+ serde_json::json!({
+ "_index": index,
+ "_score": 10.0 - i as f64,
+ "_source": { "ax_file": file, "ax_path": file,
+ "body": format!("chunk {i}") }
+ })
+ };
+ // 2 distinct indices < want 5: the per-FILE cap still applies
+ // (third `file0` neighbour dropped), the per-index cap must not.
+ let hits = vec![
+ hit("xc-shard-000", "file0.md", 0),
+ hit("xc-shard-000", "file0.md", 1),
+ hit("xc-shard-000", "file0.md", 2),
+ hit("xc-shard-000", "file1.md", 3),
+ hit("xc-shard-001", "other0.md", 4),
+ hit("xc-shard-001", "other1.md", 5),
+ ];
+ let (page, file_dropped, index_dropped) = diversify(hits, 5);
+ assert_eq!(page.len(), 5, "5 fill from 2 indices without the index cap");
+ assert_eq!(
+ file_dropped, 1,
+ "only the third same-file neighbour is capped"
+ );
+ assert_eq!(index_dropped, 0, "2 indices < 5 slots: the guard holds");
+ }
+
+ /// A hit skipped by the per-index cap must not burn its file's slot: the
+ /// per-file counters count EMITTED records only.
+ #[test]
+ fn an_index_capped_hit_does_not_consume_its_file_slot() {
+ let mk = |index: &str, file: &str, i: usize| {
+ serde_json::json!({
+ "_index": index,
+ "_score": 10.0 - i as f64,
+ "_source": { "ax_file": file, "ax_path": file,
+ "body": format!("passage {i}") }
+ })
+ };
+ // repo-a slot already spent; this hit is a NEW file in repo-a, so
+ // only the index cap skips it — and fileX's slot must stay untouched
+ // for the later hit from repo-b/fileX.
+ let hits = vec![
+ mk("repo-a", "file1", 0),
+ mk("repo-a", "fileX", 1),
+ mk("repo-b", "fileX", 2),
+ ];
+ let (page, file_dropped, index_dropped) = diversify(hits, 2);
+ assert_eq!(index_dropped, 1, "the repo-a/fileX hit is index-capped");
+ assert_eq!(file_dropped, 0);
+ let files: Vec<&str> = page
+ .iter()
+ .map(|h| {
+ h.pointer("/_source/ax_file")
+ .and_then(Value::as_str)
+ .unwrap()
+ })
+ .collect();
+ assert_eq!(files, vec!["file1", "fileX"], "fileX's slot was not burned");
+ }
}