Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(search): demote test and example paths in plain-word search (#2125)

xerj-org5e283eb3eb00
diff --git a/engine/crates/xerj-autoindex/src/def.rs b/engine/crates/xerj-autoindex/src/def.rs
index 4ec7ad6ed..9f68b8b6c 100644
--- a/engine/crates/xerj-autoindex/src/def.rs
+++ b/engine/crates/xerj-autoindex/src/def.rs
@@ -305,10 +305,80 @@ fn glob_match(pattern: &str, text: &str) -> bool {
rec(pattern.as_bytes(), text.as_bytes())
}
-fn path_is_demoted(path: &str) -> bool {
+/// True when `path` is a test, example, fuzz target, or bench
+/// (`tests/…`, `examples/…`, `fuzz/…`, `benches/…`, at any directory depth).
+///
+/// `xerj def` withholds the preferred-kind bonus on these paths.
+/// Plain-word `xerj search` and the plain-string `xerj_search` tool apply
+/// [`penalize_demoted_paths`] with the same predicate (#2122) — one set of
+/// path rules, not a second copy. `*` matches across `/`.
+pub fn path_is_demoted(path: &str) -> bool {
DEMOTED_PATH_GLOBS.iter().any(|glob| glob_match(glob, path))
}
+/// Share of its BM25 `_score` a demoted path keeps.
+///
+/// Halving puts a test file that only barely outscored the implementation
+/// behind it, and leaves a test file that scored more than twice as high —
+/// the query is about that test — in front. The hit is never removed.
+pub const DEMOTED_PATH_SCORE_FACTOR: f64 = 0.5;
+
+fn source_path_for_demotion(src: &Value) -> &str {
+ src.get("ax_path")
+ .or_else(|| src.get("path"))
+ .and_then(Value::as_str)
+ .unwrap_or("")
+}
+
+/// Ranking penalty for plain-word search over a code index (#2122).
+///
+/// Demoted paths keep [`DEMOTED_PATH_SCORE_FACTOR`] of their `_score`, then
+/// the slice is stable-sorted by that score. This is not a filter: a page
+/// that is only tests is unchanged in order, and a test that still leads
+/// after the penalty stays first. Paths the predicate does not match,
+/// including `*.test.ts` and `fixtures/`, are left alone. Hits with no
+/// numeric `_score` are not given one.
+pub fn penalize_demoted_paths(hits: &mut [Value]) {
+ let mut adjusted: Vec<(f64, Option<f64>)> = Vec::with_capacity(hits.len());
+ let mut any = false;
+ for hit in hits.iter() {
+ let src = hit.get("_source").unwrap_or(&Value::Null);
+ let demoted = path_is_demoted(source_path_for_demotion(src));
+ let base = hit.get("_score").and_then(Value::as_f64);
+ let (sort_score, rewrite) = match base {
+ Some(base) if demoted => {
+ let penalized = base * DEMOTED_PATH_SCORE_FACTOR;
+ (penalized, Some(penalized))
+ }
+ Some(base) => (base, None),
+ None => (0.0, None),
+ };
+ any |= demoted;
+ adjusted.push((sort_score, rewrite));
+ }
+ if !any {
+ return;
+ }
+ let mut order: Vec<usize> = (0..hits.len()).collect();
+ order.sort_by(|&a, &b| {
+ adjusted[b]
+ .0
+ .partial_cmp(&adjusted[a].0)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ .then(a.cmp(&b))
+ });
+ let original = hits.to_vec();
+ for (place, old) in order.into_iter().enumerate() {
+ let mut hit = original[old].clone();
+ if let Some(score) = adjusted[old].1 {
+ if let Some(obj) = hit.as_object_mut() {
+ obj.insert("_score".into(), json!(score));
+ }
+ }
+ hits[place] = hit;
+ }
+}
+
/// The old `function_score` filter matched the tokens `pub`, `public`, and
/// `export` on `code`. Split on non-letters so `publication` does not count
/// and `pub(crate)` does.
@@ -331,15 +401,8 @@ pub(crate) fn rerank_def_hits(hits: &mut [Value]) {
let src = hit.get("_source").unwrap_or(&Value::Null);
let mut bonus = 0.0;
let kind = src.get("kind").and_then(Value::as_str).unwrap_or("");
- if PREFERRED_KINDS.contains(&kind) {
- let path = src
- .get("ax_path")
- .or_else(|| src.get("path"))
- .and_then(Value::as_str)
- .unwrap_or("");
- if !path_is_demoted(path) {
- bonus += KIND_WEIGHT;
- }
+ if PREFERRED_KINDS.contains(&kind) && !path_is_demoted(source_path_for_demotion(src)) {
+ bonus += KIND_WEIGHT;
}
if src
.get("code")
@@ -779,6 +842,11 @@ mod tests {
assert!(path_is_demoted("examples/dump.rs"));
assert!(path_is_demoted("fuzz/fuzz_targets/x.rs") || path_is_demoted("tokio/fuzz/a.rs"));
assert!(!path_is_demoted("tokio/src/runtime/runtime.rs"));
+ assert!(
+ !path_is_demoted("src/Foo.test.tsx"),
+ "a test suffix is not one of the path rules"
+ );
+ assert!(!path_is_demoted("src/fixtures/transcript.json"));
assert!(!declaration_is_public("struct Runtime"));
assert!(declaration_is_public("pub(crate) struct Runtime"));
assert!(!declaration_is_public("publication"));
diff --git a/engine/crates/xerj-autoindex/src/search.rs b/engine/crates/xerj-autoindex/src/search.rs
index e04959f3f..e0ce51d9a 100644
--- a/engine/crates/xerj-autoindex/src/search.rs
+++ b/engine/crates/xerj-autoindex/src/search.rs
@@ -324,6 +324,34 @@ pub(crate) fn prefer_citable_line_on_the_same_path(hits: &mut [Value]) {
}
}
+/// How many hits to fetch before the local test/example penalty trims to `k`.
+///
+/// The extra window is the one `xerj def` uses, so an implementation just
+/// outside the engine's top `k` can outrank a higher-BM25 test file. The
+/// query stays a plain bool: a `function_score` wildcard on `ax_path` is the
+/// path `xerj def` already measured at 422ms against 6ms.
+pub fn search_fetch_size(k: usize) -> usize {
+ crate::def::def_collect_size(k)
+}
+
+/// Order a plain-word search page.
+///
+/// On a code corpus, [`crate::def::penalize_demoted_paths`] halves the score
+/// of test/example/fuzz/bench paths and the page is trimmed back to `page`
+/// after the existing same-path and catalog passes. A prose corpus skips
+/// the penalty and the trim — the node was asked for `page` hits already.
+pub(crate) fn rank_plain_search_hits(hits: &mut Vec<Value>, code_corpus: bool, page: usize) {
+ if code_corpus {
+ crate::def::penalize_demoted_paths(hits);
+ }
+ prefer_citable_line_on_the_same_path(hits);
+ let mut ordered = demote_catalog_noise(std::mem::take(hits));
+ if code_corpus {
+ ordered.truncate(page);
+ }
+ *hits = ordered;
+}
+
/// What the node held when a search or def returned no hits.
///
/// `None` means that read failed. An empty `Some` is a real answer.
@@ -670,7 +698,14 @@ pub fn run_search_cli() -> i32 {
let numeric = mapped
.as_ref()
.and(raw_mapping.as_ref().map(numeric_property_names));
- let body = search_query_body_for_mapping(&query, k, mapped.as_ref(), numeric.as_ref());
+ // A readable mapping with none of `name` / `code` / `defs` is prose.
+ // An unreadable mapping keeps the code-shaped query, and the same
+ // test/example penalty (#2122).
+ let code_corpus = !xerj_common::code_query::code_symbol_fields_absent(mapped.as_ref());
+ let mut body = search_query_body_for_mapping(&query, k, mapped.as_ref(), numeric.as_ref());
+ if code_corpus && k > 0 {
+ body["size"] = json!(search_fetch_size(k));
+ }
let mut resp = match es.search(&pattern, &body) {
Ok(v) => v,
Err(e) => {
@@ -678,18 +713,11 @@ pub fn run_search_cli() -> i32 {
return 1;
}
};
+ // Test/example penalty, then the same-path line preference and the
+ // catalog partition (#1504). `--json` sees the same order as the printer.
+ // The penalty trims the wider fetch back to `-k`.
if let Some(hits) = resp.pointer_mut("/hits/hits").and_then(Value::as_array_mut) {
- prefer_citable_line_on_the_same_path(hits);
- }
-
- // Stubs and catalogs repeat the words of a question and outrank the
- // implementation. Same partition `xerj code` applies (#1504, candle
- // `.pyi` report). `--json` sees the same order as the text printer.
- if let Some(arr) = resp.pointer("/hits/hits").and_then(Value::as_array) {
- let ordered = demote_catalog_noise(arr.clone());
- if let Some(slot) = resp.pointer_mut("/hits/hits") {
- *slot = Value::Array(ordered);
- }
+ rank_plain_search_hits(hits, code_corpus, k);
}
if as_json {
println!(
@@ -1686,4 +1714,113 @@ mod tests {
search_query_body_for_mapping("timeout", 5, Some(&prose_names), Some(&prose_numeric));
assert!(none["query"].get("match_none").is_some(), "{none}");
}
+
+ fn hit(path_key: &str, path: &str, score: f64) -> Value {
+ json!({
+ "_score": score,
+ "_source": { path_key: path }
+ })
+ }
+
+ /// #2122: a test/example path that only barely leads is ranked behind the
+ /// implementation, and it is still in the page. A test that scores more
+ /// than twice as high stays ahead of the implementation. A `*.test.ts`
+ /// file is not a demoted path — the predicate is `path_is_demoted`, not
+ /// a second rule list.
+ #[test]
+ fn code_search_penalizes_test_paths_and_keeps_them() {
+ let mut hits = vec![
+ hit("ax_path", "src/tests/render.test.tsx", 12.0),
+ hit("ax_path", "examples/App.tsx", 11.0),
+ hit("ax_path", "src/render.tsx", 9.0),
+ hit("path", "tests/render_contract.rs", 30.0),
+ hit("ax_path", "src/Foo.test.tsx", 14.0),
+ hit("ax_path", "src/fixtures/transcript.json", 8.0),
+ ];
+ rank_plain_search_hits(&mut hits, true, 5);
+ let paths: Vec<&str> = hits
+ .iter()
+ .map(|h| {
+ h["_source"]
+ .get("ax_path")
+ .or_else(|| h["_source"].get("path"))
+ .and_then(Value::as_str)
+ .unwrap()
+ })
+ .collect();
+ // 30→15 and 14 stay ahead of the implementation at 9. 12→6 drops behind
+ // it but stays on the page. 11→5.5 falls off the page of 5; it is
+ // not deleted when the page has room (asserted below).
+ assert_eq!(
+ paths,
+ vec![
+ "tests/render_contract.rs",
+ "src/Foo.test.tsx",
+ "src/render.tsx",
+ "src/fixtures/transcript.json",
+ "src/tests/render.test.tsx",
+ ],
+ "{paths:?}"
+ );
+ let mut wide = vec![
+ hit("ax_path", "examples/App.tsx", 11.0),
+ hit("ax_path", "src/render.tsx", 9.0),
+ ];
+ rank_plain_search_hits(&mut wide, true, 5);
+ assert_eq!(wide.len(), 2, "a demoted hit stays when it still fits");
+ assert_eq!(wide[0]["_source"]["ax_path"], "src/render.tsx");
+ assert_eq!(wide[1]["_source"]["ax_path"], "examples/App.tsx");
+ assert_eq!(wide[1]["_score"], json!(5.5));
+ assert_eq!(wide[0]["_score"], json!(9.0));
+
+ let mut only_tests = vec![
+ hit("ax_path", "tests/a.rs", 10.0),
+ hit("ax_path", "fuzz/targets/b.rs", 8.0),
+ ];
+ rank_plain_search_hits(&mut only_tests, true, 5);
+ assert_eq!(only_tests[0]["_source"]["ax_path"], "tests/a.rs");
+ assert_eq!(only_tests[1]["_source"]["ax_path"], "fuzz/targets/b.rs");
+ assert_eq!(only_tests[0]["_score"], json!(5.0));
+
+ let mut prose = vec![
+ hit("ax_path", "tests/a.rs", 12.0),
+ hit("ax_path", "src/render.tsx", 9.0),
+ ];
+ rank_plain_search_hits(&mut prose, false, 1);
+ assert_eq!(prose.len(), 2, "a prose page is not trimmed");
+ assert_eq!(prose[0]["_source"]["ax_path"], "tests/a.rs");
+ assert_eq!(prose[0]["_score"], json!(12.0));
+ }
+
+ /// The fetch is wider than `-k` on purpose, and the query body does not
+ /// grow a `function_score` / `ax_path` wildcard to do the demotion.
+ #[test]
+ fn code_search_window_is_local_and_wider_than_the_page() {
+ assert_eq!(search_fetch_size(5), 45);
+ assert!(search_fetch_size(5) > 5);
+ let body = search_query_body("render the transcript", 5);
+ let rendered = body["query"].to_string();
+ assert!(!rendered.contains("function_score"), "{rendered}");
+ assert!(!rendered.contains("ax_path"), "{rendered}");
+ assert_eq!(body["size"], 5, "the builder stays at k; the CLI widens");
+ }
+
+ /// Catalog stubs still sink, on a code page and a prose page.
+ #[test]
+ fn path_penalty_does_not_undo_catalog_demotion() {
+ let mut hits = vec![
+ json!({"_score": 30.0, "_source": {"ax_path": "src/model.pyi"}}),
+ hit("ax_path", "tests/test_model.py", 12.0),
+ hit("ax_path", "src/model.py", 9.0),
+ ];
+ rank_plain_search_hits(&mut hits, true, 5);
+ let paths: Vec<&str> = hits
+ .iter()
+ .map(|h| h["_source"]["ax_path"].as_str().unwrap())
+ .collect();
+ assert_eq!(
+ paths,
+ vec!["src/model.py", "tests/test_model.py", "src/model.pyi"]
+ );
+ }
}
diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rs
index 950894b96..e23efb79f 100644
--- a/engine/crates/xerj-mcp/src/lib.rs
+++ b/engine/crates/xerj-mcp/src/lib.rs
@@ -106,11 +106,14 @@ use std::time::Duration;
use serde_json::{json, Value};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use xerj_common::code_query::{
- mapped_property_names, numeric_property_names, search_query_body_for_mapping,
+ code_symbol_fields_absent, mapped_property_names, numeric_property_names,
+ search_query_body_for_mapping,
};
/// MCP protocol revision we default to when the client does not request one.
const DEFAULT_PROTOCOL_VERSION: &str = "2025-06-18";
+/// `POST /_search` page size when the caller omits `size` (`EsSearchBody`).
+const DEFAULT_SEARCH_SIZE: usize = 10;
/// Default XERJ ES-compatible endpoint.
const DEFAULT_XERJ_URL: &str = "http://localhost:9200";
@@ -1773,6 +1776,9 @@ async fn call_tool(ctx: &Ctx, msg: &Value) -> Value {
None
};
+ // Set only for a plain-string code search (#2122). `Some(page)` means
+ // the response hits are penalized and trimmed back to `page`.
+ let mut demote_page: Option<usize> = None;
let result = match name {
"xerj_search" => {
// #1961: a plain-string query is shaped from the target's mapping,
@@ -1780,16 +1786,40 @@ async fn call_tool(ctx: &Ctx, msg: &Value) -> Value {
// answers HTTP 400 `strict_unknown_fields` because the body names
// `defs` and `name`, which it does not map. A DSL object is
// forwarded untouched (no mapping read, no rewriting).
- let mapping = match args.get("query") {
- Some(Value::String(q)) if !q.trim().is_empty() => {
- fetch_mapping(ctx, search_index(&args)).await
- }
- _ => None,
+ let plain = matches!(
+ args.get("query"),
+ Some(Value::String(q)) if !q.trim().is_empty()
+ );
+ let mapping = if plain {
+ fetch_mapping(ctx, search_index(&args)).await
+ } else {
+ None
};
- match mapping {
- Some(mapping) => build_search_with_mapping(&args, Some(&mapping)),
+ let mut built = match mapping.as_ref() {
+ Some(mapping) => build_search_with_mapping(&args, Some(mapping)),
None => build_search(&args),
+ };
+ if plain_search_demotes_test_paths(&args, mapping.as_ref()) {
+ let page = match opt_u64(&args, "size") {
+ Ok(Some(n)) => n as usize,
+ Ok(None) => DEFAULT_SEARCH_SIZE,
+ Err(msg) => return tool_text(msg, true),
+ };
+ let from = match opt_u64(&args, "from") {
+ Ok(Some(n)) => n,
+ Ok(None) => 0,
+ Err(msg) => return tool_text(msg, true),
+ };
+ demote_page = Some(page);
+ // First page only: a wider fetch, then trim back to `page`
+ // after the penalty. `from` > 0 keeps the caller's window.
+ if from == 0 && page > 0 {
+ if let Ok((_, _, Some(body))) = built.as_mut() {
+ body["size"] = json!(xerj_autoindex::search::search_fetch_size(page));
+ }
+ }
}
+ built
}
"xerj_semantic_search" => build_semantic(&args),
"xerj_vector_search" => build_vector(&args),
@@ -1830,11 +1860,55 @@ async fn call_tool(ctx: &Ctx, msg: &Value) -> Value {
};
match result {
- Ok((method, path, body)) => engine_request(ctx, method, &path, body, budget).await,
+ Ok((method, path, body)) => {
+ engine_request(ctx, method, &path, body, budget, demote_page).await
+ }
Err(msg) => tool_text(msg, true),
}
}
+/// Plain-word `xerj_search` on a code corpus (#2122).
+///
+/// Same gate as `xerj search`: a readable mapping with none of `name` /
+/// `code` / `defs` is prose and is left on the engine's order. An unreadable
+/// mapping keeps the code-shaped query, so the demotion applies there too.
+/// An explicit `sort`, `rerank`, or `judge` owns the order.
+fn plain_search_demotes_test_paths(args: &Value, mapping: Option<&Value>) -> bool {
+ let Some(Value::String(q)) = args.get("query") else {
+ return false;
+ };
+ if q.trim().is_empty() {
+ return false;
+ }
+ if ["sort", "rerank", "judge"]
+ .iter()
+ .any(|key| args.get(*key).is_some_and(|v| !v.is_null()))
+ {
+ return false;
+ }
+ let mapped = mapping.and_then(mapped_property_names);
+ !code_symbol_fields_absent(mapped.as_ref())
+}
+
+/// Halve test/example/fuzz/bench scores and trim to the caller's page.
+///
+/// The penalty is [`xerj_autoindex::def::penalize_demoted_paths`] — the same
+/// `path_is_demoted` rules `xerj def` uses. A response that is not a search
+/// hit list is returned unchanged.
+fn apply_search_path_demotion(text: &str, page: usize) -> String {
+ let Ok(mut resp) = serde_json::from_str::<Value>(text) else {
+ return text.to_string();
+ };
+ {
+ let Some(hits) = resp.pointer_mut("/hits/hits").and_then(Value::as_array_mut) else {
+ return text.to_string();
+ };
+ xerj_autoindex::def::penalize_demoted_paths(hits);
+ hits.truncate(page);
+ }
+ serde_json::to_string(&resp).unwrap_or_else(|_| text.to_string())
+}
+
/// HTTP method of a proxied engine request. The engine's graph surface is not
/// POST-only (ego/overview are GET, unlink is DELETE), so builders carry it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -2831,6 +2905,7 @@ async fn engine_request(
path: &str,
body: Option<Value>,
budget: Option<usize>,
+ demote_page: Option<usize>,
) -> Value {
let url = format!("{}{}", ctx.base_url, path);
let mut req = match method {
@@ -2850,6 +2925,10 @@ async fn engine_request(
let status = resp.status();
let text = resp.text().await.unwrap_or_default();
if status.is_success() {
+ let text = match demote_page {
+ Some(page) => apply_search_path_demotion(&text, page),
+ None => text,
+ };
match budget {
Some(max_tokens) => {
tool_text(token_budget::budget_json_response(&text, max_tokens), false)
@@ -3201,6 +3280,82 @@ mod tests {
);
}
+ /// #2122: the demotion is plain-word search on a code corpus. A prose
+ /// mapping, a DSL object, and an explicit sort/rerank/judge do not take it.
+ #[test]
+ fn plain_search_demotion_follows_the_code_corpus_gate() {
+ let code = json!({
+ "ax-rust": { "mappings": { "properties": {
+ "name": {"type": "keyword"}, "code": {"type": "text"}, "defs": {"type": "text"}
+ }}}
+ });
+ let prose = json!({
+ "ax-docs": { "mappings": { "properties": {
+ "body": {"type": "text"}, "title": {"type": "text"}
+ }}}
+ });
+ assert!(plain_search_demotes_test_paths(
+ &json!({"query": "render the transcript"}),
+ Some(&code)
+ ));
+ assert!(plain_search_demotes_test_paths(
+ &json!({"query": "render the transcript"}),
+ None
+ ));
+ assert!(!plain_search_demotes_test_paths(
+ &json!({"query": "render the transcript"}),
+ Some(&prose)
+ ));
+ assert!(!plain_search_demotes_test_paths(
+ &json!({"query": {"match": {"body": "render"}}}),
+ Some(&code)
+ ));
+ assert!(!plain_search_demotes_test_paths(
+ &json!({"query": "render", "sort": ["_score"]}),
+ Some(&code)
+ ));
+ assert!(!plain_search_demotes_test_paths(
+ &json!({"query": "render", "rerank": {"query": "render"}}),
+ Some(&code)
+ ));
+ assert!(!plain_search_demotes_test_paths(
+ &json!({"query": " "}),
+ Some(&code)
+ ));
+ }
+
+ /// The same penalty the CLI applies: a barely-leading test drops behind
+ /// the implementation and stays on the page; the wider fetch is trimmed
+ /// back to the caller's size.
+ #[test]
+ fn plain_string_response_penalizes_test_paths_and_trims_the_window() {
+ let raw = r#"{"hits":{"hits":[
+ {"_id":"t","_score":12.0,"_source":{"ax_path":"src/tests/render.test.tsx"}},
+ {"_id":"i","_score":9.0,"_source":{"ax_path":"src/render.tsx"}},
+ {"_id":"s","_score":30.0,"_source":{"ax_path":"tests/contract.rs"}},
+ {"_id":"x","_score":1.0,"_source":{"ax_path":"src/extra.rs"}}
+ ]}}"#;
+ let text = apply_search_path_demotion(raw, 3);
+ let resp: Value = serde_json::from_str(&text).unwrap();
+ let paths: Vec<&str> = resp["hits"]["hits"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|h| h["_source"]["ax_path"].as_str().unwrap())
+ .collect();
+ assert_eq!(
+ paths,
+ vec![
+ "tests/contract.rs",
+ "src/render.tsx",
+ "src/tests/render.test.tsx"
+ ]
+ );
+ assert_eq!(resp["hits"]["hits"][2]["_score"], json!(6.0));
+ assert_eq!(resp["hits"]["hits"][1]["_score"], json!(9.0));
+ assert_eq!(apply_search_path_demotion("not json", 3), "not json");
+ }
+
/// A numeric mapping of a code field drops that leg (#1284 via #1601):
/// naming it fails the whole wildcard request with `For input string`.
#[test]
@@ -4077,6 +4232,10 @@ mod tests {
/// One HTTP/1.1 request, then a tiny JSON 200. Reads Content-Length so the
/// client is not left waiting on a half-consumed body.
fn read_one_http(listener: std::net::TcpListener) -> String {
+ read_one_http_reply(listener, r#"{"ok":true,"stored":true}"#.to_string())
+ }
+
+ fn read_one_http_reply(listener: std::net::TcpListener, response_body: String) -> String {
use std::io::{Read, Write};
let (mut sock, _) = listener.accept().expect("accept");
sock.set_read_timeout(Some(std::time::Duration::from_secs(5)))
@@ -4112,7 +4271,6 @@ mod tests {
Err(_) => break,
}
}
- let response_body = r#"{"ok":true,"stored":true}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response_body}",
response_body.len()
@@ -4218,6 +4376,161 @@ mod tests {
request.contains("\"text\""),
"the mapped prose field was not searched: {request}"
);
+ assert!(
+ !request.contains("\"size\""),
+ "a prose corpus must not widen the fetch for the code-path penalty: {request}"
+ );
+ }
+
+ /// #2122: plain-word search on a code index asks for the rerank window and
+ /// returns the implementation ahead of a test file that only barely led.
+ /// The test file is still in the page.
+ #[test]
+ fn search_tool_demotes_test_paths_on_a_code_index() {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let port = listener.local_addr().unwrap().port();
+ let mapping = r#"{"ax-code":{"mappings":{"properties":{"name":{"type":"keyword"},"code":{"type":"text"},"defs":{"type":"text"},"body":{"type":"text"}}}}}"#;
+ let hits = r#"{"took":1,"hits":{"total":{"value":3,"relation":"eq"},"hits":[{"_id":"t","_score":12.0,"_source":{"ax_path":"src/tests/render.test.tsx"}},{"_id":"i","_score":9.0,"_source":{"ax_path":"src/render.tsx"}},{"_id":"s","_score":30.0,"_source":{"ax_path":"tests/contract.rs"}}]}}"#;
+ let server = std::thread::spawn(move || serve_mapping_then_search(listener, mapping, hits));
+ let ctx = Ctx {
+ client: reqwest::Client::new(),
+ base_url: format!("http://127.0.0.1:{port}"),
+ auth: None,
+ };
+ let msg = json!({
+ "params": { "name": "xerj_search", "arguments": {
+ "query": "render the transcript",
+ "size": 3
+ } }
+ });
+ let res = block_on(call_tool(&ctx, &msg));
+ let request = server.join().expect("fake node thread");
+ assert_eq!(res["isError"], false, "{res}");
+ let window = xerj_autoindex::search::search_fetch_size(3);
+ assert!(
+ request.contains(&format!("\"size\":{window}")),
+ "the first page must fetch the rerank window {window}: {request}"
+ );
+ let body: Value = serde_json::from_str(&tool_body(&res)).unwrap();
+ let paths: Vec<&str> = body["hits"]["hits"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|h| h["_source"]["ax_path"].as_str().unwrap())
+ .collect();
+ assert_eq!(
+ paths,
+ vec![
+ "tests/contract.rs",
+ "src/render.tsx",
+ "src/tests/render.test.tsx"
+ ]
+ );
+ }
+
+ /// A prose corpus keeps the engine order, including a `tests/` path.
+ #[test]
+ fn search_tool_does_not_demote_paths_on_a_prose_index() {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let port = listener.local_addr().unwrap().port();
+ let mapping = r#"{"ax-docs":{"mappings":{"properties":{"body":{"type":"text"},"title":{"type":"text"},"text":{"type":"text"}}}}}"#;
+ let hits = r#"{"took":1,"hits":{"hits":[{"_id":"t","_score":12.0,"_source":{"ax_path":"tests/guide.md"}},{"_id":"i","_score":9.0,"_source":{"ax_path":"guide.md"}}]}}"#;
+ let server = std::thread::spawn(move || serve_mapping_then_search(listener, mapping, hits));
+ let ctx = Ctx {
+ client: reqwest::Client::new(),
+ base_url: format!("http://127.0.0.1:{port}"),
+ auth: None,
+ };
+ let msg = json!({
+ "params": { "name": "xerj_search", "arguments": {
+ "query": "how does the retry policy work"
+ } }
+ });
+ let res = block_on(call_tool(&ctx, &msg));
+ let request = server.join().expect("fake node thread");
+ assert_eq!(res["isError"], false, "{res}");
+ assert!(!request.contains("\"size\""), "{request}");
+ let body: Value = serde_json::from_str(&tool_body(&res)).unwrap();
+ assert_eq!(
+ body["hits"]["hits"][0]["_source"]["ax_path"],
+ "tests/guide.md"
+ );
+ assert_eq!(body["hits"]["hits"][0]["_score"], json!(12.0));
+ }
+
+ /// A DSL object is forwarded and returned in the engine's order. `from`
+ /// keeps the caller's page size instead of the rerank window.
+ #[test]
+ fn search_tool_leaves_a_dsl_query_and_a_later_page_unwidened() {
+ let hits = r#"{"hits":{"hits":[{"_id":"t","_score":12.0,"_source":{"ax_path":"tests/a.rs"}},{"_id":"i","_score":9.0,"_source":{"ax_path":"src/a.rs"}}]}}"#;
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let port = listener.local_addr().unwrap().port();
+ let body = hits.to_string();
+ let server = std::thread::spawn(move || read_one_http_reply(listener, body));
+ let ctx = Ctx {
+ client: reqwest::Client::new(),
+ base_url: format!("http://127.0.0.1:{port}"),
+ auth: None,
+ };
+ let res = block_on(call_tool(
+ &ctx,
+ &json!({
+ "params": { "name": "xerj_search", "arguments": {
+ "query": { "match": { "body": "render the transcript" } }
+ } }
+ }),
+ ));
+ let request = server.join().expect("fake node thread");
+ assert_eq!(res["isError"], false, "{res}");
+ assert!(
+ request.contains("\"match\""),
+ "a DSL query is forwarded: {request}"
+ );
+ let parsed: Value = serde_json::from_str(&tool_body(&res)).unwrap();
+ assert_eq!(
+ parsed["hits"]["hits"][0]["_source"]["ax_path"],
+ "tests/a.rs"
+ );
+
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let port = listener.local_addr().unwrap().port();
+ let mapping = r#"{"ax-code":{"mappings":{"properties":{"name":{"type":"keyword"},"code":{"type":"text"},"defs":{"type":"text"}}}}}"#;
+ let page_hits = hits.to_string();
+ let server =
+ std::thread::spawn(move || serve_mapping_then_search(listener, mapping, &page_hits));
+ let ctx = Ctx {
+ client: reqwest::Client::new(),
+ base_url: format!("http://127.0.0.1:{port}"),
+ auth: None,
+ };
+ let res = block_on(call_tool(
+ &ctx,
+ &json!({
+ "params": { "name": "xerj_search", "arguments": {
+ "query": "render the transcript",
+ "size": 3,
+ "from": 3
+ } }
+ }),
+ ));
+ let request = server.join().expect("fake node thread");
+ assert_eq!(res["isError"], false, "{res}");
+ assert!(
+ request.contains("\"from\":3"),
+ "the caller's offset is forwarded: {request}"
+ );
+ assert!(
+ request.contains("\"size\":3") && !request.contains("\"size\":43"),
+ "a later page keeps the caller's size: {request}"
+ );
+ let parsed: Value = serde_json::from_str(&tool_body(&res)).unwrap();
+ let paths: Vec<&str> = parsed["hits"]["hits"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|h| h["_source"]["ax_path"].as_str().unwrap())
+ .collect();
+ assert_eq!(paths, vec!["src/a.rs", "tests/a.rs"]);
}
fn block_on<T>(future: impl std::future::Future<Output = T>) -> T {