Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(def): rerank kind and visibility on the returned page (#1704)

xerj-orgaadaff0f9257
diff --git a/engine/crates/xerj-autoindex/src/def.rs b/engine/crates/xerj-autoindex/src/def.rs
index cfdf6a5a9..d7d146309 100644
--- a/engine/crates/xerj-autoindex/src/def.rs
+++ b/engine/crates/xerj-autoindex/src/def.rs
@@ -239,77 +239,132 @@ fn build_def_query(
// A filter clause makes every `should` optional, so without this a
// `--kind`/`--lang` filter alone matches: a name defined nowhere came
// back as every definition of that kind instead of "no definition".
- let bool_query = json!({
- "bool": {
- "should": should,
- "filter": filter,
- "minimum_should_match": 1
- }
- });
- // Kind and `pub` weights sit on function_score, not on a bare `should`:
- // a should clause would match every struct in the index. The negative
- // boosting clause only multiplies hits that already matched.
- let query = rank_adjusted_query(bool_query, mapped, numeric);
+ // Kind and `pub` weights are applied in [`rerank_def_hits`] on the
+ // returned page. They used to be a `function_score` filter. On the tokio
+ // index (12,107 docs) that wrapper took 422ms of engine time against 6ms
+ // for this bool; a `wildcard` on `ax_path` and a `match` on `code` inside
+ // the function are what made the scan leave the bounded top-k path.
json!({
"size": k,
- "query": query,
+ "query": {
+ "bool": {
+ "should": should,
+ "filter": filter,
+ "minimum_should_match": 1
+ }
+ },
"_source": ["ax_path", "path", "line", "start_line", "kind", "language", "name", "code"],
"fields": ["_passage"]
})
}
-fn rank_adjusted_query(
- bool_query: Value,
- mapped: Option<&BTreeSet<String>>,
- numeric: Option<&BTreeSet<String>>,
-) -> Value {
- // `function_score` (boost_mode sum) adds weight only to hits the bool
- // already matched. A `boosting` wrapper was tried first; this engine's
- // planner rewrites it to a bool and every hit came back with `_score`
- // 1.0, so the weights never moved the order.
- let mut functions = Vec::new();
- // A numeric `kind` or `code` cannot carry the ranking filter: the
- // wildcard would fail on that index the same way a `term` on a numeric
- // `name` does. Unreadable mappings (numeric = None) keep both weights.
- if field_lookup_ok(mapped, numeric, "kind") {
- let mut kind_filter = json!({
- "bool": { "must": [ { "terms": { "kind": PREFERRED_KINDS } } ] }
- });
- // A preferred kind under tests/examples/fuzz/benches does not get
- // the weight. `ax_path` is only named when the read mapping has it,
- // so the historical query (mapped = None) stays free of that field.
- if explicitly_mapped(mapped, "ax_path") {
- let globs: Vec<Value> = DEMOTED_PATH_GLOBS
- .iter()
- .map(|glob| json!({ "wildcard": { "ax_path": glob } }))
- .collect();
- kind_filter["bool"]["must_not"] = json!([{
- "bool": { "should": globs, "minimum_should_match": 1 }
- }]);
+/// How many hits to fetch before the local rerank. The printed page is `k`.
+/// The extra window is what lets a `pub struct` just outside the top `k`
+/// outrank a higher-BM25 test double. Capped so `-k 1000` does not pull
+/// every match.
+pub(crate) fn def_collect_size(k: usize) -> usize {
+ const WINDOW: usize = 40;
+ const CAP: usize = 200;
+ k.saturating_add(WINDOW).min(k.max(CAP))
+}
+
+/// `*` is zero or more characters, including `/` — the same reading the
+/// engine's `wildcard` query uses for [`DEMOTED_PATH_GLOBS`].
+fn glob_match(pattern: &str, text: &str) -> bool {
+ fn rec(pattern: &[u8], text: &[u8]) -> bool {
+ let mut i = 0;
+ let mut j = 0;
+ while i < pattern.len() {
+ if pattern[i] == b'*' {
+ while i < pattern.len() && pattern[i] == b'*' {
+ i += 1;
+ }
+ if i == pattern.len() {
+ return true;
+ }
+ while j <= text.len() {
+ if rec(&pattern[i..], &text[j..]) {
+ return true;
+ }
+ if j == text.len() {
+ return false;
+ }
+ j += 1;
+ }
+ return false;
+ }
+ if j >= text.len() || pattern[i] != text[j] {
+ return false;
+ }
+ i += 1;
+ j += 1;
}
- functions.push(json!({ "filter": kind_filter, "weight": 20.0 }));
- }
- if field_lookup_ok(mapped, numeric, "code") {
- functions.push(json!({
- "filter": { "bool": { "should": [
- { "match": { "code": "pub" } },
- { "match": { "code": "public" } },
- { "match": { "code": "export" } }
- ], "minimum_should_match": 1 } },
- "weight": 5.0
- }));
+ j == text.len()
}
- if functions.is_empty() {
- return bool_query;
- }
- json!({
- "function_score": {
- "query": bool_query,
- "functions": functions,
- "score_mode": "sum",
- "boost_mode": "sum"
+ rec(pattern.as_bytes(), text.as_bytes())
+}
+
+fn path_is_demoted(path: &str) -> bool {
+ DEMOTED_PATH_GLOBS.iter().any(|glob| glob_match(glob, path))
+}
+
+/// The old `function_score` filter matched the tokens `pub`, `public`, and
+/// `export` on `code`. Split on non-letters so `publication` does not count
+/// and `pub(crate)` does.
+fn declaration_is_public(code: &str) -> bool {
+ code.split(|c: char| !c.is_ascii_alphanumeric())
+ .any(|word| matches!(word, "pub" | "public" | "export"))
+}
+
+/// Add the kind and visibility weights to hits the bool query already
+/// returned, then sort. A preferred kind scores +20 unless the path is a
+/// test, example, fuzz target, or bench. A public declaration scores +5.
+/// Ties keep the engine order.
+pub(crate) fn rerank_def_hits(hits: &mut [Value]) {
+ const KIND_WEIGHT: f64 = 20.0;
+ const PUB_WEIGHT: f64 = 5.0;
+ let adjusted: Vec<f64> = hits
+ .iter()
+ .map(|hit| {
+ let base = hit.get("_score").and_then(Value::as_f64).unwrap_or(0.0);
+ let src = hit.get("_source").unwrap_or(&Value::Null);
+ let mut bonus = 0.0;
+ let kind = src.get("kind").and_then(Value::as_str).unwrap_or("");
+ if PREFERRED_KINDS.contains(&kind) {
+ let path = src
+ .get("ax_path")
+ .or_else(|| src.get("path"))
+ .and_then(Value::as_str)
+ .unwrap_or("");
+ if !path_is_demoted(path) {
+ bonus += KIND_WEIGHT;
+ }
+ }
+ if src
+ .get("code")
+ .and_then(Value::as_str)
+ .is_some_and(declaration_is_public)
+ {
+ bonus += PUB_WEIGHT;
+ }
+ base + bonus
+ })
+ .collect();
+ let mut order: Vec<usize> = (0..hits.len()).collect();
+ order.sort_by(|&a, &b| {
+ adjusted[b]
+ .partial_cmp(&adjusted[a])
+ .unwrap_or(std::cmp::Ordering::Equal)
+ .then(a.cmp(&b))
+ });
+ let original = hits.to_vec();
+ for (place, old) in order.into_iter().enumerate() {
+ let mut hit = original[old].clone();
+ if let Some(obj) = hit.as_object_mut() {
+ obj.insert("_score".into(), json!(adjusted[old]));
}
- })
+ hits[place] = hit;
+ }
}
/// Entry point for the `def` subcommand. Returns a process exit code.
@@ -413,7 +468,7 @@ pub fn run_def_cli() -> i32 {
eprintln!("xerj def: --lang ignored; no index under this prefix maps `language`");
}
}
- let body = def_query(
+ let mut body = def_query(
&query,
kind.as_deref(),
lang.as_deref(),
@@ -421,6 +476,9 @@ pub fn run_def_cli() -> i32 {
mapped.as_ref(),
numeric.as_ref(),
);
+ // Fetch a window wider than `-k`, then rerank locally. The engine's
+ // function_score path scored every match (422ms vs 6ms on tokio).
+ body["size"] = json!(def_collect_size(k));
let mut resp = match es.search(&pattern, &body) {
Ok(v) => v,
Err(e) => {
@@ -433,14 +491,19 @@ pub fn run_def_cli() -> i32 {
}
};
- let hits = unique_locations(
- resp.pointer("/hits/hits")
- .and_then(Value::as_array)
- .cloned()
- .unwrap_or_default(),
- );
- if let Some(arr) = resp.pointer_mut("/hits/hits") {
- *arr = Value::Array(hits.clone());
+ let mut hits = resp
+ .pointer("/hits/hits")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ // Rerank the wide window locally, then collapse duplicate locations and
+ // trim to `-k`. Dedupe after rerank so the best-ranked hit at each
+ // path:line is the one kept.
+ rerank_def_hits(&mut hits);
+ let mut hits = unique_locations(hits);
+ hits.truncate(k);
+ if let Some(slot) = resp.pointer_mut("/hits/hits") {
+ *slot = Value::Array(hits.clone());
}
if as_json {
@@ -558,12 +621,7 @@ mod tests {
(None, None),
] {
let body = def_query("zzqq_nonexistent", kind, lang, 5, None, None);
- // Kind/pub weights wrap the bool; the filter still has to sit on
- // that bool, not replace the name clauses.
- let b = body
- .pointer("/query/function_score/query/bool")
- .or_else(|| body.pointer("/query/bool"))
- .expect("bool query");
+ let b = body.pointer("/query/bool").expect("bool query");
assert_eq!(b["minimum_should_match"], 1, "{body}");
assert!(!b["should"].as_array().unwrap().is_empty());
let filters = b["filter"].as_array().unwrap().len();
@@ -633,44 +691,80 @@ mod tests {
);
}
- /// A struct/function outranks a type alias and a `mod` line of the same
- /// name, and a `pub` declaration outranks a private one. The weight is a
- /// function_score filter, not a `should`, so it cannot match every struct.
+ /// A struct/function outranks a type alias and a test double of the same
+ /// name, and a `pub` declaration outranks a private one. The weights are
+ /// applied to the hits already returned — not as a `function_score`,
+ /// which scored every match (422ms vs 6ms on the tokio index).
#[test]
fn preferred_kinds_and_pub_are_weighted_and_aliases_are_demoted() {
let body = def_query("Db", None, None, 5, None, None);
let rendered = body["query"].to_string();
- assert!(rendered.contains("function_score"), "{body}");
assert!(
- rendered.contains("\"weight\":20.0") || rendered.contains("\"weight\":20"),
- "{body}"
+ !rendered.contains("function_score"),
+ "the slow wrapper must stay out of the request: {body}"
);
- for kind in ["struct", "function", "enum", "trait"] {
- assert!(
- rendered.contains(&format!("\"{kind}\"")),
- "{kind} missing: {body}"
- );
- }
- // A type alias is not a preferred kind, so it does not receive the
- // weight. Path globs are not part of the historical query.
- let kinds = body.pointer("/query/function_score/functions/0/filter/bool/must/0/terms/kind");
assert!(
- kinds
- .and_then(Value::as_array)
- .unwrap()
- .iter()
- .all(|k| k.as_str() != Some("type") && k.as_str() != Some("module")),
- "{kinds:?}"
+ !rendered.contains("ax_path"),
+ "path demotion is local, not a wildcard filter: {rendered}"
);
- assert!(!rendered.contains("ax_path"), "{body}");
- let with_path = names_of(&["name", "defs", "code", "title", "kind", "ax_path"]);
- let body = def_query("Db", None, None, 5, Some(&with_path), None);
- let rendered = body["query"].to_string();
- assert!(rendered.contains("*/tests/*"), "{body}");
- assert!(rendered.contains("*/examples/*"), "{body}");
- assert!(rendered.contains("*/fuzz/*"), "{body}");
- assert!(rendered.contains("*/benches/*"), "{body}");
- assert!(rendered.contains("must_not"), "{body}");
+ assert_eq!(def_collect_size(5), 45);
+ assert_eq!(def_collect_size(180), 200);
+ assert_eq!(def_collect_size(500), 500);
+
+ let test_struct = json!({
+ "_score": 172.15,
+ "_source": {
+ "ax_path": "tokio/src/runtime/tests/task.rs",
+ "name": "Runtime",
+ "kind": "struct",
+ "line": 384,
+ "code": "#[derive(Clone)]\nstruct Runtime"
+ }
+ });
+ let src_struct = json!({
+ "_score": 170.17,
+ "_source": {
+ "ax_path": "tokio/src/runtime/runtime.rs",
+ "name": "Runtime",
+ "kind": "struct",
+ "line": 97,
+ "code": "#[derive(Debug)]\npub struct Runtime"
+ }
+ });
+ let alias = json!({
+ "_score": 170.17,
+ "_source": {
+ "ax_path": "tokio/src/runtime/runtime.rs",
+ "name": "Runtime",
+ "kind": "type",
+ "line": 12,
+ "code": "pub type Runtime = Handle"
+ }
+ });
+ let mut hits = vec![test_struct, alias.clone(), src_struct];
+ rerank_def_hits(&mut hits);
+ assert_eq!(
+ hits[0]["_source"]["ax_path"],
+ "tokio/src/runtime/runtime.rs"
+ );
+ assert_eq!(hits[0]["_source"]["kind"], "struct");
+ assert_eq!(hits[1]["_source"]["kind"], "type");
+ assert!(
+ hits[0]["_score"].as_f64().unwrap() > hits[2]["_score"].as_f64().unwrap(),
+ "src struct must outrank the higher-BM25 test double: {hits:?}"
+ );
+ // The type alias shares the src path but is not a preferred kind.
+ // It still gets the pub weight, so it sorts above the private test
+ // struct and below the pub struct.
+ assert!(hits[1]["_score"].as_f64().unwrap() > hits[2]["_score"].as_f64().unwrap());
+
+ assert!(path_is_demoted("tokio/src/runtime/tests/task.rs"));
+ assert!(path_is_demoted("examples/dump.rs"));
+ assert!(path_is_demoted("fuzz/fuzz_targets/x.rs") || path_is_demoted("tokio/fuzz/a.rs"));
+ assert!(!path_is_demoted("tokio/src/runtime/runtime.rs"));
+ assert!(!declaration_is_public("struct Runtime"));
+ assert!(declaration_is_public("pub(crate) struct Runtime"));
+ assert!(!declaration_is_public("publication"));
}
/// "flush the tree to disk" has to search the symbol body. An identifier
@@ -679,7 +773,7 @@ mod tests {
fn a_phrase_searches_impl_only_when_the_mapping_has_it() {
let code = names_of(&["name", "defs", "code", "title", "impl", "kind"]);
let phrase = def_query("flush the tree to disk", None, None, 5, Some(&code), None);
- let fields = phrase["query"]["function_score"]["query"]["bool"]["should"]
+ let fields = phrase["query"]["bool"]["should"]
.as_array()
.unwrap()
.iter()
@@ -751,11 +845,10 @@ mod tests {
assert!(!query.contains("kind"), "{query}");
assert!(query.contains("code"), "{query}");
assert!(query.contains("title"), "{query}");
- // `code` is still text, so the pub ranking weight wraps function_score.
- // The numeric `kind` must not: that filter would name `kind`.
+ // `code` is still text. The numeric `kind` must not be named: that
+ // filter would 400 the wildcard. The pub weight is applied locally.
let should = body
- .pointer("/query/function_score/query/bool/should")
- .or_else(|| body.pointer("/query/bool/should"))
+ .pointer("/query/bool/should")
.and_then(Value::as_array)
.expect("should clauses");
let fields = should