Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(search): fill _passage on the lexical path — query-term-densest line-snapped window (#174)

xerj-orgd1933e8fa510
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 1ac656576..b83b9a2ab 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -17451,6 +17451,95 @@ mod passage_scroll_tests {
}
}
+ /// #174: a LEXICAL query that requests `fields: ["_passage"]` used to be
+ /// a silent no-op — `_passage` was populated only by the kNN/semantic
+ /// executors, so a caller slicing a large `body` got the licence banner
+ /// and `#include` block instead of the code that made the file match.
+ /// The lexical path must return the query-term-densest line-snapped
+ /// window with exact byte offsets into the original field.
+ #[tokio::test]
+ async fn lexical_multi_match_fills_passage_with_exact_offsets() {
+ let state = test_state();
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("title", FieldType::Text));
+ state
+ .engine
+ .create_index("lexical-passages", schema)
+ .unwrap();
+ let index = state.engine.get_index("lexical-passages").unwrap();
+ // A "large source file": banner and includes at the head, the
+ // definition that makes the file the right answer buried ~7 KB deep.
+ let source_body = format!(
+ "/* licence banner */\n{}void addReplyNull(client *c) {{\n addReplyProto(c, \"$-1\\r\\n\", 5);\n}}\n{}",
+ "#include <deps.h>\n".repeat(400),
+ "static void trailer(void) {}\n".repeat(200),
+ );
+ index
+ .index_document(
+ Some("networking.c".into()),
+ json!({"body": source_body, "title": "networking", "page": 2}),
+ )
+ .await
+ .unwrap();
+ index.refresh().await.unwrap();
+ let app = crate::router::build_es_compat_router(state);
+
+ let response = app
+ .oneshot(
+ Request::post("/lexical-passages/_search")
+ .header("content-type", "application/json")
+ .body(Body::from(
+ json!({
+ "size": 1,
+ "fields": ["_passage"],
+ "query": {"multi_match": {
+ "query": "addReplyNull null bulk string reply",
+ "fields": ["body", "title"]
+ }}
+ })
+ .to_string(),
+ ))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::OK);
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let resp: Value = serde_json::from_slice(&bytes).unwrap();
+ let passage = &resp["hits"]["hits"][0]["fields"]["_passage"][0];
+ assert!(
+ passage.is_object(),
+ "lexical `_passage` must not be a silent no-op: {resp}"
+ );
+ assert_eq!(passage["field"], "body");
+ assert_eq!(passage["page"], 2);
+ let start = passage["start_offset"].as_u64().unwrap() as usize;
+ let end = passage["end_offset"].as_u64().unwrap() as usize;
+ let text = passage["text"].as_str().unwrap();
+ assert_eq!(
+ &source_body[start..end],
+ text,
+ "offsets must slice the original field exactly"
+ );
+ assert!(
+ text.contains("addReplyNull"),
+ "the passage must carry the code that made the file rank, got: {text:?}"
+ );
+ assert!(start > 0, "the head-of-file slice is exactly the #174 bug");
+ assert_eq!(
+ passage["ordinal"].as_u64().unwrap() as usize,
+ source_body[..start].matches('\n').count(),
+ "ordinal = zero-based line index of the window start"
+ );
+ }
+
async fn assert_ambiguous_passage_query_is_actionable_400(app: &axum::Router, body: Value) {
let response = app
.clone()
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index edabbbbb1..c733b0a71 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -16635,6 +16635,23 @@ impl Index {
page
};
+ // --- Fill `_passage` for lexical queries (issue #174) ---
+ //
+ // The `_passage` pseudo-field used to be populated only by the
+ // kNN/semantic executors; a lexical query that requested it got a
+ // silent no-op, so callers slicing a large `body` had no way to know
+ // WHERE in the file the match lived. Like highlighting, this is a
+ // post-pass over the returned page only (never the full candidate
+ // set), reads the already-materialised `hit.source`, and must run
+ // BEFORE `apply_source_filter` for the same reason highlighting does:
+ // the passage resolves against the stored document, independent of
+ // the `_source` projection.
+ let page = if passage_requested(request) {
+ apply_lexical_passages(page, query)
+ } else {
+ page
+ };
+
// --- Apply _source filtering ---
let page = apply_source_filter(page, &request.source);
@@ -25575,10 +25592,7 @@ fn passage_match_from_source(
{
return None;
}
- let page = get_field_value(source, "page")
- .or_else(|| get_field_value(source, "page_number"))
- .or_else(|| get_field_value(source, "metadata.page"))
- .and_then(|value| value.as_u64());
+ let page = passage_page_from_source(source);
Some(PassageMatch {
field: field.to_string(),
ordinal,
@@ -25589,6 +25603,424 @@ fn passage_match_from_source(
})
}
+/// Autoindex PDF page identity, shared by the semantic and lexical passage
+/// builders so both wire the same `page` provenance.
+fn passage_page_from_source(source: &Value) -> Option<u64> {
+ get_field_value(source, "page")
+ .or_else(|| get_field_value(source, "page_number"))
+ .or_else(|| get_field_value(source, "metadata.page"))
+ .and_then(|value| value.as_u64())
+}
+
+/// Maximum byte length of a query-time lexical `_passage` window.
+///
+/// Sized so an agent gets enough surrounding code to read an implementation
+/// (~50 lines of typical source), not just a keyword-in-context fragment —
+/// that distinction is the whole point of `_passage` vs `highlight` (#174).
+const LEXICAL_PASSAGE_MAX_BYTES: usize = 2048;
+
+/// Collect the field names (with any `^boost` suffix still attached) that the
+/// query's TEXT clauses target — the same clause set whose tokens
+/// `collect_highlight_terms` extracts, so passage field targeting and passage
+/// term extraction can never disagree about which clauses count.
+fn collect_passage_query_fields(query: &QueryNode, out: &mut Vec<String>) {
+ match query {
+ QueryNode::Match { field, .. } | QueryNode::MatchPhrase { field, .. } => {
+ out.push(field.clone());
+ }
+ QueryNode::MultiMatch { fields, .. } => {
+ if fields.is_empty() {
+ // ES defaults an empty multi_match field list to all fields.
+ out.push("*".to_string());
+ } else {
+ out.extend(fields.iter().cloned());
+ }
+ }
+ QueryNode::QueryString { default_field, .. } => {
+ out.push(default_field.clone().unwrap_or_else(|| "*".to_string()));
+ }
+ QueryNode::Term { field, .. } => out.push(field.clone()),
+ QueryNode::Bool {
+ must,
+ should,
+ filter,
+ ..
+ } => {
+ for q in must.iter().chain(should.iter()).chain(filter.iter()) {
+ collect_passage_query_fields(q, out);
+ }
+ }
+ QueryNode::Boosted { query, .. }
+ | QueryNode::Constant { query, .. }
+ | QueryNode::FunctionScore { query, .. } => {
+ collect_passage_query_fields(query, out);
+ }
+ _ => {}
+ }
+}
+
+/// Resolve one query field spec against a hit's `_source` into concrete
+/// `(dotted_field_name, text)` pairs.
+///
+/// - A plain name resolves via `get_field_value` (dot paths supported).
+/// - `*` / `prefix*` walk the source tree and match the flattened dotted
+/// path, mirroring `match_any_field_wildcard`'s prefix semantics.
+/// - Only string leaves qualify; internal `__xerj_passage_meta__*` keys are
+/// never passage sources. Multi-valued (array) fields are skipped: a byte
+/// offset into "the field" is ambiguous across elements.
+fn collect_passage_field_texts(source: &Value, field_spec: &str) -> Vec<(String, String)> {
+ let (name, _boost) = parse_field_boost(field_spec);
+ if !name.contains('*') {
+ if name.starts_with(PASSAGE_METADATA_PREFIX) {
+ return Vec::new();
+ }
+ return match get_field_value(source, name) {
+ Some(Value::String(s)) => vec![(name.to_string(), s)],
+ _ => Vec::new(),
+ };
+ }
+ let prefix = name.strip_suffix('*').unwrap_or(name);
+ let mut out = Vec::new();
+ fn walk(value: &Value, path: &str, prefix: &str, out: &mut Vec<(String, String)>) {
+ match value {
+ Value::Object(obj) => {
+ for (key, val) in obj {
+ if key.starts_with(PASSAGE_METADATA_PREFIX) {
+ continue;
+ }
+ let dotted = if path.is_empty() {
+ key.clone()
+ } else {
+ format!("{path}.{key}")
+ };
+ walk(val, &dotted, prefix, out);
+ }
+ }
+ Value::String(s) if path.starts_with(prefix) => {
+ out.push((path.to_string(), s.clone()));
+ }
+ _ => {}
+ }
+ }
+ walk(source, "", prefix, &mut out);
+ out
+}
+
+/// Count non-overlapping occurrences of `needle` in `hay` (both lowercase).
+fn count_term_occurrences(hay: &str, needle: &str) -> usize {
+ if needle.is_empty() {
+ return 0;
+ }
+ let mut count = 0;
+ let mut from = 0;
+ while let Some(pos) = hay[from..].find(needle) {
+ count += 1;
+ from = from + pos + needle.len();
+ }
+ count
+}
+
+/// Pick the query-term-densest window of a text, snapped to line boundaries.
+///
+/// Returns `(score, start_byte, end_byte, first_line_index)` for the best
+/// window of at most [`LEXICAL_PASSAGE_MAX_BYTES`], or `None` when no query
+/// term occurs in the text. `text_lower`/`lower_map` come from
+/// [`build_lower_offset_map`] (built once per field by the caller);
+/// `weighted_terms` carries a per-term weight computed by the caller.
+///
+/// Adapted from tantivy's snippet generator (MIT,
+/// `tantivy/src/snippet/mod.rs:205-260`): candidate windows bounded by a byte
+/// budget, scored by summing per-term weights, best window wins, ties keep
+/// the earliest. Two deliberate departures:
+///
+/// - tantivy greedily segments at token offsets, which can split the optimum
+/// across a boundary and slice mid-line. We snap candidates to LINE
+/// boundaries (the natural passage unit for source code — the driving use
+/// case of #174) and slide the window over lines, so every line-snapped
+/// window within budget is considered in O(lines).
+/// - the winning window is reduced to its tight core (first..last scored
+/// line) and then re-expanded alternately before/after, so the matches sit
+/// roughly centred in the returned passage instead of at its tail.
+///
+/// A single line longer than the whole budget (minified JS, single-line
+/// JSON) cannot be line-snapped; it degrades to a char-boundary-snapped
+/// window inside that line, centred on its first term occurrence, rescored
+/// over what the trimmed window actually contains.
+fn select_lexical_passage(
+ text: &str,
+ text_lower: &str,
+ lower_map: &[(usize, usize, usize)],
+ weighted_terms: &[(&str, f32)],
+) -> Option<(f32, usize, usize, u32)> {
+ if text.is_empty() {
+ return None;
+ }
+ let orig_len = text.len();
+
+ // All term occurrences as (start_byte_in_original, weight).
+ let mut occurrences: Vec<(usize, f32)> = Vec::new();
+ for &(term, weight) in weighted_terms {
+ if term.is_empty() {
+ continue;
+ }
+ let mut from = 0;
+ while let Some(pos) = text_lower[from..].find(term) {
+ let abs = from + pos;
+ let (oa, ob) = lower_span_to_orig(lower_map, orig_len, abs, abs + term.len());
+ if ob > oa {
+ occurrences.push((oa, weight));
+ }
+ from = abs + term.len();
+ }
+ }
+ if occurrences.is_empty() {
+ return None;
+ }
+ occurrences.sort_unstable_by_key(|&(start, _)| start);
+
+ // Line spans as [start, end) byte ranges EXCLUDING the trailing newline;
+ // window length between lines still counts interior newlines because it
+ // is measured end-to-start.
+ let bytes = text.as_bytes();
+ let mut lines: Vec<(usize, usize)> = Vec::new();
+ let mut line_start = 0;
+ for (i, &b) in bytes.iter().enumerate() {
+ if b == b'\n' {
+ lines.push((line_start, i));
+ line_start = i + 1;
+ }
+ }
+ if line_start < orig_len {
+ lines.push((line_start, orig_len));
+ }
+ if lines.is_empty() {
+ return None;
+ }
+
+ // Per-line score: sum of weights of occurrences starting in the line
+ // (occurrences are sorted, so one forward pointer suffices).
+ let mut line_scores = vec![0.0f32; lines.len()];
+ let mut occ_idx = 0;
+ for (li, &(ls, _)) in lines.iter().enumerate() {
+ let next_start = lines.get(li + 1).map_or(usize::MAX, |&(s, _)| s);
+ while occ_idx < occurrences.len() && occurrences[occ_idx].0 < next_start {
+ if occurrences[occ_idx].0 >= ls {
+ line_scores[li] += occurrences[occ_idx].1;
+ }
+ occ_idx += 1;
+ }
+ }
+
+ // Slide a line-snapped window under the byte budget. Line scores are
+ // non-negative, so for each end line the max-score window is the widest
+ // one that fits — exactly what the two-pointer maintains. Strictly
+ // greater keeps the earliest window on score ties (same rule as
+ // `choose_passage_winner`). Best is recorded as the window's TIGHT CORE
+ // (first..last scored line) so the expansion below can centre the budget
+ // around the matches instead of arriving with maximal leading filler.
+ enum Cand {
+ /// Tight core as inclusive line indices.
+ Lines(usize, usize),
+ /// Char-snapped byte window inside one oversized line (+ its line).
+ Bytes(usize, usize, usize),
+ }
+ let mut best: Option<(f32, Cand)> = None;
+ let mut lo = 0;
+ let mut acc = 0.0f32;
+ for hi in 0..lines.len() {
+ acc += line_scores[hi];
+ while lo < hi && lines[hi].1 - lines[lo].0 > LEXICAL_PASSAGE_MAX_BYTES {
+ acc -= line_scores[lo];
+ lo += 1;
+ }
+ if lines[hi].1 - lines[lo].0 <= LEXICAL_PASSAGE_MAX_BYTES {
+ if acc > 0.0 && best.as_ref().is_none_or(|&(s, _)| acc > s) {
+ // Tight core: zero-score head/tail lines carry no signal.
+ let tlo = (lo..=hi).find(|&i| line_scores[i] > 0.0).unwrap_or(lo);
+ let thi = (lo..=hi)
+ .rev()
+ .find(|&i| line_scores[i] > 0.0)
+ .unwrap_or(hi);
+ best = Some((acc, Cand::Lines(tlo, thi)));
+ }
+ } else {
+ // lo == hi and this single line alone exceeds the budget: take a
+ // char-boundary window inside it around its first occurrence.
+ debug_assert_eq!(lo, hi);
+ if line_scores[hi] > 0.0 {
+ let (ls, le) = lines[hi];
+ let anchor = occurrences
+ .iter()
+ .map(|&(s, _)| s)
+ .find(|&s| s >= ls && s < le)
+ .unwrap_or(ls);
+ let half = LEXICAL_PASSAGE_MAX_BYTES / 2;
+ let w_start0 = anchor.saturating_sub(half).max(ls);
+ let mut w_end = (w_start0 + LEXICAL_PASSAGE_MAX_BYTES).min(le);
+ let mut w_start = w_end.saturating_sub(LEXICAL_PASSAGE_MAX_BYTES).max(ls);
+ while w_start < le && !text.is_char_boundary(w_start) {
+ w_start += 1;
+ }
+ while w_end > w_start && !text.is_char_boundary(w_end) {
+ w_end -= 1;
+ }
+ if w_end > w_start {
+ // Rescore over what the trimmed window actually holds.
+ let trimmed: f32 = occurrences
+ .iter()
+ .filter(|&&(s, _)| s >= w_start && s < w_end)
+ .map(|&(_, w)| w)
+ .sum();
+ if trimmed > 0.0 && best.as_ref().is_none_or(|&(s, _)| trimmed > s) {
+ best = Some((trimmed, Cand::Bytes(w_start, w_end, hi)));
+ }
+ }
+ }
+ // No window ending at a later line may include this line.
+ acc = 0.0;
+ lo = hi + 1;
+ }
+ }
+
+ match best? {
+ (score, Cand::Bytes(start, end, line)) => Some((score, start, end, line as u32)),
+ (score, Cand::Lines(core_lo, core_hi)) => {
+ // Spend the remaining budget on context, alternating a line
+ // before / a line after the core so the matches end up roughly
+ // centred — "enough surrounding code to read an implementation"
+ // is the point of `_passage` over `highlight` (#174).
+ let mut w_lo = core_lo;
+ let mut w_hi = core_hi;
+ let mut before = true;
+ loop {
+ let fits_before =
+ w_lo > 0 && lines[w_hi].1 - lines[w_lo - 1].0 <= LEXICAL_PASSAGE_MAX_BYTES;
+ let fits_after = w_hi + 1 < lines.len()
+ && lines[w_hi + 1].1 - lines[w_lo].0 <= LEXICAL_PASSAGE_MAX_BYTES;
+ match (fits_before, fits_after) {
+ (false, false) => break,
+ (true, false) => w_lo -= 1,
+ (false, true) => w_hi += 1,
+ (true, true) => {
+ if before {
+ w_lo -= 1;
+ } else {
+ w_hi += 1;
+ }
+ before = !before;
+ }
+ }
+ }
+ Some((score, lines[w_lo].0, lines[w_hi].1, w_lo as u32))
+ }
+ }
+}
+
+/// Post-page pass that fills `hit.passage` for lexical queries when the
+/// caller opted in via `fields: ["_passage"]` (#174).
+///
+/// Runs over the returned page only, against the already-materialised
+/// `hit.source` — no new stored bytes, no re-index. For each hit the best
+/// line-snapped window across every text field the query targets wins;
+/// `ordinal` carries the zero-based line index of the passage's first line
+/// (the lexical analogue of the semantic path's chunk ordinal — there is no
+/// ingest-time chunk sequence at query time, and "which line" is exactly
+/// what a caller slicing a large file needs).
+///
+/// Term weighting adapts tantivy's corpus-rarity snippet scoring
+/// (`tantivy/src/snippet/mod.rs:421`, `1/(1+doc_freq)`, MIT) to the one
+/// document in hand: each occurrence of a term weighs `1/(1+n)²` where `n`
+/// is the term's occurrence count across the hit's candidate fields. The
+/// SQUARE is a deliberate departure from tantivy: with `1/(1+n)` a window
+/// holding n occurrences of one ubiquitous term sums to `n/(1+n) → 1`,
+/// outbidding the lone occurrence of a distinctive term (weight ½); squared,
+/// a term's total possible contribution is `n/(1+n)² ≤ ¼`, so no swarm of
+/// "null" can beat the window that contains "addReplyNull". Weights are per
+/// DOCUMENT, not per field — per-field weights would let a two-line symbol
+/// list outbid the body window the caller actually wants, because its single
+/// occurrence of every term looks maximally rare.
+fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {
+ let terms = extract_highlight_terms(query);
+ if terms.is_empty() {
+ return hits;
+ }
+ let mut field_specs: Vec<String> = Vec::new();
+ collect_passage_query_fields(query, &mut field_specs);
+ // Dedup while preserving query order: on cross-field score ties the
+ // FIRST field the query listed wins — deterministic, and closer to
+ // caller intent than alphabetical order.
+ let mut seen_specs = HashSet::new();
+ field_specs.retain(|spec| seen_specs.insert(spec.clone()));
+ if field_specs.is_empty() {
+ return hits;
+ }
+ hits.into_iter()
+ .map(|mut hit| {
+ // The kNN/semantic executors fill `passage` from exact
+ // ingest-time chunk offsets; never overwrite that provenance.
+ if hit.passage.is_some() || hit.source.is_null() {
+ return hit;
+ }
+ // Resolve every candidate field ONCE, deduped by concrete field
+ // name (`body` and `body^2` are the same field), lowercasing
+ // each text a single time.
+ let mut seen_fields: HashSet<String> = HashSet::new();
+ let mut candidates: Vec<(String, String, String, Vec<(usize, usize, usize)>)> =
+ Vec::new();
+ for spec in &field_specs {
+ for (field_name, text) in collect_passage_field_texts(&hit.source, spec) {
+ if seen_fields.insert(field_name.clone()) {
+ let (lower, map) = build_lower_offset_map(&text);
+ candidates.push((field_name, text, lower, map));
+ }
+ }
+ }
+ if candidates.is_empty() {
+ return hit;
+ }
+ // Document-level term rarity across all candidate fields,
+ // squared so one term's total contribution stays bounded (see
+ // the weighting note in this function's doc comment).
+ let weighted_terms: Vec<(&str, f32)> = terms
+ .iter()
+ .map(|term| {
+ let total: usize = candidates
+ .iter()
+ .map(|(_, _, lower, _)| count_term_occurrences(lower, term))
+ .sum();
+ let denom = 1.0 + total as f32;
+ (term.as_str(), 1.0 / (denom * denom))
+ })
+ .collect();
+ let mut best: Option<(f32, PassageMatch)> = None;
+ for (field_name, text, lower, map) in &candidates {
+ if let Some((score, start, end, ordinal)) =
+ select_lexical_passage(text, lower, map, &weighted_terms)
+ {
+ if best.as_ref().is_none_or(|(s, _)| score > *s) {
+ best = Some((
+ score,
+ PassageMatch {
+ field: field_name.clone(),
+ ordinal,
+ start_offset: start as u64,
+ end_offset: end as u64,
+ text: text[start..end].to_string(),
+ page: None,
+ },
+ ));
+ }
+ }
+ }
+ if let Some((_, mut passage)) = best {
+ passage.page = passage_page_from_source(&hit.source);
+ hit.passage = Some(passage);
+ }
+ hit
+ })
+ .collect()
+}
+
/// Shared tail of every top-level kNN executor (brute force and HNSW):
/// rank by score desc, cap the candidate pool at `k`, window with
/// `from`/`size`, and shape the response. Centralised so the exact and
@@ -35399,6 +35831,232 @@ mod highlight_multibyte_tests {
}
}
+#[cfg(test)]
+mod lexical_passage_tests {
+ //! Query-time `_passage` selection for the lexical path (#174): the
+ //! query-term-densest window of the source, snapped to line boundaries,
+ //! with exact byte offsets so a caller can slice a large field.
+ use super::*;
+ use serde_json::json;
+
+ fn select(text: &str, weighted: &[(&str, f32)]) -> Option<(f32, usize, usize, u32)> {
+ let (lower, map) = build_lower_offset_map(text);
+ select_lexical_passage(text, &lower, &map, weighted)
+ }
+
+ /// The head of the file (licence banner, includes) must lose to the deep
+ /// window that actually contains the query terms — the exact failure in
+ /// #174 where head-slicing `body` returned zero relevant characters.
+ #[test]
+ fn deep_definition_beats_file_head() {
+ let mut text = String::new();
+ text.push_str("/* Copyright banner licence text */\n");
+ for i in 0..200 {
+ text.push_str(&format!("#include <header_{i}.h>\n"));
+ }
+ let def_line_idx = text.lines().count();
+ text.push_str("void addReplyNull(client *c) {\n");
+ text.push_str(" addReplyProto(c, \"$-1\\r\\n\", 5);\n");
+ text.push_str("}\n");
+ for _ in 0..100 {
+ text.push_str("static int unrelated_trailer(void) { return 0; }\n");
+ }
+ let weighted = [("addreplynull", 0.5f32), ("reply", 0.25f32)];
+ let (score, start, end, ordinal) = select(&text, &weighted).expect("a window");
+ assert!(score > 0.0);
+ let passage = &text[start..end];
+ assert!(
+ passage.contains("addReplyNull"),
+ "window must cover the definition, got: {passage:?}"
+ );
+ // Line-snapped: starts at a line start, ends at a line end.
+ assert!(start == 0 || text.as_bytes()[start - 1] == b'\n');
+ assert!(end == text.len() || text.as_bytes()[end] == b'\n');
+ assert!(end - start <= LEXICAL_PASSAGE_MAX_BYTES);
+ // `ordinal` is the zero-based line index of the window start; the
+ // window is budget-limited so it starts well past the banner and at
+ // or before the definition line.
+ assert!(ordinal > 0, "must not sit at the file head");
+ assert!((ordinal as usize) <= def_line_idx);
+ }
+
+ /// A lone occurrence of a rare, distinctive term must outweigh a region
+ /// dense in a term that appears everywhere (tantivy's rarity weighting
+ /// adapted to one document).
+ #[test]
+ fn rare_term_outweighs_common_term_swarm() {
+ let mut text = String::new();
+ for _ in 0..40 {
+ text.push_str("null null null null common swamp line\n");
+ }
+ for _ in 0..80 {
+ text.push_str("padding line with nothing relevant at all\n");
+ }
+ text.push_str("the distinctive addreplynull definition lives here\n");
+ let null_count = count_term_occurrences(&text, "null");
+ let rare_count = count_term_occurrences(&text, "addreplynull");
+ assert_eq!(rare_count, 1);
+ // NOTE: "addreplynull" contains "null", so null_count includes it.
+ // Same squared-rarity formula as `apply_lexical_passages`.
+ let w = |n: usize| {
+ let denom = 1.0 + n as f32;
+ 1.0 / (denom * denom)
+ };
+ let weighted = [("addreplynull", w(rare_count)), ("null", w(null_count))];
+ let (_, start, end, _) = select(&text, &weighted).expect("a window");
+ assert!(
+ text[start..end].contains("distinctive"),
+ "rare-term window must win, got: {:?}",
+ &text[start..end]
+ );
+ }
+
+ /// A single line longer than the whole budget (minified JS) cannot be
+ /// line-snapped: the fallback takes a char-boundary window inside the
+ /// line, centred on the first term occurrence.
+ #[test]
+ fn oversized_single_line_falls_back_to_char_window() {
+ let mut text = "é".repeat(3000);
+ text.push_str("needle");
+ text.push_str(&"é".repeat(3000));
+ let weighted = [("needle", 0.5f32)];
+ let (_, start, end, ordinal) = select(&text, &weighted).expect("a window");
+ assert_eq!(ordinal, 0);
+ assert!(end - start <= LEXICAL_PASSAGE_MAX_BYTES);
+ assert!(text.is_char_boundary(start) && text.is_char_boundary(end));
+ assert!(text[start..end].contains("needle"));
+ }
+
+ #[test]
+ fn no_term_occurrence_returns_none() {
+ assert!(select("plain text without the words\n", &[("absent", 1.0)]).is_none());
+ assert!(select("", &[("absent", 1.0)]).is_none());
+ }
+
+ /// Exact slice contract: `start_offset`/`end_offset` must reproduce
+ /// `text` byte-for-byte when sliced from the original field.
+ #[test]
+ fn apply_fills_passage_with_exact_offsets_and_page() {
+ let body = format!(
+ "{}fn the_answer() {{\n compute_bulk_reply()\n}}\n{}",
+ "// filler line of no consequence\n".repeat(120),
+ "// trailing filler\n".repeat(120)
+ );
+ let query = QueryNode::MultiMatch {
+ fields: vec!["body".into(), "title".into()],
+ query: "the_answer bulk reply".into(),
+ match_type: Default::default(),
+ operator: None,
+ analyzer: None,
+ boost: None,
+ };
+ let hit = Hit {
+ id: "doc-1".into(),
+ score: 1.0,
+ source: json!({"body": body, "title": "unrelated", "page": 3}),
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ };
+ let out = apply_lexical_passages(vec![hit], &query);
+ let passage = out[0].passage.as_ref().expect("lexical passage filled");
+ assert_eq!(passage.field, "body");
+ assert_eq!(passage.page, Some(3));
+ let (s, e) = (passage.start_offset as usize, passage.end_offset as usize);
+ assert_eq!(&body[s..e], passage.text, "offsets must slice exactly");
+ assert!(passage.text.contains("the_answer"));
+ assert!(e - s <= LEXICAL_PASSAGE_MAX_BYTES);
+ // Line-snapped window with the match centred in surrounding context,
+ // and `ordinal` = zero-based line index of the window start.
+ assert!(
+ s == 0 || body.as_bytes()[s - 1] == b'\n',
+ "line-snapped start"
+ );
+ assert!(
+ e == body.len() || body.as_bytes()[e] == b'\n',
+ "line-snapped end"
+ );
+ assert_eq!(
+ passage.ordinal as usize,
+ body[..s].matches('\n').count(),
+ "ordinal = line index of window start"
+ );
+ assert!(passage.ordinal > 0, "window must not hug the file head");
+ }
+
+ /// A short symbol-list field must not capture the passage from the body:
+ /// term rarity is weighted per DOCUMENT, so `defs` containing each term
+ /// once does not look artificially rare next to `body`.
+ #[test]
+ fn short_symbol_field_does_not_capture_the_passage() {
+ let body = format!(
+ "{}void addReplyNull(client *c) {{\n addReply(c, shared.nullbulk);\n addReply(c, shared.bulk);\n}}\n{}",
+ "// preamble\n".repeat(50),
+ "// tail\n".repeat(50)
+ );
+ let query = QueryNode::MultiMatch {
+ fields: vec!["body".into(), "defs".into()],
+ query: "addReplyNull bulk".into(),
+ match_type: Default::default(),
+ operator: None,
+ analyzer: None,
+ boost: None,
+ };
+ let hit = Hit {
+ id: "doc-1".into(),
+ score: 1.0,
+ source: json!({"body": body, "defs": "addReplyNull bulk"}),
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ };
+ let out = apply_lexical_passages(vec![hit], &query);
+ let passage = out[0].passage.as_ref().expect("lexical passage filled");
+ assert_eq!(
+ passage.field, "body",
+ "body window holds more weighted occurrences than the symbol list"
+ );
+ }
+
+ /// A semantic-path passage (exact ingest-time chunk provenance) must
+ /// never be overwritten by the query-time selector.
+ #[test]
+ fn existing_semantic_passage_is_preserved() {
+ let query = QueryNode::Match {
+ field: "body".into(),
+ query: "needle".into(),
+ operator: Default::default(),
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ };
+ let semantic = PassageMatch {
+ field: "body".into(),
+ ordinal: 7,
+ start_offset: 10,
+ end_offset: 20,
+ text: "from-chunks".into(),
+ page: None,
+ };
+ let hit = Hit {
+ id: "doc-1".into(),
+ score: 1.0,
+ source: json!({"body": "needle here\n"}),
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: Some(semantic.clone()),
+ };
+ let out = apply_lexical_passages(vec![hit], &query);
+ assert_eq!(out[0].passage.as_ref(), Some(&semantic));
+ }
+}
+
#[cfg(test)]
mod knn_num_candidates_tests {
use super::*;
diff --git a/engine/crates/xerj-query/src/executor.rs b/engine/crates/xerj-query/src/executor.rs
index e85f72dfd..8c3b6290f 100644
--- a/engine/crates/xerj-query/src/executor.rs
+++ b/engine/crates/xerj-query/src/executor.rs
@@ -62,21 +62,29 @@ pub struct Hit {
/// Names of named queries that matched this document (only present when `_name` was used).
#[serde(skip_serializing_if = "Vec::is_empty")]
pub matched_queries: Vec<String>,
- /// Winning semantic passage, populated only when the caller explicitly
- /// requests the `_passage` pseudo-field.
+ /// Winning passage, populated only when the caller explicitly requests
+ /// the `_passage` pseudo-field.
///
- /// This is derived from compact ingest-time byte offsets. It is not
- /// persisted as a second text copy and is absent from ordinary responses.
+ /// Semantic/kNN hits derive it from compact ingest-time byte offsets;
+ /// lexical hits compute the query-term-densest line-snapped window over
+ /// the returned page at query time. Either way it is not persisted as a
+ /// second text copy and is absent from ordinary responses.
#[serde(rename = "_passage", skip_serializing_if = "Option::is_none", default)]
pub passage: Option<PassageMatch>,
}
-/// Provenance for the passage that supplied a semantic hit's max score.
+/// Provenance for the passage that made a hit relevant: the chunk that
+/// supplied a semantic hit's max score, or the query-term-densest window of
+/// a lexical hit.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PassageMatch {
/// Original `semantic_text` field, rather than its derived vector field.
+ /// For lexical passages: the queried text field the window came from.
pub field: String,
- /// Zero-based ordinal in the deterministic ingest-time chunk sequence.
+ /// Semantic hits: zero-based ordinal in the deterministic ingest-time
+ /// chunk sequence. Lexical hits have no chunk sequence; here this is the
+ /// zero-based LINE index where the passage starts — the "which line of
+ /// the file" a caller slicing a large document needs.
pub ordinal: u32,
/// UTF-8 byte offsets into `field`.
pub start_offset: u64,