AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): per-hit _source mapping suppression on scroll continuation (#637)
xerj-org201475cb57c7
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 3c249d5ed..822f856aa 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -14631,21 +14631,28 @@ async fn search_impl(.min(sc_cfg.scroll_max_keep_alive_secs);let keep_alive = std::time::Duration::from_secs(keep_alive_secs);let now = Instant::now();- // #637: capture the FULL first-page `_source` suppression decision, not- // just request-level `_source: false`. search_impl's own render above- // also omits `_source` when `stored_fields` implies it (and the request- // left `_source` unspecified) or the mapping sets `_source.enabled:- // false`. Both are request/index-level (constant across the snapshot),- // so a single captured boolean keeps continuation pages in parity with- // this first page. The remaining per-hit null-source case is already- // handled directly in `scroll_page_response`. (The- // `/:index/_search_scroll` route's own first page does not suppress these- // cases, so its capture site is intentionally left as request-level only- // — matching ITS render.) Computed before the struct literal moves- // `scroll_index` into `ctx.index`.+ // #637: capture the REQUEST-LEVEL part of the first-page `_source`+ // suppression decision — `_source: false`, and `stored_fields` implying+ // suppression when the request left `_source` unspecified. These are+ // genuinely constant across the whole snapshot (index-independent), so a+ // single captured boolean is correct for them.+ //+ // The mapping `_source.enabled: false` case is deliberately NOT folded in+ // here: a scroll can span MULTIPLE indices with divergent `_source`+ // settings, and the first page suppresses that case PER HIT using each+ // hit's own index. Collapsing it to `scroll_index` (the first index)+ // would strip `_source` from every continuation hit of a source-ENABLED+ // index whenever the first index disabled source — a new first-page/+ // continuation split and silent `_source` loss on the export/reindex API+ // (found by the #658 correctness skeptic). So `scroll_page_response`+ // re-derives the mapping check per hit from `h.index` instead.+ //+ // The per-hit null-source case is likewise handled in+ // `scroll_page_response`. (The `/:index/_search_scroll` route's own first+ // page does not suppress any of these, so its capture site stays+ // request-level only — matching ITS render; see #659.)let scroll_source_disabled = matches!(body.source, Some(Value::Bool(false)))- || (suppress_source_for_stored && body.source.is_none())- || mapping_source_disabled(&state, &scroll_index);+ || (suppress_source_for_stored && body.source.is_none());let ctx = xerj_engine::engine::ScrollContext {index: scroll_index,hits: snapshot,@@ -20796,11 +20803,17 @@ async fn scroll_page_response(_ => None,}});- // #624: honor the scroll's opening `_source: false` —- // omit `_source` here (the sentinels have already been- // consumed into `inner_hits` above), matching the first- // page. inner_hits are emitted regardless.- if !source_disabled {+ // #624/#637: honor the first page's `_source` suppression+ // (the sentinels have already been consumed into+ // `inner_hits` above), matching search_impl exactly.+ // `source_disabled` carries the request-level constant+ // (`_source:false`, stored_fields-implied); the mapping+ // `_source.enabled:false` case is re-derived PER HIT from+ // this hit's own index, because a scroll can span indices+ // with divergent `_source` settings and the first page+ // suppresses that case per-hit (#658 skeptic). inner_hits+ // are emitted regardless.+ if !source_disabled && !mapping_source_disabled(state, &h.index) {o.insert("_source".to_string(), src);}if let Some(inner) = collapse_inner {diff --git a/engine/crates/xerj-api/tests/scroll_source_suppression_parity.rs b/engine/crates/xerj-api/tests/scroll_source_suppression_parity.rsindex d322f1f55..e41fa4130 100644--- a/engine/crates/xerj-api/tests/scroll_source_suppression_parity.rs+++ b/engine/crates/xerj-api/tests/scroll_source_suppression_parity.rs@@ -166,3 +166,97 @@ async fn scroll_continuation_omits_source_under_stored_fields() {stored_fields without _source (first-page parity), got: {hit}");}++/// Open a scroll and page through EVERY page, returning all hits across all pages.+async fn scroll_all_hits(app: &axum::Router, open_path: &str, open_body: Value) -> Vec<Value> {+ let (st, first) = call(app, "POST", open_path, open_body).await;+ assert_eq!(st, StatusCode::OK, "scroll open: {first}");+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();+ let mut all: Vec<Value> = first["hits"]["hits"]+ .as_array()+ .cloned()+ .unwrap_or_default();+ loop {+ let (st, page) = call(+ app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "5m", "scroll_id": sid}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "continuation: {page}");+ let hits = page["hits"]["hits"].as_array().cloned().unwrap_or_default();+ if hits.is_empty() {+ break;+ }+ all.extend(hits);+ }+ all+}++/// Multi-index scroll where the FIRST-resolved index disables `_source` and a+/// second index enables it. Suppression must be decided PER HIT from each hit's+/// own index — not from a single snapshot constant keyed on the first index —+/// or continuation pages silently drop `_source` from the source-ENABLED index+/// (a regression the #658 correctness skeptic caught: silent data-loss on the+/// export/reindex API). Every page, first and continuation, must agree with the+/// first page: `a_off` hits omit `_source`, `b_on` hits keep it.+#[tokio::test]+async fn scroll_multi_index_source_suppression_is_per_hit() {+ let (app, _dir) = app().await;+ seed(+ &app,+ "a_off",+ json!({"mappings": {+ "_source": {"enabled": false},+ "properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}+ }}),+ )+ .await;+ seed(+ &app,+ "b_on",+ json!({"mappings": {+ "properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}+ }}),+ )+ .await;++ // Small size so several continuation pages exist; sort v asc interleaves the+ // two indices across pages so continuation pages carry BOTH.+ let hits = scroll_all_hits(+ &app,+ "/a_off,b_on/_search?scroll=5m",+ json!({"query": {"match_all": {}}, "size": 3, "sort": [{"v": "asc"}]}),+ )+ .await;++ let mut a_seen = 0;+ let mut b_seen = 0;+ for h in &hits {+ match h["_index"].as_str() {+ Some("a_off") => {+ a_seen += 1;+ assert!(+ h.get("_source").is_none(),+ "#658: a_off (source-disabled) hit must omit _source on every page: {h}"+ );+ }+ Some("b_on") => {+ b_seen += 1;+ assert!(+ h.get("_source").and_then(Value::as_object).is_some(),+ "#658: b_on (source-ENABLED) hit must KEEP _source on every page, \+ including continuation — per-hit suppression, not a first-index \+ constant: {h}"+ );+ }+ other => panic!("unexpected _index {other:?} in {h}"),+ }+ }+ assert_eq!(+ a_seen, 4,+ "expected all 4 a_off hits across pages: {hits:?}"+ );+ assert_eq!(b_seen, 4, "expected all 4 b_on hits across pages: {hits:?}");+}