Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): per-hit _source mapping suppression on scroll continuation (#637)

xerj-org201475cb57c7
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 3c249d5ed..822f856aa 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -14631,21 +14631,28 @@ async fn search_impl(
.min(sc_cfg.scroll_max_keep_alive_secs);
let keep_alive = std::time::Duration::from_secs(keep_alive_secs);
let now = Instant::now();
- // #637: capture the FULL first-page `_source` suppression decision, not
- // just request-level `_source: false`. search_impl's own render above
- // also omits `_source` when `stored_fields` implies it (and the request
- // left `_source` unspecified) or the mapping sets `_source.enabled:
- // false`. Both are request/index-level (constant across the snapshot),
- // so a single captured boolean keeps continuation pages in parity with
- // this first page. The remaining per-hit null-source case is already
- // handled directly in `scroll_page_response`. (The
- // `/:index/_search_scroll` route's own first page does not suppress these
- // cases, so its capture site is intentionally left as request-level only
- // — matching ITS render.) Computed before the struct literal moves
- // `scroll_index` into `ctx.index`.
+ // #637: capture the REQUEST-LEVEL part of the first-page `_source`
+ // suppression decision — `_source: false`, and `stored_fields` implying
+ // suppression when the request left `_source` unspecified. These are
+ // genuinely constant across the whole snapshot (index-independent), so a
+ // single captured boolean is correct for them.
+ //
+ // The mapping `_source.enabled: false` case is deliberately NOT folded in
+ // here: a scroll can span MULTIPLE indices with divergent `_source`
+ // settings, and the first page suppresses that case PER HIT using each
+ // hit's own index. Collapsing it to `scroll_index` (the first index)
+ // would strip `_source` from every continuation hit of a source-ENABLED
+ // index whenever the first index disabled source — a new first-page/
+ // continuation split and silent `_source` loss on the export/reindex API
+ // (found by the #658 correctness skeptic). So `scroll_page_response`
+ // re-derives the mapping check per hit from `h.index` instead.
+ //
+ // The per-hit null-source case is likewise handled in
+ // `scroll_page_response`. (The `/:index/_search_scroll` route's own first
+ // page does not suppress any of these, so its capture site stays
+ // request-level only — matching ITS render; see #659.)
let scroll_source_disabled = matches!(body.source, Some(Value::Bool(false)))
- || (suppress_source_for_stored && body.source.is_none())
- || mapping_source_disabled(&state, &scroll_index);
+ || (suppress_source_for_stored && body.source.is_none());
let ctx = xerj_engine::engine::ScrollContext {
index: scroll_index,
hits: snapshot,
@@ -20796,11 +20803,17 @@ async fn scroll_page_response(
_ => None,
}
});
- // #624: honor the scroll's opening `_source: false` —
- // omit `_source` here (the sentinels have already been
- // consumed into `inner_hits` above), matching the first
- // page. inner_hits are emitted regardless.
- if !source_disabled {
+ // #624/#637: honor the first page's `_source` suppression
+ // (the sentinels have already been consumed into
+ // `inner_hits` above), matching search_impl exactly.
+ // `source_disabled` carries the request-level constant
+ // (`_source:false`, stored_fields-implied); the mapping
+ // `_source.enabled:false` case is re-derived PER HIT from
+ // this hit's own index, because a scroll can span indices
+ // with divergent `_source` settings and the first page
+ // suppresses that case per-hit (#658 skeptic). inner_hits
+ // are emitted regardless.
+ if !source_disabled && !mapping_source_disabled(state, &h.index) {
o.insert("_source".to_string(), src);
}
if let Some(inner) = collapse_inner {
diff --git a/engine/crates/xerj-api/tests/scroll_source_suppression_parity.rs b/engine/crates/xerj-api/tests/scroll_source_suppression_parity.rs
index d322f1f55..e41fa4130 100644
--- a/engine/crates/xerj-api/tests/scroll_source_suppression_parity.rs
+++ b/engine/crates/xerj-api/tests/scroll_source_suppression_parity.rs
@@ -166,3 +166,97 @@ async fn scroll_continuation_omits_source_under_stored_fields() {
stored_fields without _source (first-page parity), got: {hit}"
);
}
+
+/// Open a scroll and page through EVERY page, returning all hits across all pages.
+async fn scroll_all_hits(app: &axum::Router, open_path: &str, open_body: Value) -> Vec<Value> {
+ let (st, first) = call(app, "POST", open_path, open_body).await;
+ assert_eq!(st, StatusCode::OK, "scroll open: {first}");
+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();
+ let mut all: Vec<Value> = first["hits"]["hits"]
+ .as_array()
+ .cloned()
+ .unwrap_or_default();
+ loop {
+ let (st, page) = call(
+ app,
+ "POST",
+ "/_search/scroll",
+ json!({"scroll": "5m", "scroll_id": sid}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "continuation: {page}");
+ let hits = page["hits"]["hits"].as_array().cloned().unwrap_or_default();
+ if hits.is_empty() {
+ break;
+ }
+ all.extend(hits);
+ }
+ all
+}
+
+/// Multi-index scroll where the FIRST-resolved index disables `_source` and a
+/// second index enables it. Suppression must be decided PER HIT from each hit's
+/// own index — not from a single snapshot constant keyed on the first index —
+/// or continuation pages silently drop `_source` from the source-ENABLED index
+/// (a regression the #658 correctness skeptic caught: silent data-loss on the
+/// export/reindex API). Every page, first and continuation, must agree with the
+/// first page: `a_off` hits omit `_source`, `b_on` hits keep it.
+#[tokio::test]
+async fn scroll_multi_index_source_suppression_is_per_hit() {
+ let (app, _dir) = app().await;
+ seed(
+ &app,
+ "a_off",
+ json!({"mappings": {
+ "_source": {"enabled": false},
+ "properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}
+ }}),
+ )
+ .await;
+ seed(
+ &app,
+ "b_on",
+ json!({"mappings": {
+ "properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}
+ }}),
+ )
+ .await;
+
+ // Small size so several continuation pages exist; sort v asc interleaves the
+ // two indices across pages so continuation pages carry BOTH.
+ let hits = scroll_all_hits(
+ &app,
+ "/a_off,b_on/_search?scroll=5m",
+ json!({"query": {"match_all": {}}, "size": 3, "sort": [{"v": "asc"}]}),
+ )
+ .await;
+
+ let mut a_seen = 0;
+ let mut b_seen = 0;
+ for h in &hits {
+ match h["_index"].as_str() {
+ Some("a_off") => {
+ a_seen += 1;
+ assert!(
+ h.get("_source").is_none(),
+ "#658: a_off (source-disabled) hit must omit _source on every page: {h}"
+ );
+ }
+ Some("b_on") => {
+ b_seen += 1;
+ assert!(
+ h.get("_source").and_then(Value::as_object).is_some(),
+ "#658: b_on (source-ENABLED) hit must KEEP _source on every page, \
+ including continuation — per-hit suppression, not a first-index \
+ constant: {h}"
+ );
+ }
+ other => panic!("unexpected _index {other:?} in {h}"),
+ }
+ }
+ assert_eq!(
+ a_seen, 4,
+ "expected all 4 a_off hits across pages: {hits:?}"
+ );
+ assert_eq!(b_seen, 4, "expected all 4 b_on hits across pages: {hits:?}");
+}