Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): scroll continuation honours `_source: false` from the opening request (#624)

xerj-orgbb5eb756ae7d
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 389954272..add4719db 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -14478,6 +14478,7 @@ async fn search_impl(
// first page's own `_seq_no`/`_version` emission above.
seq_no_primary_term: emit_seq_no,
version: emit_version,
+ source_disabled: matches!(body.source, Some(Value::Bool(false))),
created: now,
keep_alive,
expires_at: now + keep_alive,
@@ -20198,6 +20199,7 @@ pub async fn search_with_scroll(
page_size,
seq_no_primary_term: emit_seq_no,
version: emit_version,
+ source_disabled: matches!(body.source, Some(Value::Bool(false))),
created: now,
keep_alive,
expires_at: now + keep_alive,
@@ -20483,6 +20485,12 @@ async fn scroll_page_response(
// borrows `ctx.hits`.
let emit_seq_no = ctx.seq_no_primary_term;
let emit_version = ctx.version;
+ // #624: the opening request's `_source: false` governs the whole
+ // scroll — omit `_source` on continuation pages too (ES fixes source
+ // selection at open; the `_search?scroll=` first page already omits
+ // it via search_impl). inner_hits still render from the collapse
+ // group snapshot regardless.
+ let source_disabled = ctx.source_disabled;
// Detect whether the initial search sorted by a non-score key;
// in that case `max_score` must be null on scroll pages too.
@@ -20603,7 +20611,13 @@ async fn scroll_page_response(
_ => None,
}
});
- o.insert("_source".to_string(), src);
+ // #624: honor the scroll's opening `_source: false` —
+ // omit `_source` here (the sentinels have already been
+ // consumed into `inner_hits` above), matching the first
+ // page. inner_hits are emitted regardless.
+ if !source_disabled {
+ o.insert("_source".to_string(), src);
+ }
if let Some(inner) = collapse_inner {
o.insert("inner_hits".to_string(), inner);
}
@@ -20704,6 +20718,7 @@ mod passage_scroll_tests {
page_size: 1,
seq_no_primary_term: false,
version: false,
+ source_disabled: false,
created: now,
keep_alive: Duration::from_secs(60),
expires_at: now + Duration::from_secs(60),
@@ -20762,6 +20777,7 @@ mod passage_scroll_tests {
page_size: 1,
seq_no_primary_term: false,
version: false,
+ source_disabled: false,
created: now,
keep_alive: Duration::from_secs(60),
expires_at: now + Duration::from_secs(60),
@@ -20816,6 +20832,7 @@ mod passage_scroll_tests {
page_size: 1,
seq_no_primary_term: false,
version: false,
+ source_disabled: false,
created: now,
keep_alive: Duration::from_secs(60),
expires_at: now + Duration::from_secs(60),
diff --git a/engine/crates/xerj-api/tests/collapse_scroll_source_false_repro.rs b/engine/crates/xerj-api/tests/collapse_scroll_source_false_repro.rs
new file mode 100644
index 000000000..cc5b4e0e7
--- /dev/null
+++ b/engine/crates/xerj-api/tests/collapse_scroll_source_false_repro.rs
@@ -0,0 +1,119 @@
+//! Issue #624: a collapse `_search?scroll=` continuation under `_source: false`.
+//!
+//! Two facts, established by reproduction:
+//! 1. `inner_hits` DO render (the scroll snapshot is `merged_hits.clone()` taken
+//! BEFORE `_source` suppression, so the context keeps the sentinel-laden
+//! source) — #622's cautious "no inner_hits" scoping note was wrong.
+//! 2. The real bug: the continuation used to EMIT `_source` even though the
+//! opening request set `_source: false` — a divergence from ES and from the
+//! `_search?scroll=` first page (search_impl). Fixed by capturing
+//! `source_disabled` once at scroll-open (`ScrollContext`) and omitting
+//! `_source` on continuation pages, while still rendering `inner_hits`.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+#[tokio::test]
+async fn collapse_scroll_source_false_still_renders_inner_hits() {
+ let (app, _dir) = app().await;
+ let (st, _) = call(
+ &app,
+ "PUT",
+ "/docs",
+ json!({"mappings": {"properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK);
+ for (id, grp, v) in [
+ ("a0", "A", 10),
+ ("a1", "A", 11),
+ ("b0", "B", 20),
+ ("b1", "B", 21),
+ ] {
+ let (st, _) = call(
+ &app,
+ "POST",
+ &format!("/docs/_doc/{id}"),
+ json!({"grp": grp, "v": v}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::CREATED, "index {id}");
+ }
+ let _ = call(&app, "POST", "/docs/_refresh", Value::Null).await;
+
+ // Collapse scroll, size 1, sorted grp asc, WITH `_source: false`.
+ let (st, first) = call(
+ &app,
+ "POST",
+ "/docs/_search?scroll=5m",
+ json!({
+ "query": {"match_all": {}},
+ "size": 1,
+ "_source": false,
+ "sort": [{"grp": "asc"}],
+ "collapse": {"field": "grp", "inner_hits": {"name": "members", "size": 10, "sort": [{"v": "asc"}]}}
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "collapse scroll start: {first}");
+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();
+
+ // Continue to page 2 (group B).
+ let (st, page2) = call(
+ &app,
+ "POST",
+ "/_search/scroll",
+ json!({"scroll": "5m", "scroll_id": sid}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "continuation: {page2}");
+ let leader = &page2["hits"]["hits"][0];
+
+ // #624 finding: inner_hits DO render under _source:false (the scroll snapshot
+ // retains the sentinel-laden source), so the filed "no inner_hits" premise is
+ // wrong — assert they render (a guard for that good behavior).
+ assert!(
+ leader.pointer("/inner_hits/members/hits/hits").is_some(),
+ "#624: collapse scroll continuation renders inner_hits even under _source:false: {leader}"
+ );
+ // The REAL bug: under `_source: false` the continuation must OMIT `_source`
+ // entirely (ES does; the `_search?scroll=` first page does via search_impl).
+ // The scroll continuation currently emits it — this assertion fails before
+ // the fix.
+ assert!(
+ leader.get("_source").is_none(),
+ "#624: scroll continuation must omit _source under _source:false (ES parity), got: {leader}"
+ );
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 164f6a1f7..0ca7224fd 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -170,6 +170,12 @@ pub struct ScrollContext {
/// Whether to emit `_version` on every page. Same once-at-open semantics
/// as `seq_no_primary_term`.
pub version: bool,
+ /// Whether the opening request suppressed `_source` (`"_source": false`).
+ /// Captured once at open — like `seq_no_primary_term`/`version`, ES fixes
+ /// the `_source` selection at the scroll-opening request and applies it to
+ /// every continuation page — so the continuation render omits `_source`
+ /// when this is set, matching the `_search?scroll=` first page (#624).
+ pub source_disabled: bool,
pub created: Instant,
/// The keep-alive window last requested for this context. A
/// continuation without an explicit `scroll` parameter re-arms the
diff --git a/engine/crates/xerj-engine/tests/search_context_ttl.rs b/engine/crates/xerj-engine/tests/search_context_ttl.rs
index 8892538c8..c31553859 100644
--- a/engine/crates/xerj-engine/tests/search_context_ttl.rs
+++ b/engine/crates/xerj-engine/tests/search_context_ttl.rs
@@ -38,6 +38,7 @@ fn scroll_ctx(expires_in: i64) -> ScrollContext {
page_size: 10,
seq_no_primary_term: false,
version: false,
+ source_disabled: false,
created: now,
keep_alive,
expires_at,