AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): scroll continuation honours `_source: false` from the opening request (#624)
xerj-orgbb5eb756ae7d
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 389954272..add4719db 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -14478,6 +14478,7 @@ async fn search_impl(// first page's own `_seq_no`/`_version` emission above.seq_no_primary_term: emit_seq_no,version: emit_version,+ source_disabled: matches!(body.source, Some(Value::Bool(false))),created: now,keep_alive,expires_at: now + keep_alive,@@ -20198,6 +20199,7 @@ pub async fn search_with_scroll(page_size,seq_no_primary_term: emit_seq_no,version: emit_version,+ source_disabled: matches!(body.source, Some(Value::Bool(false))),created: now,keep_alive,expires_at: now + keep_alive,@@ -20483,6 +20485,12 @@ async fn scroll_page_response(// borrows `ctx.hits`.let emit_seq_no = ctx.seq_no_primary_term;let emit_version = ctx.version;+ // #624: the opening request's `_source: false` governs the whole+ // scroll — omit `_source` on continuation pages too (ES fixes source+ // selection at open; the `_search?scroll=` first page already omits+ // it via search_impl). inner_hits still render from the collapse+ // group snapshot regardless.+ let source_disabled = ctx.source_disabled;// Detect whether the initial search sorted by a non-score key;// in that case `max_score` must be null on scroll pages too.@@ -20603,7 +20611,13 @@ async fn scroll_page_response(_ => None,}});- o.insert("_source".to_string(), src);+ // #624: honor the scroll's opening `_source: false` —+ // omit `_source` here (the sentinels have already been+ // consumed into `inner_hits` above), matching the first+ // page. inner_hits are emitted regardless.+ if !source_disabled {+ o.insert("_source".to_string(), src);+ }if let Some(inner) = collapse_inner {o.insert("inner_hits".to_string(), inner);}@@ -20704,6 +20718,7 @@ mod passage_scroll_tests {page_size: 1,seq_no_primary_term: false,version: false,+ source_disabled: false,created: now,keep_alive: Duration::from_secs(60),expires_at: now + Duration::from_secs(60),@@ -20762,6 +20777,7 @@ mod passage_scroll_tests {page_size: 1,seq_no_primary_term: false,version: false,+ source_disabled: false,created: now,keep_alive: Duration::from_secs(60),expires_at: now + Duration::from_secs(60),@@ -20816,6 +20832,7 @@ mod passage_scroll_tests {page_size: 1,seq_no_primary_term: false,version: false,+ source_disabled: false,created: now,keep_alive: Duration::from_secs(60),expires_at: now + Duration::from_secs(60),diff --git a/engine/crates/xerj-api/tests/collapse_scroll_source_false_repro.rs b/engine/crates/xerj-api/tests/collapse_scroll_source_false_repro.rsnew file mode 100644index 000000000..cc5b4e0e7--- /dev/null+++ b/engine/crates/xerj-api/tests/collapse_scroll_source_false_repro.rs@@ -0,0 +1,119 @@+//! Issue #624: a collapse `_search?scroll=` continuation under `_source: false`.+//!+//! Two facts, established by reproduction:+//! 1. `inner_hits` DO render (the scroll snapshot is `merged_hits.clone()` taken+//! BEFORE `_source` suppression, so the context keeps the sentinel-laden+//! source) — #622's cautious "no inner_hits" scoping note was wrong.+//! 2. The real bug: the continuation used to EMIT `_source` even though the+//! opening request set `_source: false` — a divergence from ES and from the+//! `_search?scroll=` first page (search_impl). Fixed by capturing+//! `source_disabled` once at scroll-open (`ScrollContext`) and omitting+//! `_source` on continuation pages, while still rendering `inner_hits`.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++#[tokio::test]+async fn collapse_scroll_source_false_still_renders_inner_hits() {+ let (app, _dir) = app().await;+ let (st, _) = call(+ &app,+ "PUT",+ "/docs",+ json!({"mappings": {"properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}}}),+ )+ .await;+ assert_eq!(st, StatusCode::OK);+ for (id, grp, v) in [+ ("a0", "A", 10),+ ("a1", "A", 11),+ ("b0", "B", 20),+ ("b1", "B", 21),+ ] {+ let (st, _) = call(+ &app,+ "POST",+ &format!("/docs/_doc/{id}"),+ json!({"grp": grp, "v": v}),+ )+ .await;+ assert_eq!(st, StatusCode::CREATED, "index {id}");+ }+ let _ = call(&app, "POST", "/docs/_refresh", Value::Null).await;++ // Collapse scroll, size 1, sorted grp asc, WITH `_source: false`.+ let (st, first) = call(+ &app,+ "POST",+ "/docs/_search?scroll=5m",+ json!({+ "query": {"match_all": {}},+ "size": 1,+ "_source": false,+ "sort": [{"grp": "asc"}],+ "collapse": {"field": "grp", "inner_hits": {"name": "members", "size": 10, "sort": [{"v": "asc"}]}}+ }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "collapse scroll start: {first}");+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();++ // Continue to page 2 (group B).+ let (st, page2) = call(+ &app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "5m", "scroll_id": sid}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "continuation: {page2}");+ let leader = &page2["hits"]["hits"][0];++ // #624 finding: inner_hits DO render under _source:false (the scroll snapshot+ // retains the sentinel-laden source), so the filed "no inner_hits" premise is+ // wrong — assert they render (a guard for that good behavior).+ assert!(+ leader.pointer("/inner_hits/members/hits/hits").is_some(),+ "#624: collapse scroll continuation renders inner_hits even under _source:false: {leader}"+ );+ // The REAL bug: under `_source: false` the continuation must OMIT `_source`+ // entirely (ES does; the `_search?scroll=` first page does via search_impl).+ // The scroll continuation currently emits it — this assertion fails before+ // the fix.+ assert!(+ leader.get("_source").is_none(),+ "#624: scroll continuation must omit _source under _source:false (ES parity), got: {leader}"+ );+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 164f6a1f7..0ca7224fd 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -170,6 +170,12 @@ pub struct ScrollContext {/// Whether to emit `_version` on every page. Same once-at-open semantics/// as `seq_no_primary_term`.pub version: bool,+ /// Whether the opening request suppressed `_source` (`"_source": false`).+ /// Captured once at open — like `seq_no_primary_term`/`version`, ES fixes+ /// the `_source` selection at the scroll-opening request and applies it to+ /// every continuation page — so the continuation render omits `_source`+ /// when this is set, matching the `_search?scroll=` first page (#624).+ pub source_disabled: bool,pub created: Instant,/// The keep-alive window last requested for this context. A/// continuation without an explicit `scroll` parameter re-arms thediff --git a/engine/crates/xerj-engine/tests/search_context_ttl.rs b/engine/crates/xerj-engine/tests/search_context_ttl.rsindex 8892538c8..c31553859 100644--- a/engine/crates/xerj-engine/tests/search_context_ttl.rs+++ b/engine/crates/xerj-engine/tests/search_context_ttl.rs@@ -38,6 +38,7 @@ fn scroll_ctx(expires_in: i64) -> ScrollContext {page_size: 10,seq_no_primary_term: false,version: false,+ source_disabled: false,created: now,keep_alive,expires_at,