AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): _search_scroll first page suppresses _source like search_impl (#659)
xerj-org5d9c9af0b788
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 1a0f40baf..abd953712 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -20330,6 +20330,19 @@ pub async fn search_with_scroll(// itself — resolved at construction time in the engine, from the// same read as `_source` — not hardcoded, and omitted entirely// when not requested (#428/#440).+ // #659: bring THIS route's first page into `_source` suppression parity+ // with search_impl. The request-level part (`_source:false`, or+ // `stored_fields` implying suppression when `_source` is unspecified) is+ // constant across the snapshot; the mapping `_source.enabled:false` part+ // is per-hit (applied in the hits_json build below). The continuation+ // then matches via `mapping_source_check: true` on the ctx.+ let (suppress_source_for_stored, _) = body+ .stored_fields+ .as_ref()+ .map(parse_stored_fields)+ .unwrap_or((false, vec![]));+ let scroll_request_source_disabled = matches!(body.source, Some(Value::Bool(false)))+ || (suppress_source_for_stored && body.source.is_none());let first_page: Vec<EsHit> = all_hits.iter().take(page_size)@@ -20391,13 +20404,12 @@ pub async fn search_with_scroll(page_size,seq_no_primary_term: emit_seq_no,version: emit_version,- source_disabled: matches!(body.source, Some(Value::Bool(false))),- // `_search_scroll` (this route) does NOT apply mapping- // `_source.enabled:false` suppression on its first page (it emits- // `_source` unconditionally), so its continuation must not either —- // otherwise it gains a new first-page/continuation split. That- // route's broader `_source` gap is tracked in #659.- mapping_source_check: false,+ source_disabled: scroll_request_source_disabled,+ // #659: this route's first page now applies the per-hit mapping+ // `_source.enabled:false` suppression (below), so the continuation+ // must too — matching search_impl and keeping the route+ // self-consistent page to page.+ mapping_source_check: true,created: now,keep_alive,expires_at: now + keep_alive,@@ -20433,13 +20445,21 @@ pub async fn search_with_scroll(if let Some(pt) = h.primary_term {o.insert("_primary_term".to_string(), json!(pt));}- o.insert(- "_source".to_string(),- match &h.source {- Some(src) => src.clone(),- None => Value::Null,- },- );+ // #659: omit `_source` under the same conditions search_impl+ // does — request-level (`_source:false`/stored_fields-implied) or+ // the per-hit mapping `_source.enabled:false`. Otherwise emit the+ // (possibly null) source as before.+ let suppress =+ scroll_request_source_disabled || mapping_source_disabled(&state, &h.index);+ if !suppress {+ o.insert(+ "_source".to_string(),+ match &h.source {+ Some(src) => src.clone(),+ None => Value::Null,+ },+ );+ }if let Some(fields) = &h.fields {o.insert("fields".to_string(),diff --git a/engine/crates/xerj-api/tests/search_scroll_route_source_suppression.rs b/engine/crates/xerj-api/tests/search_scroll_route_source_suppression.rsnew file mode 100644index 000000000..5c0256be1--- /dev/null+++ b/engine/crates/xerj-api/tests/search_scroll_route_source_suppression.rs@@ -0,0 +1,163 @@+//! Issue #659 (follow-up to #637): the `/:index/_search_scroll` route+//! (`search_with_scroll`) emits `_source` UNCONDITIONALLY on its first page — it+//! ignores request `_source: false`, mapping `_source.enabled: false`, and+//! `stored_fields`-implied suppression that `search_impl` (`_search?scroll=`)+//! honours. #637 already made the SHARED continuation renderer suppress these+//! per route; this issue brings THIS route's first page into the same parity so+//! it is both ES-correct and self-consistent (first page and continuation agree).++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++async fn seed(app: &axum::Router, index: &str, mappings: Value) {+ let (st, body) = call(app, "PUT", &format!("/{index}"), mappings).await;+ assert_eq!(st, StatusCode::OK, "create {index}: {body}");+ for (id, v) in [("d0", 10), ("d1", 11), ("d2", 12), ("d3", 13)] {+ let (st, _) = call(+ app,+ "POST",+ &format!("/{index}/_doc/{id}"),+ json!({"grp": "g", "v": v}),+ )+ .await;+ assert_eq!(st, StatusCode::CREATED, "index {id}");+ }+ let _ = call(app, "POST", &format!("/{index}/_refresh"), Value::Null).await;+}++/// mapping `_source.enabled: false` via the `_search_scroll` route: the first+/// page must OMIT `_source` (matching `search_impl`). Fails before the fix — the+/// route emits `_source` unconditionally.+#[tokio::test]+async fn search_scroll_firstpage_omits_source_under_mapping_disabled() {+ let (app, _dir) = app().await;+ seed(+ &app,+ "docs",+ json!({"mappings": {+ "_source": {"enabled": false},+ "properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}+ }}),+ )+ .await;++ let (st, first) = call(+ &app,+ "POST",+ "/docs/_search_scroll?scroll=5m",+ json!({"query": {"match_all": {}}, "size": 1, "sort": [{"v": "asc"}]}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "search_scroll open: {first}");+ assert!(+ first["hits"]["hits"][0].get("_source").is_none(),+ "#659: _search_scroll first page must omit _source when the mapping sets \+ _source.enabled:false (search_impl parity), got: {first}"+ );+}++/// request `_source: false` via the `_search_scroll` route: the first page must+/// OMIT `_source`. Fails before the fix (the route emits it, then the+/// continuation omits it — a pre-existing first-page/continuation split).+#[tokio::test]+async fn search_scroll_firstpage_omits_source_under_source_false() {+ let (app, _dir) = app().await;+ seed(+ &app,+ "docs",+ json!({"mappings": {+ "properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}+ }}),+ )+ .await;++ let (st, first) = call(+ &app,+ "POST",+ "/docs/_search_scroll?scroll=5m",+ json!({"query": {"match_all": {}}, "size": 1, "_source": false, "sort": [{"v": "asc"}]}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "search_scroll open: {first}");+ assert!(+ first["hits"]["hits"][0].get("_source").is_none(),+ "#659: _search_scroll first page must omit _source under _source:false, got: {first}"+ );+}++/// After the fix, the route stays SELF-CONSISTENT: first page and continuation+/// agree (both omit) for a mapping-disabled index. Guards the parity #637 cared+/// about while #659 makes both pages correct.+#[tokio::test]+async fn search_scroll_firstpage_and_continuation_both_omit_under_mapping_disabled() {+ let (app, _dir) = app().await;+ seed(+ &app,+ "docs",+ json!({"mappings": {+ "_source": {"enabled": false},+ "properties": {"grp": {"type": "keyword"}, "v": {"type": "long"}}+ }}),+ )+ .await;++ let (st, first) = call(+ &app,+ "POST",+ "/docs/_search_scroll?scroll=5m",+ json!({"query": {"match_all": {}}, "size": 1, "sort": [{"v": "asc"}]}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "open: {first}");+ let first_has = first["hits"]["hits"][0].get("_source").is_some();+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();++ let (st, page2) = call(+ &app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "5m", "scroll_id": sid}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "continuation: {page2}");+ let cont_has = page2["hits"]["hits"][0].get("_source").is_some();++ assert!(+ !first_has && !cont_has,+ "#659: _search_scroll first page AND continuation must both omit _source for a \+ mapping-disabled index (first={first_has}, continuation={cont_has}); \+ page1={first} page2={page2}"+ );+}