AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(scroll): carry real _seq_no/_version on the same read as _source (#440)
Vincenzo Lombardo7a8913491bb1
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex d4552b188..29eeba8ea 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -10712,12 +10712,10 @@ async fn search_impl(.get(&idx_name).copied().unwrap_or(u64::MAX);- let hit_seq = state- .engine- .get_index(&idx_name)- .ok()- .and_then(|idx| idx.lookup_seq_no(&h.id))- .unwrap_or(0);+ // Read from the hit itself — resolved at construction time in+ // the engine, from the same read as `_source` — rather than a+ // live re-lookup here (#440).+ let hit_seq = h.seq_no.unwrap_or(0);if hit_seq <= max_seq {kept.push((idx_name, h));} else {@@ -12070,15 +12068,12 @@ async fn search_impl(continue;}"_version" => {- // Real per-doc `_version` via the same- // resolver GET/_mget use (external-version- // map wins, then the engine write counter).- let v = state- .engine- .get_index(&idx_name)- .ok()- .and_then(|idx| idx.lookup_version(&h.id))- .unwrap_or(1);+ // Read from the hit itself — resolved at+ // construction time in the engine (same+ // external-version-map-wins resolver GET/+ // _mget use), not a later live re-lookup+ // here (#440).+ let v = h.version.unwrap_or(1);fmap.insert("_version".to_string(), Value::Array(vec![json!(v)]));continue;}@@ -13255,16 +13250,12 @@ async fn search_impl(hit_inner_hits = Some(Value::Object(combined));}- // Pull the real seq_no (and a placeholder primary_term of 1)- // from the engine's version_map so multi-write docs surface- // their actual sequence number instead of a synthetic 0.+ // Real seq_no (and a placeholder primary_term of 1), read from+ // the hit itself — resolved at construction time in the engine,+ // from the same read as `_source`, not a later live re-lookup+ // here (#440).let (real_seq_no, real_primary_term) = if emit_seq_no {- let sn = state- .engine- .get_index(&idx_name)- .ok()- .and_then(|idx| idx.lookup_seq_no(&h.id))- .unwrap_or(hit_idx as u64);+ let sn = h.seq_no.unwrap_or(hit_idx as u64);(Some(sn), Some(1u64))} else {(None, None)@@ -13288,16 +13279,13 @@ async fn search_impl(None},version: if emit_version {- // Real per-doc `_version` via the shared resolver- // (external-version map wins so reindexes with+ // Real per-doc `_version`, read from the hit itself —+ // resolved at construction time in the engine (external-+ // version map wins so reindexes with// `version_type=external[_gte]` echo the caller's- // value, then the engine write counter).- state- .engine- .get_index(&idx_name)- .ok()- .and_then(|idx| idx.lookup_version(&h.id))- .or(Some(1))+ // value, then the engine write counter), not a later+ // live re-lookup here (#440).+ h.version.or(Some(1))} else {None},@@ -14263,6 +14251,11 @@ async fn search_impl(hits: snapshot,position: scroll_page_size,page_size: scroll_page_size,+ // Captured once at open time (real ES semantics — see the+ // field's doc comment) from the same flags that gated this+ // first page's own `_seq_no`/`_version` emission above.+ seq_no_primary_term: emit_seq_no,+ version: emit_version,created: now,keep_alive,expires_at: now + keep_alive,@@ -14433,42 +14426,7 @@ async fn search_impl(if any_disabled_seqno {if let Some(hits) = response_body.pointer_mut("/hits/hits") {if let Some(arr) = hits.as_array_mut() {- for hit in arr.iter_mut() {- let ix = hit- .get("_index")- .and_then(Value::as_str)- .unwrap_or("")- .to_string();- let disabled = state- .engine- .index_settings- .get(&ix)- .map(|v| {- let s = v.clone();- let as_bool = |val: &Value| {- val.as_bool().unwrap_or_else(|| {- val.as_str().map(|x| x == "true").unwrap_or(false)- })- };- s.pointer("/index/disable_sequence_numbers")- .map(as_bool)- .unwrap_or(false)- || s.get("index")- .and_then(|i| i.get("index.disable_sequence_numbers"))- .map(as_bool)- .unwrap_or(false)- || s.get("index.disable_sequence_numbers")- .map(as_bool)- .unwrap_or(false)- })- .unwrap_or(false);- if disabled {- if let Some(obj) = hit.as_object_mut() {- obj.insert("_seq_no".into(), json!(-2i64));- obj.insert("_primary_term".into(), json!(0));- }- }- }+ apply_disable_seqno_sentinel(&state, arr);}}}@@ -19834,17 +19792,34 @@ pub async fn search_with_scroll(// Index name stays paired with its hit (#414).let hits_only: Vec<(String, xerj_query::executor::Hit)> = all_hits.clone();- // Return first page.+ // Control flags for meta-field emission — mirrors `search_impl`'s+ // `emit_seq_no`/`emit_version` (es_compat.rs ~11349).+ let emit_seq_no = body.seq_no_primary_term.unwrap_or(false);+ let emit_version = body.version.unwrap_or(false);++ // Return first page. `_seq_no`/`_version` are read from the hit+ // itself — resolved at construction time in the engine, from the+ // same read as `_source` — not hardcoded, and omitted entirely+ // when not requested (#428/#440).let first_page: Vec<EsHit> = all_hits.iter().take(page_size)- .map(|(idx_name, h)| EsHit {+ .enumerate()+ .map(|(hit_idx, (idx_name, h))| EsHit {index: idx_name.clone(),id: h.id.clone(),score: Some(h.score as f64),- version: Some(1),- seq_no: Some(0),- primary_term: Some(1),+ version: if emit_version {+ h.version.or(Some(1))+ } else {+ None+ },+ seq_no: if emit_seq_no {+ Some(h.seq_no.unwrap_or(hit_idx as u64))+ } else {+ None+ },+ primary_term: if emit_seq_no { Some(1) } else { None },source: if h.source.is_null() {None} else {@@ -19890,6 +19865,8 @@ pub async fn search_with_scroll(hits: hits_only,position: page_size,page_size,+ seq_no_primary_term: emit_seq_no,+ version: emit_version,created: now,keep_alive,expires_at: now + keep_alive,@@ -20053,6 +20030,50 @@ pub async fn next_scroll_path(.await}+/// Apply the ES `index.disable_sequence_numbers` sentinel — `_seq_no: -2`,+/// `_primary_term: 0` — to every hit in `hits` whose own `_index` setting+/// has it enabled. Shared by `search_impl` (plain `_search`) and+/// `scroll_page_response` (#440: scroll had no equivalent at all, so a+/// continuation page for such an index reported whatever `_seq_no` the+/// snapshotted hit carried instead of the sentinel real ES uses).+fn apply_disable_seqno_sentinel(state: &AppState, hits: &mut [Value]) {+ for hit in hits.iter_mut() {+ let ix = hit+ .get("_index")+ .and_then(Value::as_str)+ .unwrap_or("")+ .to_string();+ let disabled = state+ .engine+ .index_settings+ .get(&ix)+ .map(|v| {+ let s = v.clone();+ let as_bool = |val: &Value| {+ val.as_bool()+ .unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false))+ };+ s.pointer("/index/disable_sequence_numbers")+ .map(as_bool)+ .unwrap_or(false)+ || s.get("index")+ .and_then(|i| i.get("index.disable_sequence_numbers"))+ .map(as_bool)+ .unwrap_or(false)+ || s.get("index.disable_sequence_numbers")+ .map(as_bool)+ .unwrap_or(false)+ })+ .unwrap_or(false);+ if disabled {+ if let Some(obj) = hit.as_object_mut() {+ obj.insert("_seq_no".into(), json!(-2i64));+ obj.insert("_primary_term".into(), json!(0));+ }+ }+ }+}+async fn scroll_page_response(state: &AppState,scroll_id: String,@@ -20089,6 +20110,12 @@ async fn scroll_page_response(let page_size = ctx.page_size.max(1);let position = ctx.position;let total = ctx.hits.len() as u64;+ // Captured once at scroll-open time (real ES semantics) — see+ // `ScrollContext::seq_no_primary_term`/`::version`'s doc+ // comments. Read into locals before the per-hit closure below+ // borrows `ctx.hits`.+ let emit_seq_no = ctx.seq_no_primary_term;+ let emit_version = ctx.version;// Detect whether the initial search sorted by a non-score key;// in that case `max_score` must be null on scroll pages too.@@ -20098,19 +20125,34 @@ async fn scroll_page_response(.map(|(_, h)| !h.sort.is_empty()).unwrap_or(false);+ // `_seq_no`/`_version` are read from each hit itself — resolved+ // at construction time in the engine, from the same read as+ // `_source`, snapshotted into `ctx.hits` at scroll-open (a+ // single request-duration window, never re-read live) — not+ // hardcoded, and omitted when the flag wasn't requested+ // (#428/#440).let page_hits: Vec<EsHit> = ctx.hits.iter().skip(position).take(page_size)- .map(|(hit_index, h)| EsHit {+ .enumerate()+ .map(|(hit_idx, (hit_index, h))| EsHit {// Per-hit index, not the context-level one (#414).index: hit_index.clone(),id: h.id.clone(),score: Some(h.score as f64),- version: Some(1),- seq_no: Some(0),- primary_term: Some(1),+ version: if emit_version {+ h.version.or(Some(1))+ } else {+ None+ },+ seq_no: if emit_seq_no {+ Some(h.seq_no.unwrap_or(hit_idx as u64))+ } else {+ None+ },+ primary_term: if emit_seq_no { Some(1) } else { None },source: if h.source.is_null() {None} else {@@ -20148,6 +20190,51 @@ async fn scroll_page_response(} else {page_hits.first().and_then(|h| h.score)};+ let mut hits_json: Vec<Value> = page_hits+ .iter()+ .map(|h| {+ let mut o = serde_json::Map::new();+ o.insert("_index".to_string(), Value::String(h.index.clone()));+ o.insert("_id".to_string(), Value::String(h.id.clone()));+ o.insert(+ "_score".to_string(),+ match h.score {+ Some(s) => json!(s),+ None => Value::Null,+ },+ );+ // Previously always omitted regardless of `version`/+ // `seq_no_primary_term` — `EsHit` carried them but+ // this hand-built map never read them out (#428).+ if let Some(v) = h.version {+ o.insert("_version".to_string(), json!(v));+ }+ if let Some(sn) = h.seq_no {+ o.insert("_seq_no".to_string(), json!(sn));+ }+ if let Some(pt) = h.primary_term {+ o.insert("_primary_term".to_string(), json!(pt));+ }+ if let Some(src) = &h.source {+ o.insert("_source".to_string(), src.clone());+ }+ if let Some(sort) = &h.sort {+ o.insert("sort".to_string(), Value::Array(sort.clone()));+ }+ if let Some(fields) = &h.fields {+ o.insert(+ "fields".to_string(),+ serde_json::to_value(fields).unwrap_or(Value::Null),+ );+ }+ Value::Object(o)+ })+ .collect();+ // `index.disable_sequence_numbers` sentinel (`_seq_no: -2`,+ // `_primary_term: 0`) — scroll previously had no equivalent of+ // `search_impl`'s handling at all (#440).+ apply_disable_seqno_sentinel(state, &mut hits_json);+let mut resp = json!({"_scroll_id": scroll_id,"took": 0u64,@@ -20156,28 +20243,7 @@ async fn scroll_page_response("hits": {"total": { "value": total, "relation": "eq" },"max_score": max_score,- "hits": page_hits.iter().map(|h| {- let mut o = serde_json::Map::new();- o.insert("_index".to_string(), Value::String(h.index.clone()));- o.insert("_id".to_string(), Value::String(h.id.clone()));- o.insert("_score".to_string(), match h.score {- Some(s) => json!(s),- None => Value::Null,- });- if let Some(src) = &h.source {- o.insert("_source".to_string(), src.clone());- }- if let Some(sort) = &h.sort {- o.insert("sort".to_string(), Value::Array(sort.clone()));- }- if let Some(fields) = &h.fields {- o.insert(- "fields".to_string(),- serde_json::to_value(fields).unwrap_or(Value::Null),- );- }- Value::Object(o)- }).collect::<Vec<_>>()+ "hits": hits_json}});@@ -20214,6 +20280,8 @@ mod passage_scroll_tests {id: id.to_string(),score: 1.0,source: json!({"company": "ACME"}),+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -20244,6 +20312,8 @@ mod passage_scroll_tests {],position: 1,page_size: 1,+ seq_no_primary_term: false,+ version: false,created: now,keep_alive: Duration::from_secs(60),expires_at: now + Duration::from_secs(60),@@ -20300,6 +20370,8 @@ mod passage_scroll_tests {],position: 1,page_size: 1,+ seq_no_primary_term: false,+ version: false,created: now,keep_alive: Duration::from_secs(60),expires_at: now + Duration::from_secs(60),@@ -20352,6 +20424,8 @@ mod passage_scroll_tests {],position: 1,page_size: 1,+ seq_no_primary_term: false,+ version: false,created: now,keep_alive: Duration::from_secs(60),expires_at: now + Duration::from_secs(60),diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rsindex 8045da7ef..1289e6719 100644--- a/engine/crates/xerj-api/src/native.rs+++ b/engine/crates/xerj-api/src/native.rs@@ -97,6 +97,8 @@ mod passage_wire_tests {id: "report-page".to_string(),score: 0.91,source: serde_json::json!({"company": "ACME"}),+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,diff --git a/engine/crates/xerj-api/tests/scroll_seq_no_torn_read.rs b/engine/crates/xerj-api/tests/scroll_seq_no_torn_read.rsnew file mode 100644index 000000000..f3fec5478--- /dev/null+++ b/engine/crates/xerj-api/tests/scroll_seq_no_torn_read.rs@@ -0,0 +1,327 @@+//! Issue #440: `_seq_no`/`_version` on a scroll page must come from the SAME+//! read that produced `_source`, not a later pass over the version map.+//!+//! Two repair attempts on PR #431 were refuted by live reproduction:+//! 1. resolve live at render time — window is the whole scroll lifetime.+//! 2. resolve once at scroll-open time, from a context-level index lookup+//! — narrower, but still a SECOND read, so a write landing between the+//! `_source` read and the `_seq_no` read pairs a stale body with a live+//! sequence number. Since `_seq_no` is fed back as `if_seq_no`, that is+//! a silent lost update, not a cosmetic staleness.+//!+//! The fix threads `seq_no`/`version` onto `xerj_query::executor::Hit`+//! itself, populated in the engine at the exact point `_source` is read.+//! These tests exercise that guarantee from the outside, through the real+//! HTTP routes — the in-module unit tests only ever injected a synthetic+//! `ScrollContext` with fabricated `Hit`s, which is exactly why the original+//! bug (and PR #431's two subsequent regressions) had no version-map entry+//! to expose it.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn json_req(+ app: &axum::Router,+ method: &str,+ path: &str,+ body: Value,+) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ let v: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ (status, v)+}++async fn create_index(app: &axum::Router, index: &str) {+ let (st, body) = json_req(+ app,+ "PUT",+ &format!("/{index}"),+ json!({"mappings": {"properties": {"v": {"type": "long"}}}}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "create {index}: {body}");+}++async fn refresh(app: &axum::Router, index: &str) {+ let (st, body) = json_req(app, "POST", &format!("/{index}/_refresh"), Value::Null).await;+ assert_eq!(st, StatusCode::OK, "refresh {index}: {body}");+}++async fn bulk_index(app: &axum::Router, index: &str, ids: impl Iterator<Item = usize>) {+ let mut bulk = String::new();+ for i in ids {+ bulk.push_str(&format!("{}\n", json!({"index": {"_id": i.to_string()}})));+ bulk.push_str(&format!("{}\n", json!({"v": i})));+ }+ let response = app+ .clone()+ .oneshot(+ Request::builder()+ .method("POST")+ .uri(format!("/{index}/_bulk"))+ .header("content-type", "application/x-ndjson")+ .body(Body::from(bulk))+ .unwrap(),+ )+ .await+ .unwrap();+ assert_eq!(response.status(), StatusCode::OK);+}++/// Multi-index scroll, ids colliding across indices (the reindex/CDC shape+/// #440 measured against) — every hit's `_seq_no`/`_version` must agree with+/// what a plain `_search` on the hit's OWN index reports for the same doc.+///+/// PR #431's own repair attempt #2 (resolve once at scroll-open, from a+/// context-level index) failed exactly this scenario: 4 of 8 hits carried+/// values read out of the OTHER index's version map. This test's structure+/// — colliding ids, differing per-index write counts so `_seq_no` is not+/// merely a page ordinal — reproduces the shape that caught it.+#[tokio::test]+async fn multi_index_scroll_seq_no_matches_plain_search_per_index() {+ let (app, _dir) = app().await;+ create_index(&app, "sq_a").await;+ create_index(&app, "sq_b").await;++ // Different write counts per index so `_seq_no` is not simply the page+ // ordinal (which would let a wrong-index lookup pass unnoticed if the+ // two indices' sequence numbers happened to line up).+ bulk_index(&app, "sq_a", 0..4).await;+ bulk_index(&app, "sq_a", 0..4).await; // re-index sq_a's docs once more+ bulk_index(&app, "sq_b", 0..4).await;+ refresh(&app, "sq_a").await;+ refresh(&app, "sq_b").await;++ let (st, first) = json_req(+ &app,+ "POST",+ "/sq_a,sq_b/_search?scroll=2m",+ json!({+ "query": {"match_all": {}},+ "size": 3,+ "seq_no_primary_term": true,+ "version": true,+ }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "scroll start: {first}");++ let mut sid = first["_scroll_id"].as_str().map(str::to_string);+ let mut page = first;+ let mut scrolled: Vec<(String, String, i64, i64)> = Vec::new();++ for _ in 0..16 {+ let hits = page["hits"]["hits"].as_array().cloned().unwrap_or_default();+ if hits.is_empty() {+ break;+ }+ for h in &hits {+ scrolled.push((+ h["_index"].as_str().unwrap_or_default().to_string(),+ h["_id"].as_str().unwrap_or_default().to_string(),+ h["_seq_no"]+ .as_i64()+ .expect("scrolled hit must carry _seq_no"),+ h["_version"]+ .as_i64()+ .expect("scrolled hit must carry _version"),+ ));+ }+ let Some(id) = sid.clone() else { break };+ let (st, next) = json_req(+ &app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "2m", "scroll_id": id}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "scroll continuation: {next}");+ sid = next["_scroll_id"].as_str().map(str::to_string).or(sid);+ page = next;+ }+ assert_eq!(+ scrolled.len(),+ 8,+ "must have scrolled every hit across both indices"+ );++ // Ground truth: a direct GET on the hit's OWN index — the same+ // resolver (`lookup_seq_no`/`lookup_version`) `_mget`/plain `_search`+ // use, isolated from any query-matching ambiguity.+ for (idx, id, scrolled_seq, scrolled_version) in &scrolled {+ let (st, truth) = json_req(&app, "GET", &format!("/{idx}/_doc/{id}"), Value::Null).await;+ assert_eq!(st, StatusCode::OK, "truth GET {idx}/{id}: {truth}");+ assert_eq!(+ truth["_seq_no"].as_i64(),+ Some(*scrolled_seq),+ "scrolled _seq_no for {idx}/{id} does not match a direct GET on its own index \+ — a hit is carrying another index's version-map entry (#440)"+ );+ assert_eq!(+ truth["_version"].as_i64(),+ Some(*scrolled_version),+ "scrolled _version for {idx}/{id} does not match a direct GET on its own index"+ );+ }+}++/// The direct #440 repro: a document already captured in the scroll+/// snapshot is updated AFTER the scroll opens but BEFORE its continuation+/// page is fetched. The scrolled page must report the SNAPSHOT's+/// `_seq_no`/`_source` pair — not a live `_seq_no` beside a stale+/// `_source`, which is exactly the torn read that lets a caller's+/// `if_seq_no` CAS silently overwrite the concurrent update.+#[tokio::test]+async fn scroll_continuation_reports_the_snapshot_pair_not_a_torn_one() {+ let (app, _dir) = app().await;+ create_index(&app, "torn").await;+ bulk_index(&app, "torn", 0..4).await;+ refresh(&app, "torn").await;++ // page_size 1 forces every hit onto its own continuation page.+ let (st, first) = json_req(+ &app,+ "POST",+ "/torn/_search?scroll=5m",+ json!({+ "query": {"match_all": {}},+ "size": 1,+ "sort": [{"v": "asc"}],+ "seq_no_primary_term": true,+ "version": true,+ }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "scroll start: {first}");+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();+ let page1_id = first["hits"]["hits"][0]["_id"]+ .as_str()+ .unwrap()+ .to_string();+ let page1_seq = first["hits"]["hits"][0]["_seq_no"]+ .as_i64()+ .expect("_seq_no on page 1");++ // Concurrent write AFTER the scroll snapshot: bump doc 1's `v` and its+ // real seq_no/version. If the scroll re-derives against the LIVE+ // index, page 2 (doc "1") would report this write's seq_no next to+ // page 1's already-returned doc (id "0") — but the direct exposure is+ // doc "1" itself, captured below.+ let (st, upd) = json_req(&app, "PUT", "/torn/_doc/1", json!({"v": 999})).await;+ assert_eq!(st, StatusCode::OK, "concurrent update: {upd}");+ let live_seq_after_update = upd["_seq_no"].as_i64().expect("_seq_no on update");++ // Continue the scroll to the page holding doc "1" (sorted by v asc,+ // doc "1" is the second hit — id "0" was v=0, id "1" was v=1 pre-update).+ let (st, page2) = json_req(+ &app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "5m", "scroll_id": sid}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "scroll continuation: {page2}");+ let hit = &page2["hits"]["hits"][0];+ assert_eq!(+ hit["_id"], "1",+ "page 2 must be the doc updated concurrently"+ );+ let scrolled_seq = hit["_seq_no"].as_i64().expect("_seq_no on page 2");+ let scrolled_source_v = hit["_source"]["v"].as_i64();++ assert_ne!(+ scrolled_seq, live_seq_after_update,+ "scrolled page reported the LIVE post-update _seq_no ({live_seq_after_update}) — \+ a caller round-tripping this as if_seq_no would CAS successfully over the \+ concurrent write, a silent lost update (#440)"+ );+ assert_eq!(+ scrolled_source_v,+ Some(1),+ "scrolled _source must be the SNAPSHOT body (v=1, pre-update), not the live one — \+ found {scrolled_source_v:?}"+ );+ assert_ne!(+ page1_seq, scrolled_seq,+ "sanity: two different docs must not share a _seq_no"+ );+ let _ = page1_id;+}++/// #440's "adjacent, smaller, separate" gap: `disable_sequence_numbers`+/// applies its `-2`/`0` sentinel in `search_impl`'s response body but+/// `scroll_page_response` had no equivalent at all — a scroll continuation+/// page for such an index reported whatever `_seq_no` the snapshotted hit+/// carried (or, pre-fix, a hardcoded `0`) instead of ES's `-2` sentinel.+#[tokio::test]+async fn scroll_continuation_honours_disable_sequence_numbers() {+ let (app, _dir) = app().await;+ let (st, body) = json_req(+ &app,+ "PUT",+ "/noseq",+ json!({+ "settings": {"index": {"disable_sequence_numbers": true}},+ "mappings": {"properties": {"v": {"type": "long"}}}+ }),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "create noseq: {body}");+ bulk_index(&app, "noseq", 0..3).await;+ refresh(&app, "noseq").await;++ let (st, first) = json_req(+ &app,+ "POST",+ "/noseq/_search?scroll=2m",+ json!({"query": {"match_all": {}}, "size": 1, "sort": [{"v": "asc"}]}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "scroll start: {first}");+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();++ let (st, page2) = json_req(+ &app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "2m", "scroll_id": sid}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "scroll continuation: {page2}");+ let hit = &page2["hits"]["hits"][0];+ assert_eq!(+ hit["_seq_no"].as_i64(),+ Some(-2),+ "continuation page for a disable_sequence_numbers index must carry the -2 \+ sentinel, got: {hit}"+ );+ assert_eq!(+ hit["_primary_term"].as_i64(),+ Some(0),+ "continuation page for a disable_sequence_numbers index must carry primary_term 0"+ );+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 10657b4b7..d0ab8b6e8 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -161,6 +161,15 @@ pub struct ScrollContext {pub hits: Vec<(String, xerj_query::executor::Hit)>,pub position: usize,pub page_size: usize,+ /// Whether to emit `_seq_no`/`_primary_term` on every page. Real ES+ /// fixes this at the scroll-opening request and does not re-read it+ /// from continuation bodies — the flag governs the whole scroll's+ /// lifetime, not each page — so it's captured once here, not re-parsed+ /// per continuation.+ pub seq_no_primary_term: bool,+ /// Whether to emit `_version` on every page. Same once-at-open semantics+ /// as `seq_no_primary_term`.+ pub version: bool,pub created: Instant,/// The keep-alive window last requested for this context. A/// continuation without an explicit `scroll` parameter re-arms thediff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 4c72dea51..47fa3e251 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -10671,6 +10671,7 @@ impl Index {generated_embedding_companion_fields(&schema.schema)};Some(knn_result_from_scored(+ self,request,field,scored,@@ -11176,6 +11177,7 @@ impl Index {generated_embedding_companion_fields(&schema.schema)};let mut result = knn_result_from_scored(+ self,request,field,scored,@@ -11270,6 +11272,7 @@ impl Index {generated_embedding_companion_fields(&schema.schema)};let mut result = knn_result_from_scored(+ self,request,"",merged,@@ -12235,15 +12238,27 @@ impl Index {let hits: Vec<Hit> = scored.into_iter()- .map(|(id, score, source)| Hit {- id,- score,- source,- sort: Vec::new(),- explain: None,- highlight: None,- matched_queries: Vec::new(),- passage: None,+ .map(|(id, score, source)| {+ // Same read the final page's `_source` is served from+ // (post-truncate, post-filter) — not a later pass (#440).+ let (seq_no, version) = self+ .store+ .version_map+ .get(&id)+ .map(|ver| self.hit_seq_version(&id, &ver))+ .unwrap_or((None, None));+ Hit {+ id,+ score,+ source,+ seq_no,+ version,+ sort: Vec::new(),+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ passage: None,+ }}).collect();@@ -12638,6 +12653,32 @@ impl Index {/// The internal WAL counter starts at 1 so the first WAL op has seq=1,/// but ES semantics number seq_no from 0 per shard. We subtract 1 to/// match the ES wire format.+ /// Same transform as [`Self::lookup_seq_no`]/[`Self::lookup_version`],+ /// but from a `VersionEntry` the caller already has in scope — for+ /// building a `Hit` inline with the doc's version-map entry already in+ /// hand, so `_seq_no`/`_version` come from the SAME read as `_source`,+ /// never a later pass over the version map (#440: a second pass lets a+ /// concurrent write pair a stale `_source` with a live `_seq_no`, which+ /// is a silent lost update once a caller round-trips it as `if_seq_no`).+ fn hit_seq_version(+ &self,+ id: &str,+ ver: &xerj_storage::version_map::VersionEntry,+ ) -> (Option<u64>, Option<u64>) {+ if ver.deleted {+ return (None, None);+ }+ (+ Some(ver.seq_no.saturating_sub(1)),+ Some(+ self.external_versions+ .get(id)+ .map(|v| *v)+ .unwrap_or(ver.version),+ ),+ )+ }+pub fn lookup_seq_no(&self, id: &str) -> Option<u64> {self.store.version_map.get(id).and_then(|entry| {if entry.deleted {@@ -14470,9 +14511,16 @@ impl Index {if let Some(src) = self.memtable.get_doc_source_as_value(&hit.id) {hit.source = src;}+ // Resolved at the SAME moment as the deferred+ // `source` above, not a later pass (#440).+ if let Some(ver) = self.store.version_map.get(&hit.id) {+ let (seq_no, version) = self.hit_seq_version(&hit.id, &ver);+ hit.seq_no = seq_no;+ hit.version = version;+ }}seen_ids.insert(hit.id.clone());- let seq = self.lookup_seq_no(&hit.id).unwrap_or(u64::MAX);+ let seq = hit.seq_no.unwrap_or(u64::MAX);topk.offer(hit, seq, self);}} else if all_hits.len() < materialisation_limit && !seen_ids.contains(&hit.id)@@ -14509,12 +14557,22 @@ impl Index {if !rejected {let source = src_arc.map(|a| (*a).clone()).unwrap_or(Value::Null);seen_ids.insert(id.clone());- let seq = self.lookup_seq_no(&id).unwrap_or(u64::MAX);+ // Same read `source` came from — not a later+ // pass (#440) — and reused below for ranking.+ let (seq_no, version) = self+ .store+ .version_map+ .get(&id)+ .map(|ver| self.hit_seq_version(&id, &ver))+ .unwrap_or((None, None));+ let seq = seq_no.unwrap_or(u64::MAX);topk.offer(Hit {id,score: 1.0,source,+ seq_no,+ version,sort: Vec::new(),explain: None,highlight: None,@@ -14534,6 +14592,13 @@ impl Index {id,score: 1.0,source: Value::Null,+ // Deferred with `source`: `fill_memtable_sources`+ // resolves both from the SAME read once this+ // hit is known to be on the returned page+ // (#440) — not here, where most candidates+ // never make the final page.+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -15022,6 +15087,10 @@ impl Index {id: doc_id,score,source: Value::Null, // filled in below+ // Deferred with `source` — see the deferred-`Hit`+ // comment in `admit_hit!`'s unsorted `count` arm (#440).+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -15147,10 +15216,24 @@ impl Index {}}let score = score_doc(&source, &doc_id);+ // Same read `source` (above) is cloned from — not+ // a later pass (#440). NOT the tuple's own+ // `seq_no` (that's the memtable's raw arrival+ // ordinal, used only for the `#191` competitive+ // tie-break above — a different number from the+ // ES-wire `_seq_no`/`_version` this resolves).+ let (hit_seq_no, hit_version) = self+ .store+ .version_map+ .get(&doc_id)+ .map(|ver| self.hit_seq_version(&doc_id, &ver))+ .unwrap_or((None, None));let hit = Hit {id: doc_id,score,source: (*source).clone(),+ seq_no: hit_seq_no,+ version: hit_version,sort: Vec::new(),explain: None,highlight: None,@@ -16316,6 +16399,14 @@ impl Index {.and_then(Value::as_str).unwrap_or("").to_string();+ // Resolved here, from the SAME+ // version-map entry the+ // deleted/stale check below+ // reads, so `_seq_no`/`_version`+ // never diverge from `source`+ // (#440).+ let mut hit_seq_no = None;+ let mut hit_version = None;if let Some(ver) = self.store.version_map.get(&id) {// Tombstoned, or SUPERSEDED —// the live version resides in@@ -16336,6 +16427,9 @@ impl Index {{continue;}+ let (s, v) = self.hit_seq_version(&id, &ver);+ hit_seq_no = s;+ hit_version = v;}// Dedup against memtable/earlier// segments WITHOUT touching@@ -16351,6 +16445,8 @@ impl Index {id,score: sh.score,source,+ seq_no: hit_seq_no,+ version: hit_version,sort: Vec::new(),explain: None,highlight: None,@@ -16358,8 +16454,7 @@ impl Index {passage: None,};if let Some(topk) = sort_topk.as_mut() {- let seq =- self.lookup_seq_no(&hit.id).unwrap_or(u64::MAX);+ let seq = hit_seq_no.unwrap_or(u64::MAX);topk.offer(hit, seq, self);} else {all_hits.push(hit);@@ -19140,6 +19235,19 @@ impl Index {// happen for FTS hits from the active memtable that were// found by score but whose source was not cloned inline.}+ // Catch-all for whichever construction sites deliberately+ // left `seq_no`/`version` unresolved alongside a deferred+ // `source` (#440) — resolved here, at the SAME moment as the+ // `source` fill above, only for hits that reach this final,+ // already-bounded page. A no-op for hits already populated+ // at construction time (the overwhelming majority).+ if h.seq_no.is_none() {+ if let Some(ver) = self.store.version_map.get(&h.id) {+ let (seq_no, version) = self.hit_seq_version(&h.id, &ver);+ h.seq_no = seq_no;+ h.version = version;+ }+ }h}).collect()@@ -20748,10 +20856,20 @@ impl Index {if !seen.insert(id.clone()) {continue;}+ // Same read the winner's `_source` is hydrated from+ // (post-select_nth winners only) — not a later pass (#440).+ let (seq_no, version) = self+ .store+ .version_map+ .get(id)+ .map(|ver| self.hit_seq_version(id, &ver))+ .unwrap_or((None, None));hits.push(Hit {id: id.clone(),score: 1.0,source: (**source).clone(),+ seq_no,+ version,sort: Vec::new(),explain: None,highlight: None,@@ -20777,10 +20895,20 @@ impl Index {continue;}let source = doc.get("_source").cloned().unwrap_or(doc);+ // Same read `source` was just parsed from — not a later+ // pass (#440).+ let (seq_no, version) = self+ .store+ .version_map+ .get(&id)+ .map(|ver| self.hit_seq_version(&id, &ver))+ .unwrap_or((None, None));hits.push(Hit {id,score: 1.0,source,+ seq_no,+ version,sort: Vec::new(),explain: None,highlight: None,@@ -22359,10 +22487,22 @@ impl Index {continue;}let source = doc.get("_source").cloned().unwrap_or(doc);+ // Same read `source` was just parsed from — not a later pass+ // (#440). Not `_seq` (discarded above): that's the pre-hydration+ // ranking key from `seg_seqs`, a different number from the+ // ES-wire `_seq_no`/`_version` this resolves.+ let (seq_no, version) = self+ .store+ .version_map+ .get(&id)+ .map(|ver| self.hit_seq_version(&id, &ver))+ .unwrap_or((None, None));hits.push(Hit {id,score,source,+ seq_no,+ version,sort: Vec::new(),explain: None,highlight: None,@@ -23564,6 +23704,11 @@ impl Index {Err(_) => continue,};let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");+ // Captured from the SAME version-map entry the+ // deleted/superseded check reads, so `_seq_no`/`_version` never+ // diverge from the `source` parsed above (#440).+ let mut hit_seq_no = None;+ let mut hit_version = None;if let Some(ver) = self.store.version_map.get(id_ref) {if ver.deleted {continue;@@ -23574,6 +23719,9 @@ impl Index {continue;}}+ let (s, v) = self.hit_seq_version(id_ref, &ver);+ hit_seq_no = s;+ hit_version = v;}*total_count += 1;if seen_ids.contains(id_ref) {@@ -23594,12 +23742,14 @@ impl Index {}let id = id_ref.to_string();seen_ids.insert(id.clone());- let seq = self.lookup_seq_no(&id).unwrap_or(u64::MAX);+ let seq = hit_seq_no.unwrap_or(u64::MAX);topk.offer_keyed(Hit {id,score,source: source_ref.clone(),+ seq_no: hit_seq_no,+ version: hit_version,sort: key,explain: None,highlight: None,@@ -23674,6 +23824,11 @@ impl Index {Err(_) => continue,};let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");+ // Captured from the SAME version-map entry the+ // deleted/superseded check reads, so `_seq_no`/`_version` never+ // diverge from the `source` parsed above (#440).+ let mut hit_seq_no = None;+ let mut hit_version = None;if let Some(ver) = self.store.version_map.get(id_ref) {if ver.deleted {continue;@@ -23684,6 +23839,9 @@ impl Index {continue;}}+ let (s, v) = self.hit_seq_version(id_ref, &ver);+ hit_seq_no = s;+ hit_version = v;}// Re-test the full query — the pre-filter is only a superset.let matched = if is_match_all {@@ -23731,6 +23889,8 @@ impl Index {id,score,source,+ seq_no: hit_seq_no,+ version: hit_version,sort: Vec::new(),explain: None,highlight: None,@@ -24202,6 +24362,11 @@ impl Index {};let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");+ // Captured from the SAME version-map entry the+ // deleted/superseded check reads, so `_seq_no`/`_version` never+ // diverge from the `source` parsed above (#440).+ let mut hit_seq_no = None;+ let mut hit_version = None;if let Some(ver) = self.store.version_map.get(id_ref) {if ver.deleted {continue;@@ -24216,6 +24381,9 @@ impl Index {continue;}}+ let (s, v) = self.hit_seq_version(id_ref, &ver);+ hit_seq_no = s;+ hit_version = v;}let matched = if is_match_all {@@ -24291,12 +24459,14 @@ impl Index {}let id: String = id_ref.to_string();seen_ids.insert(id.clone());- let seq = self.lookup_seq_no(&id).unwrap_or(u64::MAX);+ let seq = hit_seq_no.unwrap_or(u64::MAX);topk.offer_keyed(Hit {id,score,source: source_ref.clone(),+ seq_no: hit_seq_no,+ version: hit_version,sort: key,explain: None,highlight: None,@@ -24342,6 +24512,8 @@ impl Index {id,score,source,+ seq_no: hit_seq_no,+ version: hit_version,sort: Vec::new(),explain: None,highlight: None,@@ -25855,6 +26027,8 @@ mod source_filter_tests {id: id.to_string(),score,source,+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -26054,6 +26228,8 @@ mod savings_tests {id: id.to_string(),score: 1.0,source,+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -29597,6 +29773,7 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {/// `from`/`size`, and shape the response. Centralised so the exact and/// approximate paths cannot drift on hits format or total semantics.fn knn_result_from_scored(+ index: &Index,request: &SearchRequest,vector_field: &str,mut scored: Vec<(String, f32, Value, Option<u32>)>,@@ -29650,10 +29827,21 @@ fn knn_result_from_scored(.then(|| passage_match_from_source(&source, vector_field, ordinal)).flatten();strip_internal_passage_metadata(&mut source);+ // Same read `source` was carried in from (already the bounded+ // final page — `.skip/.take` above already applied) — not a+ // later pass (#440).+ let (seq_no, version) = index+ .store+ .version_map+ .get(&id)+ .map(|ver| index.hit_seq_version(&id, &ver))+ .unwrap_or((None, None));Hit {id,score,source,+ seq_no,+ version,sort: Vec::new(),explain: None,highlight: None,@@ -42570,6 +42758,8 @@ mod lexical_passage_tests {id: "doc-1".into(),score: 1.0,source: json!({"body": body, "title": "unrelated", "page": 3}),+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -42626,6 +42816,8 @@ mod lexical_passage_tests {id: "doc-1".into(),score: 1.0,source: json!({"body": body, "defs": "addReplyNull bulk"}),+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -42661,6 +42853,8 @@ mod lexical_passage_tests {id: "doc-1".into(),score: 1.0,source: json!({"body": body, "language": "rust", "symbol_count": 1}),+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -42702,6 +42896,8 @@ mod lexical_passage_tests {id: "doc-1".into(),score: 1.0,source: json!({"body": body}),+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -42714,6 +42910,8 @@ mod lexical_passage_tests {// Same body, but with a stray `language` field and no// `symbol_count` — not autoindex's code-doc shape.source: json!({"body": body, "language": "rust"}),+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,@@ -42760,6 +42958,8 @@ mod lexical_passage_tests {id: "doc-1".into(),score: 1.0,source: json!({"body": "needle here\n"}),+ seq_no: None,+ version: None,sort: Vec::new(),explain: None,highlight: None,diff --git a/engine/crates/xerj-engine/tests/search_context_ttl.rs b/engine/crates/xerj-engine/tests/search_context_ttl.rsindex d9cbefad2..8892538c8 100644--- a/engine/crates/xerj-engine/tests/search_context_ttl.rs+++ b/engine/crates/xerj-engine/tests/search_context_ttl.rs@@ -36,6 +36,8 @@ fn scroll_ctx(expires_in: i64) -> ScrollContext {hits: Vec::new(),position: 0,page_size: 10,+ seq_no_primary_term: false,+ version: false,created: now,keep_alive,expires_at,diff --git a/engine/crates/xerj-query/src/executor.rs b/engine/crates/xerj-query/src/executor.rsindex 2ae835208..d1ca6fd0a 100644--- a/engine/crates/xerj-query/src/executor.rs+++ b/engine/crates/xerj-query/src/executor.rs@@ -50,6 +50,18 @@ pub struct Hit {/// The document source fields (filtered per `_source`).#[serde(rename = "_source")]pub source: serde_json::Value,+ /// ES-wire (0-based) sequence number, read from the SAME version-map+ /// entry `source` was resolved from — never re-resolved later, so a+ /// concurrent write can't pair a stale `source` with a live `seq_no`+ /// (#440). `None` for a doc that is unknown/tombstoned at read time, or+ /// when the deferred-hydration path hasn't reached this hit yet+ /// (resolved in that case by whatever later fills `source`).+ #[serde(skip_serializing_if = "Option::is_none")]+ pub seq_no: Option<u64>,+ /// Document version, from the same read as `seq_no`/`source`. Same+ /// `None` semantics as `seq_no`.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub version: Option<u64>,/// The sort key values for this hit (used by `search_after`).#[serde(skip_serializing_if = "Vec::is_empty")]pub sort: Vec<serde_json::Value>,@@ -542,6 +554,8 @@ mod tests {id: id.to_string(),score,source: serde_json::Value::Null,+ seq_no: None,+ version: None,sort: vec![],explain: None,highlight: None,