Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(scroll): carry real _seq_no/_version on the same read as _source (#440)

Vincenzo Lombardo7a8913491bb1
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index d4552b188..29eeba8ea 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -10712,12 +10712,10 @@ async fn search_impl(
.get(&idx_name)
.copied()
.unwrap_or(u64::MAX);
- let hit_seq = state
- .engine
- .get_index(&idx_name)
- .ok()
- .and_then(|idx| idx.lookup_seq_no(&h.id))
- .unwrap_or(0);
+ // Read from the hit itself — resolved at construction time in
+ // the engine, from the same read as `_source` — rather than a
+ // live re-lookup here (#440).
+ let hit_seq = h.seq_no.unwrap_or(0);
if hit_seq <= max_seq {
kept.push((idx_name, h));
} else {
@@ -12070,15 +12068,12 @@ async fn search_impl(
continue;
}
"_version" => {
- // Real per-doc `_version` via the same
- // resolver GET/_mget use (external-version
- // map wins, then the engine write counter).
- let v = state
- .engine
- .get_index(&idx_name)
- .ok()
- .and_then(|idx| idx.lookup_version(&h.id))
- .unwrap_or(1);
+ // Read from the hit itself — resolved at
+ // construction time in the engine (same
+ // external-version-map-wins resolver GET/
+ // _mget use), not a later live re-lookup
+ // here (#440).
+ let v = h.version.unwrap_or(1);
fmap.insert("_version".to_string(), Value::Array(vec![json!(v)]));
continue;
}
@@ -13255,16 +13250,12 @@ async fn search_impl(
hit_inner_hits = Some(Value::Object(combined));
}
- // Pull the real seq_no (and a placeholder primary_term of 1)
- // from the engine's version_map so multi-write docs surface
- // their actual sequence number instead of a synthetic 0.
+ // Real seq_no (and a placeholder primary_term of 1), read from
+ // the hit itself — resolved at construction time in the engine,
+ // from the same read as `_source`, not a later live re-lookup
+ // here (#440).
let (real_seq_no, real_primary_term) = if emit_seq_no {
- let sn = state
- .engine
- .get_index(&idx_name)
- .ok()
- .and_then(|idx| idx.lookup_seq_no(&h.id))
- .unwrap_or(hit_idx as u64);
+ let sn = h.seq_no.unwrap_or(hit_idx as u64);
(Some(sn), Some(1u64))
} else {
(None, None)
@@ -13288,16 +13279,13 @@ async fn search_impl(
None
},
version: if emit_version {
- // Real per-doc `_version` via the shared resolver
- // (external-version map wins so reindexes with
+ // Real per-doc `_version`, read from the hit itself —
+ // resolved at construction time in the engine (external-
+ // version map wins so reindexes with
// `version_type=external[_gte]` echo the caller's
- // value, then the engine write counter).
- state
- .engine
- .get_index(&idx_name)
- .ok()
- .and_then(|idx| idx.lookup_version(&h.id))
- .or(Some(1))
+ // value, then the engine write counter), not a later
+ // live re-lookup here (#440).
+ h.version.or(Some(1))
} else {
None
},
@@ -14263,6 +14251,11 @@ async fn search_impl(
hits: snapshot,
position: scroll_page_size,
page_size: scroll_page_size,
+ // Captured once at open time (real ES semantics — see the
+ // field's doc comment) from the same flags that gated this
+ // first page's own `_seq_no`/`_version` emission above.
+ seq_no_primary_term: emit_seq_no,
+ version: emit_version,
created: now,
keep_alive,
expires_at: now + keep_alive,
@@ -14433,42 +14426,7 @@ async fn search_impl(
if any_disabled_seqno {
if let Some(hits) = response_body.pointer_mut("/hits/hits") {
if let Some(arr) = hits.as_array_mut() {
- for hit in arr.iter_mut() {
- let ix = hit
- .get("_index")
- .and_then(Value::as_str)
- .unwrap_or("")
- .to_string();
- let disabled = state
- .engine
- .index_settings
- .get(&ix)
- .map(|v| {
- let s = v.clone();
- let as_bool = |val: &Value| {
- val.as_bool().unwrap_or_else(|| {
- val.as_str().map(|x| x == "true").unwrap_or(false)
- })
- };
- s.pointer("/index/disable_sequence_numbers")
- .map(as_bool)
- .unwrap_or(false)
- || s.get("index")
- .and_then(|i| i.get("index.disable_sequence_numbers"))
- .map(as_bool)
- .unwrap_or(false)
- || s.get("index.disable_sequence_numbers")
- .map(as_bool)
- .unwrap_or(false)
- })
- .unwrap_or(false);
- if disabled {
- if let Some(obj) = hit.as_object_mut() {
- obj.insert("_seq_no".into(), json!(-2i64));
- obj.insert("_primary_term".into(), json!(0));
- }
- }
- }
+ apply_disable_seqno_sentinel(&state, arr);
}
}
}
@@ -19834,17 +19792,34 @@ pub async fn search_with_scroll(
// Index name stays paired with its hit (#414).
let hits_only: Vec<(String, xerj_query::executor::Hit)> = all_hits.clone();
- // Return first page.
+ // Control flags for meta-field emission — mirrors `search_impl`'s
+ // `emit_seq_no`/`emit_version` (es_compat.rs ~11349).
+ let emit_seq_no = body.seq_no_primary_term.unwrap_or(false);
+ let emit_version = body.version.unwrap_or(false);
+
+ // Return first page. `_seq_no`/`_version` are read from the hit
+ // itself — resolved at construction time in the engine, from the
+ // same read as `_source` — not hardcoded, and omitted entirely
+ // when not requested (#428/#440).
let first_page: Vec<EsHit> = all_hits
.iter()
.take(page_size)
- .map(|(idx_name, h)| EsHit {
+ .enumerate()
+ .map(|(hit_idx, (idx_name, h))| EsHit {
index: idx_name.clone(),
id: h.id.clone(),
score: Some(h.score as f64),
- version: Some(1),
- seq_no: Some(0),
- primary_term: Some(1),
+ version: if emit_version {
+ h.version.or(Some(1))
+ } else {
+ None
+ },
+ seq_no: if emit_seq_no {
+ Some(h.seq_no.unwrap_or(hit_idx as u64))
+ } else {
+ None
+ },
+ primary_term: if emit_seq_no { Some(1) } else { None },
source: if h.source.is_null() {
None
} else {
@@ -19890,6 +19865,8 @@ pub async fn search_with_scroll(
hits: hits_only,
position: page_size,
page_size,
+ seq_no_primary_term: emit_seq_no,
+ version: emit_version,
created: now,
keep_alive,
expires_at: now + keep_alive,
@@ -20053,6 +20030,50 @@ pub async fn next_scroll_path(
.await
}
+/// Apply the ES `index.disable_sequence_numbers` sentinel — `_seq_no: -2`,
+/// `_primary_term: 0` — to every hit in `hits` whose own `_index` setting
+/// has it enabled. Shared by `search_impl` (plain `_search`) and
+/// `scroll_page_response` (#440: scroll had no equivalent at all, so a
+/// continuation page for such an index reported whatever `_seq_no` the
+/// snapshotted hit carried instead of the sentinel real ES uses).
+fn apply_disable_seqno_sentinel(state: &AppState, hits: &mut [Value]) {
+ for hit in hits.iter_mut() {
+ let ix = hit
+ .get("_index")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string();
+ let disabled = state
+ .engine
+ .index_settings
+ .get(&ix)
+ .map(|v| {
+ let s = v.clone();
+ let as_bool = |val: &Value| {
+ val.as_bool()
+ .unwrap_or_else(|| val.as_str().map(|x| x == "true").unwrap_or(false))
+ };
+ s.pointer("/index/disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index")
+ .and_then(|i| i.get("index.disable_sequence_numbers"))
+ .map(as_bool)
+ .unwrap_or(false)
+ || s.get("index.disable_sequence_numbers")
+ .map(as_bool)
+ .unwrap_or(false)
+ })
+ .unwrap_or(false);
+ if disabled {
+ if let Some(obj) = hit.as_object_mut() {
+ obj.insert("_seq_no".into(), json!(-2i64));
+ obj.insert("_primary_term".into(), json!(0));
+ }
+ }
+ }
+}
+
async fn scroll_page_response(
state: &AppState,
scroll_id: String,
@@ -20089,6 +20110,12 @@ async fn scroll_page_response(
let page_size = ctx.page_size.max(1);
let position = ctx.position;
let total = ctx.hits.len() as u64;
+ // Captured once at scroll-open time (real ES semantics) — see
+ // `ScrollContext::seq_no_primary_term`/`::version`'s doc
+ // comments. Read into locals before the per-hit closure below
+ // borrows `ctx.hits`.
+ let emit_seq_no = ctx.seq_no_primary_term;
+ let emit_version = ctx.version;
// Detect whether the initial search sorted by a non-score key;
// in that case `max_score` must be null on scroll pages too.
@@ -20098,19 +20125,34 @@ async fn scroll_page_response(
.map(|(_, h)| !h.sort.is_empty())
.unwrap_or(false);
+ // `_seq_no`/`_version` are read from each hit itself — resolved
+ // at construction time in the engine, from the same read as
+ // `_source`, snapshotted into `ctx.hits` at scroll-open (a
+ // single request-duration window, never re-read live) — not
+ // hardcoded, and omitted when the flag wasn't requested
+ // (#428/#440).
let page_hits: Vec<EsHit> = ctx
.hits
.iter()
.skip(position)
.take(page_size)
- .map(|(hit_index, h)| EsHit {
+ .enumerate()
+ .map(|(hit_idx, (hit_index, h))| EsHit {
// Per-hit index, not the context-level one (#414).
index: hit_index.clone(),
id: h.id.clone(),
score: Some(h.score as f64),
- version: Some(1),
- seq_no: Some(0),
- primary_term: Some(1),
+ version: if emit_version {
+ h.version.or(Some(1))
+ } else {
+ None
+ },
+ seq_no: if emit_seq_no {
+ Some(h.seq_no.unwrap_or(hit_idx as u64))
+ } else {
+ None
+ },
+ primary_term: if emit_seq_no { Some(1) } else { None },
source: if h.source.is_null() {
None
} else {
@@ -20148,6 +20190,51 @@ async fn scroll_page_response(
} else {
page_hits.first().and_then(|h| h.score)
};
+ let mut hits_json: Vec<Value> = page_hits
+ .iter()
+ .map(|h| {
+ let mut o = serde_json::Map::new();
+ o.insert("_index".to_string(), Value::String(h.index.clone()));
+ o.insert("_id".to_string(), Value::String(h.id.clone()));
+ o.insert(
+ "_score".to_string(),
+ match h.score {
+ Some(s) => json!(s),
+ None => Value::Null,
+ },
+ );
+ // Previously always omitted regardless of `version`/
+ // `seq_no_primary_term` — `EsHit` carried them but
+ // this hand-built map never read them out (#428).
+ if let Some(v) = h.version {
+ o.insert("_version".to_string(), json!(v));
+ }
+ if let Some(sn) = h.seq_no {
+ o.insert("_seq_no".to_string(), json!(sn));
+ }
+ if let Some(pt) = h.primary_term {
+ o.insert("_primary_term".to_string(), json!(pt));
+ }
+ if let Some(src) = &h.source {
+ o.insert("_source".to_string(), src.clone());
+ }
+ if let Some(sort) = &h.sort {
+ o.insert("sort".to_string(), Value::Array(sort.clone()));
+ }
+ if let Some(fields) = &h.fields {
+ o.insert(
+ "fields".to_string(),
+ serde_json::to_value(fields).unwrap_or(Value::Null),
+ );
+ }
+ Value::Object(o)
+ })
+ .collect();
+ // `index.disable_sequence_numbers` sentinel (`_seq_no: -2`,
+ // `_primary_term: 0`) — scroll previously had no equivalent of
+ // `search_impl`'s handling at all (#440).
+ apply_disable_seqno_sentinel(state, &mut hits_json);
+
let mut resp = json!({
"_scroll_id": scroll_id,
"took": 0u64,
@@ -20156,28 +20243,7 @@ async fn scroll_page_response(
"hits": {
"total": { "value": total, "relation": "eq" },
"max_score": max_score,
- "hits": page_hits.iter().map(|h| {
- let mut o = serde_json::Map::new();
- o.insert("_index".to_string(), Value::String(h.index.clone()));
- o.insert("_id".to_string(), Value::String(h.id.clone()));
- o.insert("_score".to_string(), match h.score {
- Some(s) => json!(s),
- None => Value::Null,
- });
- if let Some(src) = &h.source {
- o.insert("_source".to_string(), src.clone());
- }
- if let Some(sort) = &h.sort {
- o.insert("sort".to_string(), Value::Array(sort.clone()));
- }
- if let Some(fields) = &h.fields {
- o.insert(
- "fields".to_string(),
- serde_json::to_value(fields).unwrap_or(Value::Null),
- );
- }
- Value::Object(o)
- }).collect::<Vec<_>>()
+ "hits": hits_json
}
});
@@ -20214,6 +20280,8 @@ mod passage_scroll_tests {
id: id.to_string(),
score: 1.0,
source: json!({"company": "ACME"}),
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -20244,6 +20312,8 @@ mod passage_scroll_tests {
],
position: 1,
page_size: 1,
+ seq_no_primary_term: false,
+ version: false,
created: now,
keep_alive: Duration::from_secs(60),
expires_at: now + Duration::from_secs(60),
@@ -20300,6 +20370,8 @@ mod passage_scroll_tests {
],
position: 1,
page_size: 1,
+ seq_no_primary_term: false,
+ version: false,
created: now,
keep_alive: Duration::from_secs(60),
expires_at: now + Duration::from_secs(60),
@@ -20352,6 +20424,8 @@ mod passage_scroll_tests {
],
position: 1,
page_size: 1,
+ seq_no_primary_term: false,
+ version: false,
created: now,
keep_alive: Duration::from_secs(60),
expires_at: now + Duration::from_secs(60),
diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rs
index 8045da7ef..1289e6719 100644
--- a/engine/crates/xerj-api/src/native.rs
+++ b/engine/crates/xerj-api/src/native.rs
@@ -97,6 +97,8 @@ mod passage_wire_tests {
id: "report-page".to_string(),
score: 0.91,
source: serde_json::json!({"company": "ACME"}),
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
diff --git a/engine/crates/xerj-api/tests/scroll_seq_no_torn_read.rs b/engine/crates/xerj-api/tests/scroll_seq_no_torn_read.rs
new file mode 100644
index 000000000..f3fec5478
--- /dev/null
+++ b/engine/crates/xerj-api/tests/scroll_seq_no_torn_read.rs
@@ -0,0 +1,327 @@
+//! Issue #440: `_seq_no`/`_version` on a scroll page must come from the SAME
+//! read that produced `_source`, not a later pass over the version map.
+//!
+//! Two repair attempts on PR #431 were refuted by live reproduction:
+//! 1. resolve live at render time — window is the whole scroll lifetime.
+//! 2. resolve once at scroll-open time, from a context-level index lookup
+//! — narrower, but still a SECOND read, so a write landing between the
+//! `_source` read and the `_seq_no` read pairs a stale body with a live
+//! sequence number. Since `_seq_no` is fed back as `if_seq_no`, that is
+//! a silent lost update, not a cosmetic staleness.
+//!
+//! The fix threads `seq_no`/`version` onto `xerj_query::executor::Hit`
+//! itself, populated in the engine at the exact point `_source` is read.
+//! These tests exercise that guarantee from the outside, through the real
+//! HTTP routes — the in-module unit tests only ever injected a synthetic
+//! `ScrollContext` with fabricated `Hit`s, which is exactly why the original
+//! bug (and PR #431's two subsequent regressions) had no version-map entry
+//! to expose it.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn json_req(
+ app: &axum::Router,
+ method: &str,
+ path: &str,
+ body: Value,
+) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let v: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ (status, v)
+}
+
+async fn create_index(app: &axum::Router, index: &str) {
+ let (st, body) = json_req(
+ app,
+ "PUT",
+ &format!("/{index}"),
+ json!({"mappings": {"properties": {"v": {"type": "long"}}}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "create {index}: {body}");
+}
+
+async fn refresh(app: &axum::Router, index: &str) {
+ let (st, body) = json_req(app, "POST", &format!("/{index}/_refresh"), Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "refresh {index}: {body}");
+}
+
+async fn bulk_index(app: &axum::Router, index: &str, ids: impl Iterator<Item = usize>) {
+ let mut bulk = String::new();
+ for i in ids {
+ bulk.push_str(&format!("{}\n", json!({"index": {"_id": i.to_string()}})));
+ bulk.push_str(&format!("{}\n", json!({"v": i})));
+ }
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method("POST")
+ .uri(format!("/{index}/_bulk"))
+ .header("content-type", "application/x-ndjson")
+ .body(Body::from(bulk))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::OK);
+}
+
+/// Multi-index scroll, ids colliding across indices (the reindex/CDC shape
+/// #440 measured against) — every hit's `_seq_no`/`_version` must agree with
+/// what a plain `_search` on the hit's OWN index reports for the same doc.
+///
+/// PR #431's own repair attempt #2 (resolve once at scroll-open, from a
+/// context-level index) failed exactly this scenario: 4 of 8 hits carried
+/// values read out of the OTHER index's version map. This test's structure
+/// — colliding ids, differing per-index write counts so `_seq_no` is not
+/// merely a page ordinal — reproduces the shape that caught it.
+#[tokio::test]
+async fn multi_index_scroll_seq_no_matches_plain_search_per_index() {
+ let (app, _dir) = app().await;
+ create_index(&app, "sq_a").await;
+ create_index(&app, "sq_b").await;
+
+ // Different write counts per index so `_seq_no` is not simply the page
+ // ordinal (which would let a wrong-index lookup pass unnoticed if the
+ // two indices' sequence numbers happened to line up).
+ bulk_index(&app, "sq_a", 0..4).await;
+ bulk_index(&app, "sq_a", 0..4).await; // re-index sq_a's docs once more
+ bulk_index(&app, "sq_b", 0..4).await;
+ refresh(&app, "sq_a").await;
+ refresh(&app, "sq_b").await;
+
+ let (st, first) = json_req(
+ &app,
+ "POST",
+ "/sq_a,sq_b/_search?scroll=2m",
+ json!({
+ "query": {"match_all": {}},
+ "size": 3,
+ "seq_no_primary_term": true,
+ "version": true,
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "scroll start: {first}");
+
+ let mut sid = first["_scroll_id"].as_str().map(str::to_string);
+ let mut page = first;
+ let mut scrolled: Vec<(String, String, i64, i64)> = Vec::new();
+
+ for _ in 0..16 {
+ let hits = page["hits"]["hits"].as_array().cloned().unwrap_or_default();
+ if hits.is_empty() {
+ break;
+ }
+ for h in &hits {
+ scrolled.push((
+ h["_index"].as_str().unwrap_or_default().to_string(),
+ h["_id"].as_str().unwrap_or_default().to_string(),
+ h["_seq_no"]
+ .as_i64()
+ .expect("scrolled hit must carry _seq_no"),
+ h["_version"]
+ .as_i64()
+ .expect("scrolled hit must carry _version"),
+ ));
+ }
+ let Some(id) = sid.clone() else { break };
+ let (st, next) = json_req(
+ &app,
+ "POST",
+ "/_search/scroll",
+ json!({"scroll": "2m", "scroll_id": id}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "scroll continuation: {next}");
+ sid = next["_scroll_id"].as_str().map(str::to_string).or(sid);
+ page = next;
+ }
+ assert_eq!(
+ scrolled.len(),
+ 8,
+ "must have scrolled every hit across both indices"
+ );
+
+ // Ground truth: a direct GET on the hit's OWN index — the same
+ // resolver (`lookup_seq_no`/`lookup_version`) `_mget`/plain `_search`
+ // use, isolated from any query-matching ambiguity.
+ for (idx, id, scrolled_seq, scrolled_version) in &scrolled {
+ let (st, truth) = json_req(&app, "GET", &format!("/{idx}/_doc/{id}"), Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "truth GET {idx}/{id}: {truth}");
+ assert_eq!(
+ truth["_seq_no"].as_i64(),
+ Some(*scrolled_seq),
+ "scrolled _seq_no for {idx}/{id} does not match a direct GET on its own index \
+ — a hit is carrying another index's version-map entry (#440)"
+ );
+ assert_eq!(
+ truth["_version"].as_i64(),
+ Some(*scrolled_version),
+ "scrolled _version for {idx}/{id} does not match a direct GET on its own index"
+ );
+ }
+}
+
+/// The direct #440 repro: a document already captured in the scroll
+/// snapshot is updated AFTER the scroll opens but BEFORE its continuation
+/// page is fetched. The scrolled page must report the SNAPSHOT's
+/// `_seq_no`/`_source` pair — not a live `_seq_no` beside a stale
+/// `_source`, which is exactly the torn read that lets a caller's
+/// `if_seq_no` CAS silently overwrite the concurrent update.
+#[tokio::test]
+async fn scroll_continuation_reports_the_snapshot_pair_not_a_torn_one() {
+ let (app, _dir) = app().await;
+ create_index(&app, "torn").await;
+ bulk_index(&app, "torn", 0..4).await;
+ refresh(&app, "torn").await;
+
+ // page_size 1 forces every hit onto its own continuation page.
+ let (st, first) = json_req(
+ &app,
+ "POST",
+ "/torn/_search?scroll=5m",
+ json!({
+ "query": {"match_all": {}},
+ "size": 1,
+ "sort": [{"v": "asc"}],
+ "seq_no_primary_term": true,
+ "version": true,
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "scroll start: {first}");
+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();
+ let page1_id = first["hits"]["hits"][0]["_id"]
+ .as_str()
+ .unwrap()
+ .to_string();
+ let page1_seq = first["hits"]["hits"][0]["_seq_no"]
+ .as_i64()
+ .expect("_seq_no on page 1");
+
+ // Concurrent write AFTER the scroll snapshot: bump doc 1's `v` and its
+ // real seq_no/version. If the scroll re-derives against the LIVE
+ // index, page 2 (doc "1") would report this write's seq_no next to
+ // page 1's already-returned doc (id "0") — but the direct exposure is
+ // doc "1" itself, captured below.
+ let (st, upd) = json_req(&app, "PUT", "/torn/_doc/1", json!({"v": 999})).await;
+ assert_eq!(st, StatusCode::OK, "concurrent update: {upd}");
+ let live_seq_after_update = upd["_seq_no"].as_i64().expect("_seq_no on update");
+
+ // Continue the scroll to the page holding doc "1" (sorted by v asc,
+ // doc "1" is the second hit — id "0" was v=0, id "1" was v=1 pre-update).
+ let (st, page2) = json_req(
+ &app,
+ "POST",
+ "/_search/scroll",
+ json!({"scroll": "5m", "scroll_id": sid}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "scroll continuation: {page2}");
+ let hit = &page2["hits"]["hits"][0];
+ assert_eq!(
+ hit["_id"], "1",
+ "page 2 must be the doc updated concurrently"
+ );
+ let scrolled_seq = hit["_seq_no"].as_i64().expect("_seq_no on page 2");
+ let scrolled_source_v = hit["_source"]["v"].as_i64();
+
+ assert_ne!(
+ scrolled_seq, live_seq_after_update,
+ "scrolled page reported the LIVE post-update _seq_no ({live_seq_after_update}) — \
+ a caller round-tripping this as if_seq_no would CAS successfully over the \
+ concurrent write, a silent lost update (#440)"
+ );
+ assert_eq!(
+ scrolled_source_v,
+ Some(1),
+ "scrolled _source must be the SNAPSHOT body (v=1, pre-update), not the live one — \
+ found {scrolled_source_v:?}"
+ );
+ assert_ne!(
+ page1_seq, scrolled_seq,
+ "sanity: two different docs must not share a _seq_no"
+ );
+ let _ = page1_id;
+}
+
+/// #440's "adjacent, smaller, separate" gap: `disable_sequence_numbers`
+/// applies its `-2`/`0` sentinel in `search_impl`'s response body but
+/// `scroll_page_response` had no equivalent at all — a scroll continuation
+/// page for such an index reported whatever `_seq_no` the snapshotted hit
+/// carried (or, pre-fix, a hardcoded `0`) instead of ES's `-2` sentinel.
+#[tokio::test]
+async fn scroll_continuation_honours_disable_sequence_numbers() {
+ let (app, _dir) = app().await;
+ let (st, body) = json_req(
+ &app,
+ "PUT",
+ "/noseq",
+ json!({
+ "settings": {"index": {"disable_sequence_numbers": true}},
+ "mappings": {"properties": {"v": {"type": "long"}}}
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "create noseq: {body}");
+ bulk_index(&app, "noseq", 0..3).await;
+ refresh(&app, "noseq").await;
+
+ let (st, first) = json_req(
+ &app,
+ "POST",
+ "/noseq/_search?scroll=2m",
+ json!({"query": {"match_all": {}}, "size": 1, "sort": [{"v": "asc"}]}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "scroll start: {first}");
+ let sid = first["_scroll_id"].as_str().expect("scroll id").to_string();
+
+ let (st, page2) = json_req(
+ &app,
+ "POST",
+ "/_search/scroll",
+ json!({"scroll": "2m", "scroll_id": sid}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "scroll continuation: {page2}");
+ let hit = &page2["hits"]["hits"][0];
+ assert_eq!(
+ hit["_seq_no"].as_i64(),
+ Some(-2),
+ "continuation page for a disable_sequence_numbers index must carry the -2 \
+ sentinel, got: {hit}"
+ );
+ assert_eq!(
+ hit["_primary_term"].as_i64(),
+ Some(0),
+ "continuation page for a disable_sequence_numbers index must carry primary_term 0"
+ );
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 10657b4b7..d0ab8b6e8 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -161,6 +161,15 @@ pub struct ScrollContext {
pub hits: Vec<(String, xerj_query::executor::Hit)>,
pub position: usize,
pub page_size: usize,
+ /// Whether to emit `_seq_no`/`_primary_term` on every page. Real ES
+ /// fixes this at the scroll-opening request and does not re-read it
+ /// from continuation bodies — the flag governs the whole scroll's
+ /// lifetime, not each page — so it's captured once here, not re-parsed
+ /// per continuation.
+ pub seq_no_primary_term: bool,
+ /// Whether to emit `_version` on every page. Same once-at-open semantics
+ /// as `seq_no_primary_term`.
+ pub version: bool,
pub created: Instant,
/// The keep-alive window last requested for this context. A
/// continuation without an explicit `scroll` parameter re-arms the
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 4c72dea51..47fa3e251 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -10671,6 +10671,7 @@ impl Index {
generated_embedding_companion_fields(&schema.schema)
};
Some(knn_result_from_scored(
+ self,
request,
field,
scored,
@@ -11176,6 +11177,7 @@ impl Index {
generated_embedding_companion_fields(&schema.schema)
};
let mut result = knn_result_from_scored(
+ self,
request,
field,
scored,
@@ -11270,6 +11272,7 @@ impl Index {
generated_embedding_companion_fields(&schema.schema)
};
let mut result = knn_result_from_scored(
+ self,
request,
"",
merged,
@@ -12235,15 +12238,27 @@ impl Index {
let hits: Vec<Hit> = scored
.into_iter()
- .map(|(id, score, source)| Hit {
- id,
- score,
- source,
- sort: Vec::new(),
- explain: None,
- highlight: None,
- matched_queries: Vec::new(),
- passage: None,
+ .map(|(id, score, source)| {
+ // Same read the final page's `_source` is served from
+ // (post-truncate, post-filter) — not a later pass (#440).
+ let (seq_no, version) = self
+ .store
+ .version_map
+ .get(&id)
+ .map(|ver| self.hit_seq_version(&id, &ver))
+ .unwrap_or((None, None));
+ Hit {
+ id,
+ score,
+ source,
+ seq_no,
+ version,
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ }
})
.collect();
@@ -12638,6 +12653,32 @@ impl Index {
/// The internal WAL counter starts at 1 so the first WAL op has seq=1,
/// but ES semantics number seq_no from 0 per shard. We subtract 1 to
/// match the ES wire format.
+ /// Same transform as [`Self::lookup_seq_no`]/[`Self::lookup_version`],
+ /// but from a `VersionEntry` the caller already has in scope — for
+ /// building a `Hit` inline with the doc's version-map entry already in
+ /// hand, so `_seq_no`/`_version` come from the SAME read as `_source`,
+ /// never a later pass over the version map (#440: a second pass lets a
+ /// concurrent write pair a stale `_source` with a live `_seq_no`, which
+ /// is a silent lost update once a caller round-trips it as `if_seq_no`).
+ fn hit_seq_version(
+ &self,
+ id: &str,
+ ver: &xerj_storage::version_map::VersionEntry,
+ ) -> (Option<u64>, Option<u64>) {
+ if ver.deleted {
+ return (None, None);
+ }
+ (
+ Some(ver.seq_no.saturating_sub(1)),
+ Some(
+ self.external_versions
+ .get(id)
+ .map(|v| *v)
+ .unwrap_or(ver.version),
+ ),
+ )
+ }
+
pub fn lookup_seq_no(&self, id: &str) -> Option<u64> {
self.store.version_map.get(id).and_then(|entry| {
if entry.deleted {
@@ -14470,9 +14511,16 @@ impl Index {
if let Some(src) = self.memtable.get_doc_source_as_value(&hit.id) {
hit.source = src;
}
+ // Resolved at the SAME moment as the deferred
+ // `source` above, not a later pass (#440).
+ if let Some(ver) = self.store.version_map.get(&hit.id) {
+ let (seq_no, version) = self.hit_seq_version(&hit.id, &ver);
+ hit.seq_no = seq_no;
+ hit.version = version;
+ }
}
seen_ids.insert(hit.id.clone());
- let seq = self.lookup_seq_no(&hit.id).unwrap_or(u64::MAX);
+ let seq = hit.seq_no.unwrap_or(u64::MAX);
topk.offer(hit, seq, self);
}
} else if all_hits.len() < materialisation_limit && !seen_ids.contains(&hit.id)
@@ -14509,12 +14557,22 @@ impl Index {
if !rejected {
let source = src_arc.map(|a| (*a).clone()).unwrap_or(Value::Null);
seen_ids.insert(id.clone());
- let seq = self.lookup_seq_no(&id).unwrap_or(u64::MAX);
+ // Same read `source` came from — not a later
+ // pass (#440) — and reused below for ranking.
+ let (seq_no, version) = self
+ .store
+ .version_map
+ .get(&id)
+ .map(|ver| self.hit_seq_version(&id, &ver))
+ .unwrap_or((None, None));
+ let seq = seq_no.unwrap_or(u64::MAX);
topk.offer(
Hit {
id,
score: 1.0,
source,
+ seq_no,
+ version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -14534,6 +14592,13 @@ impl Index {
id,
score: 1.0,
source: Value::Null,
+ // Deferred with `source`: `fill_memtable_sources`
+ // resolves both from the SAME read once this
+ // hit is known to be on the returned page
+ // (#440) — not here, where most candidates
+ // never make the final page.
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -15022,6 +15087,10 @@ impl Index {
id: doc_id,
score,
source: Value::Null, // filled in below
+ // Deferred with `source` — see the deferred-`Hit`
+ // comment in `admit_hit!`'s unsorted `count` arm (#440).
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -15147,10 +15216,24 @@ impl Index {
}
}
let score = score_doc(&source, &doc_id);
+ // Same read `source` (above) is cloned from — not
+ // a later pass (#440). NOT the tuple's own
+ // `seq_no` (that's the memtable's raw arrival
+ // ordinal, used only for the `#191` competitive
+ // tie-break above — a different number from the
+ // ES-wire `_seq_no`/`_version` this resolves).
+ let (hit_seq_no, hit_version) = self
+ .store
+ .version_map
+ .get(&doc_id)
+ .map(|ver| self.hit_seq_version(&doc_id, &ver))
+ .unwrap_or((None, None));
let hit = Hit {
id: doc_id,
score,
source: (*source).clone(),
+ seq_no: hit_seq_no,
+ version: hit_version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -16316,6 +16399,14 @@ impl Index {
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
+ // Resolved here, from the SAME
+ // version-map entry the
+ // deleted/stale check below
+ // reads, so `_seq_no`/`_version`
+ // never diverge from `source`
+ // (#440).
+ let mut hit_seq_no = None;
+ let mut hit_version = None;
if let Some(ver) = self.store.version_map.get(&id) {
// Tombstoned, or SUPERSEDED —
// the live version resides in
@@ -16336,6 +16427,9 @@ impl Index {
{
continue;
}
+ let (s, v) = self.hit_seq_version(&id, &ver);
+ hit_seq_no = s;
+ hit_version = v;
}
// Dedup against memtable/earlier
// segments WITHOUT touching
@@ -16351,6 +16445,8 @@ impl Index {
id,
score: sh.score,
source,
+ seq_no: hit_seq_no,
+ version: hit_version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -16358,8 +16454,7 @@ impl Index {
passage: None,
};
if let Some(topk) = sort_topk.as_mut() {
- let seq =
- self.lookup_seq_no(&hit.id).unwrap_or(u64::MAX);
+ let seq = hit_seq_no.unwrap_or(u64::MAX);
topk.offer(hit, seq, self);
} else {
all_hits.push(hit);
@@ -19140,6 +19235,19 @@ impl Index {
// happen for FTS hits from the active memtable that were
// found by score but whose source was not cloned inline.
}
+ // Catch-all for whichever construction sites deliberately
+ // left `seq_no`/`version` unresolved alongside a deferred
+ // `source` (#440) — resolved here, at the SAME moment as the
+ // `source` fill above, only for hits that reach this final,
+ // already-bounded page. A no-op for hits already populated
+ // at construction time (the overwhelming majority).
+ if h.seq_no.is_none() {
+ if let Some(ver) = self.store.version_map.get(&h.id) {
+ let (seq_no, version) = self.hit_seq_version(&h.id, &ver);
+ h.seq_no = seq_no;
+ h.version = version;
+ }
+ }
h
})
.collect()
@@ -20748,10 +20856,20 @@ impl Index {
if !seen.insert(id.clone()) {
continue;
}
+ // Same read the winner's `_source` is hydrated from
+ // (post-select_nth winners only) — not a later pass (#440).
+ let (seq_no, version) = self
+ .store
+ .version_map
+ .get(id)
+ .map(|ver| self.hit_seq_version(id, &ver))
+ .unwrap_or((None, None));
hits.push(Hit {
id: id.clone(),
score: 1.0,
source: (**source).clone(),
+ seq_no,
+ version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -20777,10 +20895,20 @@ impl Index {
continue;
}
let source = doc.get("_source").cloned().unwrap_or(doc);
+ // Same read `source` was just parsed from — not a later
+ // pass (#440).
+ let (seq_no, version) = self
+ .store
+ .version_map
+ .get(&id)
+ .map(|ver| self.hit_seq_version(&id, &ver))
+ .unwrap_or((None, None));
hits.push(Hit {
id,
score: 1.0,
source,
+ seq_no,
+ version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -22359,10 +22487,22 @@ impl Index {
continue;
}
let source = doc.get("_source").cloned().unwrap_or(doc);
+ // Same read `source` was just parsed from — not a later pass
+ // (#440). Not `_seq` (discarded above): that's the pre-hydration
+ // ranking key from `seg_seqs`, a different number from the
+ // ES-wire `_seq_no`/`_version` this resolves.
+ let (seq_no, version) = self
+ .store
+ .version_map
+ .get(&id)
+ .map(|ver| self.hit_seq_version(&id, &ver))
+ .unwrap_or((None, None));
hits.push(Hit {
id,
score,
source,
+ seq_no,
+ version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -23564,6 +23704,11 @@ impl Index {
Err(_) => continue,
};
let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");
+ // Captured from the SAME version-map entry the
+ // deleted/superseded check reads, so `_seq_no`/`_version` never
+ // diverge from the `source` parsed above (#440).
+ let mut hit_seq_no = None;
+ let mut hit_version = None;
if let Some(ver) = self.store.version_map.get(id_ref) {
if ver.deleted {
continue;
@@ -23574,6 +23719,9 @@ impl Index {
continue;
}
}
+ let (s, v) = self.hit_seq_version(id_ref, &ver);
+ hit_seq_no = s;
+ hit_version = v;
}
*total_count += 1;
if seen_ids.contains(id_ref) {
@@ -23594,12 +23742,14 @@ impl Index {
}
let id = id_ref.to_string();
seen_ids.insert(id.clone());
- let seq = self.lookup_seq_no(&id).unwrap_or(u64::MAX);
+ let seq = hit_seq_no.unwrap_or(u64::MAX);
topk.offer_keyed(
Hit {
id,
score,
source: source_ref.clone(),
+ seq_no: hit_seq_no,
+ version: hit_version,
sort: key,
explain: None,
highlight: None,
@@ -23674,6 +23824,11 @@ impl Index {
Err(_) => continue,
};
let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");
+ // Captured from the SAME version-map entry the
+ // deleted/superseded check reads, so `_seq_no`/`_version` never
+ // diverge from the `source` parsed above (#440).
+ let mut hit_seq_no = None;
+ let mut hit_version = None;
if let Some(ver) = self.store.version_map.get(id_ref) {
if ver.deleted {
continue;
@@ -23684,6 +23839,9 @@ impl Index {
continue;
}
}
+ let (s, v) = self.hit_seq_version(id_ref, &ver);
+ hit_seq_no = s;
+ hit_version = v;
}
// Re-test the full query — the pre-filter is only a superset.
let matched = if is_match_all {
@@ -23731,6 +23889,8 @@ impl Index {
id,
score,
source,
+ seq_no: hit_seq_no,
+ version: hit_version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -24202,6 +24362,11 @@ impl Index {
};
let id_ref = doc.get("_id").and_then(Value::as_str).unwrap_or("");
+ // Captured from the SAME version-map entry the
+ // deleted/superseded check reads, so `_seq_no`/`_version` never
+ // diverge from the `source` parsed above (#440).
+ let mut hit_seq_no = None;
+ let mut hit_version = None;
if let Some(ver) = self.store.version_map.get(id_ref) {
if ver.deleted {
continue;
@@ -24216,6 +24381,9 @@ impl Index {
continue;
}
}
+ let (s, v) = self.hit_seq_version(id_ref, &ver);
+ hit_seq_no = s;
+ hit_version = v;
}
let matched = if is_match_all {
@@ -24291,12 +24459,14 @@ impl Index {
}
let id: String = id_ref.to_string();
seen_ids.insert(id.clone());
- let seq = self.lookup_seq_no(&id).unwrap_or(u64::MAX);
+ let seq = hit_seq_no.unwrap_or(u64::MAX);
topk.offer_keyed(
Hit {
id,
score,
source: source_ref.clone(),
+ seq_no: hit_seq_no,
+ version: hit_version,
sort: key,
explain: None,
highlight: None,
@@ -24342,6 +24512,8 @@ impl Index {
id,
score,
source,
+ seq_no: hit_seq_no,
+ version: hit_version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -25855,6 +26027,8 @@ mod source_filter_tests {
id: id.to_string(),
score,
source,
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -26054,6 +26228,8 @@ mod savings_tests {
id: id.to_string(),
score: 1.0,
source,
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -29597,6 +29773,7 @@ fn apply_lexical_passages(hits: Vec<Hit>, query: &QueryNode) -> Vec<Hit> {
/// `from`/`size`, and shape the response. Centralised so the exact and
/// approximate paths cannot drift on hits format or total semantics.
fn knn_result_from_scored(
+ index: &Index,
request: &SearchRequest,
vector_field: &str,
mut scored: Vec<(String, f32, Value, Option<u32>)>,
@@ -29650,10 +29827,21 @@ fn knn_result_from_scored(
.then(|| passage_match_from_source(&source, vector_field, ordinal))
.flatten();
strip_internal_passage_metadata(&mut source);
+ // Same read `source` was carried in from (already the bounded
+ // final page — `.skip/.take` above already applied) — not a
+ // later pass (#440).
+ let (seq_no, version) = index
+ .store
+ .version_map
+ .get(&id)
+ .map(|ver| index.hit_seq_version(&id, &ver))
+ .unwrap_or((None, None));
Hit {
id,
score,
source,
+ seq_no,
+ version,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -42570,6 +42758,8 @@ mod lexical_passage_tests {
id: "doc-1".into(),
score: 1.0,
source: json!({"body": body, "title": "unrelated", "page": 3}),
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -42626,6 +42816,8 @@ mod lexical_passage_tests {
id: "doc-1".into(),
score: 1.0,
source: json!({"body": body, "defs": "addReplyNull bulk"}),
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -42661,6 +42853,8 @@ mod lexical_passage_tests {
id: "doc-1".into(),
score: 1.0,
source: json!({"body": body, "language": "rust", "symbol_count": 1}),
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -42702,6 +42896,8 @@ mod lexical_passage_tests {
id: "doc-1".into(),
score: 1.0,
source: json!({"body": body}),
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -42714,6 +42910,8 @@ mod lexical_passage_tests {
// Same body, but with a stray `language` field and no
// `symbol_count` — not autoindex's code-doc shape.
source: json!({"body": body, "language": "rust"}),
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
@@ -42760,6 +42958,8 @@ mod lexical_passage_tests {
id: "doc-1".into(),
score: 1.0,
source: json!({"body": "needle here\n"}),
+ seq_no: None,
+ version: None,
sort: Vec::new(),
explain: None,
highlight: None,
diff --git a/engine/crates/xerj-engine/tests/search_context_ttl.rs b/engine/crates/xerj-engine/tests/search_context_ttl.rs
index d9cbefad2..8892538c8 100644
--- a/engine/crates/xerj-engine/tests/search_context_ttl.rs
+++ b/engine/crates/xerj-engine/tests/search_context_ttl.rs
@@ -36,6 +36,8 @@ fn scroll_ctx(expires_in: i64) -> ScrollContext {
hits: Vec::new(),
position: 0,
page_size: 10,
+ seq_no_primary_term: false,
+ version: false,
created: now,
keep_alive,
expires_at,
diff --git a/engine/crates/xerj-query/src/executor.rs b/engine/crates/xerj-query/src/executor.rs
index 2ae835208..d1ca6fd0a 100644
--- a/engine/crates/xerj-query/src/executor.rs
+++ b/engine/crates/xerj-query/src/executor.rs
@@ -50,6 +50,18 @@ pub struct Hit {
/// The document source fields (filtered per `_source`).
#[serde(rename = "_source")]
pub source: serde_json::Value,
+ /// ES-wire (0-based) sequence number, read from the SAME version-map
+ /// entry `source` was resolved from — never re-resolved later, so a
+ /// concurrent write can't pair a stale `source` with a live `seq_no`
+ /// (#440). `None` for a doc that is unknown/tombstoned at read time, or
+ /// when the deferred-hydration path hasn't reached this hit yet
+ /// (resolved in that case by whatever later fills `source`).
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub seq_no: Option<u64>,
+ /// Document version, from the same read as `seq_no`/`source`. Same
+ /// `None` semantics as `seq_no`.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub version: Option<u64>,
/// The sort key values for this hit (used by `search_after`).
#[serde(skip_serializing_if = "Vec::is_empty")]
pub sort: Vec<serde_json::Value>,
@@ -542,6 +554,8 @@ mod tests {
id: id.to_string(),
score,
source: serde_json::Value::Null,
+ seq_no: None,
+ version: None,
sort: vec![],
explain: None,
highlight: None,