AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(es-compat): real per-doc _version + ES seq_no wire convention for doc CRUD (RC4-W2 A1-A3)
xerj-org8cb41ae44b87
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 7254f6185..a825ced00 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -1572,7 +1572,12 @@ pub async fn index_doc_auto(match idx.index_document(None, doc).await {Ok(resp) => {state.metrics.record_doc_indexed(&index);- let er = EsDocResponse::created(&index, &resp.id, resp.seq_no);+ let er = EsDocResponse::created(+ &index,+ &resp.id,+ resp.version,+ resp.seq_no.saturating_sub(1),+ );(StatusCode::CREATED, Json(er)).into_response()}Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),@@ -1689,11 +1694,22 @@ pub async fn index_doc(// op_type=create means fail if document already exists.let is_create_op = params.op_type.as_deref() == Some("create");+ // The wire (ES) numbers seq_no from 0; the internal WAL counter from 1.+ // `if_seq_no` arrives in ES form → +1 before the engine CAS compare+ // (which runs on internal seqs), so a value read from GET/_search/_mget+ // or a previous write response round-trips correctly.+ let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));+if is_create_op {match idx.create_document(id.clone(), doc).await {Ok(resp) => {state.metrics.record_doc_indexed(&index);- let er = EsDocResponse::created(&index, &resp.id, resp.seq_no);+ let er = EsDocResponse::created(+ &index,+ &resp.id,+ resp.version,+ resp.seq_no.saturating_sub(1),+ );(StatusCode::CREATED, Json(er)).into_response()}Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),@@ -1708,7 +1724,7 @@ pub async fn index_doc(idx.index_document_with_version(Some(id.clone()),doc,- params.if_seq_no,+ if_seq_no_internal,params.if_primary_term,).await@@ -1717,7 +1733,7 @@ pub async fn index_doc(idx.index_document_with_version(Some(id.clone()),doc,- params.if_seq_no,+ if_seq_no_internal,params.if_primary_term,).await@@ -1726,12 +1742,17 @@ pub async fn index_doc(Ok(resp) => {state.metrics.record_doc_indexed(&index);let is_update = resp.result == "updated";- let mut er = EsDocResponse::created(&index, &resp.id, resp.seq_no);- er.version = resp.version;- let status = if is_update {- StatusCode::OK+ let wire_seq = resp.seq_no.saturating_sub(1);+ let (er, status) = if is_update {+ (+ EsDocResponse::updated(&index, &resp.id, resp.version, wire_seq),+ StatusCode::OK,+ )} else {- StatusCode::CREATED+ (+ EsDocResponse::created(&index, &resp.id, resp.version, wire_seq),+ StatusCode::CREATED,+ )};(status, Json(er)).into_response()}@@ -1793,7 +1814,12 @@ pub async fn create_doc(match idx.create_document(id.clone(), doc).await {Ok(resp) => {state.metrics.record_doc_indexed(&index);- let er = EsDocResponse::created(&index, &resp.id, resp.seq_no);+ let er = EsDocResponse::created(+ &index,+ &resp.id,+ resp.version,+ resp.seq_no.saturating_sub(1),+ );(StatusCode::CREATED, Json(er)).into_response()}Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),@@ -1837,7 +1863,14 @@ pub async fn get_doc(Ok(Some(source)) => {// Apply _source filtering based on query params.let filtered = apply_get_doc_source_filter(source, ¶ms);- let resp = EsGetResponse::found(&index, &id, 1, 1, filtered);+ // Real per-doc metadata (was hardcoded `_version: 1` /+ // `_seq_no: 1`, which broke read-then-CAS flows): same+ // resolvers `_mget` and `_search` hits use. `lookup_seq_no`+ // yields the ES wire form (0-based); `lookup_version` prefers+ // an external version when one was supplied.+ let version = idx.lookup_version(&id).unwrap_or(1);+ let seq_no = idx.lookup_seq_no(&id).unwrap_or(0);+ let resp = EsGetResponse::found(&index, &id, version, seq_no, filtered);Json(resp).into_response()}Ok(None) => {@@ -1913,24 +1946,55 @@ fn source_field_matches(field: &str, pattern: &str) -> bool {pub struct DeleteDocParams {/// `refresh=true|wait_for` — accepted without error; memtable is always visible.pub refresh: Option<String>,+ /// Optimistic concurrency: expected sequence number (ES wire form, 0-based).+ pub if_seq_no: Option<u64>,+ /// Optimistic concurrency: expected primary term (must accompany if_seq_no).+ pub if_primary_term: Option<u64>,}pub async fn delete_doc(State(state): State<AppState>,Path((index, id)): Path<(String, String)>,- Query(_params): Query<DeleteDocParams>,+ Query(params): Query<DeleteDocParams>,) -> impl IntoResponse {let idx = match state.engine.get_index(&index) {Ok(i) => i,Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),};- match idx.delete_document(&id).await {- Ok(_) => {- let seq_no = current_timestamp_micros();- let resp = EsDeleteDocResponse::deleted(&index, &id, seq_no);+ // Wire → internal seq translation, same convention as the index path.+ let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));++ match idx+ .delete_document_versioned(&id, if_seq_no_internal, params.if_primary_term)+ .await+ {+ // Live doc tombstoned → 200 `deleted` with the doc's REAL bumped+ // version and the tombstone's seq (ES wire form). Was: always+ // 200 `deleted` `_version: 1` with a timestamp-garbage seq_no,+ // even for missing docs and ignored CAS params.+ Ok(outcome) if outcome.found => {+ let resp = EsDeleteDocResponse::deleted(+ &index,+ &id,+ outcome.version,+ outcome.seq_no.saturating_sub(1),+ );Json(resp).into_response()}+ // Nothing to delete → ES answers 404 with a RESULT body (not an+ // error body): `result: "not_found"`, version bookkeeping still+ // advancing (live-verified against ES 8.13.4).+ Ok(outcome) => {+ let resp = EsDeleteDocResponse::not_found(+ &index,+ &id,+ outcome.version,+ outcome.seq_no.saturating_sub(1),+ );+ (StatusCode::NOT_FOUND, Json(resp)).into_response()+ }+ // CAS mismatch surfaces as 409 version_conflict_engine_exception.Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),}}@@ -7965,17 +8029,14 @@ pub async fn search(continue;}"_version" => {- // Echo the caller-supplied external `_version`- // (`version_type=external[_gte]`) from the same- // map the top-level and inner_hits `_version`- // paths use. Internal per-doc version increments- // aren't tracked yet, so freshly-indexed docs- // default to 1.+ // Real per-doc `_version` via the same+ // resolver GET/_mget use (external-version+ // map wins, then the engine write counter).let v = state.engine.get_index(&idx_name).ok()- .and_then(|idx| idx.external_versions.get(&h.id).map(|v| *v))+ .and_then(|idx| idx.lookup_version(&h.id)).unwrap_or(1);fmap.insert("_version".to_string(), Value::Array(vec![json!(v)]));continue;@@ -9045,15 +9106,14 @@ pub async fn search(hit_obj.insert("_score".to_string(), score);}if emit_version_ih {- // Look up the external version map so- // inner_hits reflect the caller-supplied- // `_version` just like top-level hits.+ // Real per-doc `_version` (external map+ // wins) so inner_hits match top-level hits.let id_str = m.get("_id").and_then(Value::as_str).unwrap_or("");let v = state.engine.get_index(&idx_name).ok()- .and_then(|idx| idx.external_versions.get(id_str).map(|v| *v))+ .and_then(|idx| idx.lookup_version(id_str)).unwrap_or(1);hit_obj.insert("_version".to_string(), json!(v));}@@ -9176,15 +9236,16 @@ pub async fn search(None},version: if emit_version {- // Prefer the external-version map (set by- // `version_type=external[_gte]`) so reindexes with- // explicit versions echo the caller's value.- let ext = state+ // Real per-doc `_version` via the shared resolver+ // (external-version map wins so reindexes with+ // `version_type=external[_gte]` echo the caller's+ // value, then the engine write counter).+ state.engine.get_index(&idx_name).ok()- .and_then(|idx| idx.external_versions.get(&h.id).map(|v| *v));- ext.or(Some(1))+ .and_then(|idx| idx.lookup_version(&h.id))+ .or(Some(1))} else {None},@@ -11524,7 +11585,7 @@ pub async fn update_doc(&index,&resp.id,resp.version,- resp.seq_no,+ resp.seq_no.saturating_sub(1),);Json(er).into_response()}@@ -11550,7 +11611,8 @@ pub async fn update_doc(let er = crate::responses::EsDocResponse::created(&index,&resp.id,- resp.seq_no,+ resp.version,+ resp.seq_no.saturating_sub(1),);Json(er).into_response()}@@ -11574,7 +11636,7 @@ pub async fn update_doc(&index,&resp.id,resp.version,- resp.seq_no,+ resp.seq_no.saturating_sub(1),);Json(er).into_response()}@@ -12104,13 +12166,15 @@ pub async fn mget(match idx.get_document(&req_doc.id).await {Ok(Some(source)) => {- // Real per-doc `_seq_no` — same source the `_search` hit- // metadata uses (`lookup_seq_no`); was hardcoded 0.+ // Real per-doc `_seq_no` / `_version` — same resolvers the+ // `_search` hit metadata and GET `_doc` use; were hardcoded+ // 0 / 1.let seq_no = idx.lookup_seq_no(&req_doc.id).unwrap_or(0);+ let version = idx.lookup_version(&req_doc.id).unwrap_or(1);docs.push(json!({"_index": req_doc.index,"_id": req_doc.id,- "_version": 1,+ "_version": version,"_seq_no": seq_no,"_primary_term": 1,"found": true,@@ -12189,13 +12253,14 @@ pub async fn mget_index(match state.engine.get_index(ix) {Ok(idx) => match idx.get_document(id).await {Ok(Some(source)) => {- // Real per-doc `_seq_no` (was hardcoded 0) — matches the- // `_search` hit metadata path.+ // Real per-doc `_seq_no` / `_version` (were hardcoded+ // 0 / 1) — matches the `_search` hit metadata path.let seq_no = idx.lookup_seq_no(id).unwrap_or(0);+ let version = idx.lookup_version(id).unwrap_or(1);docs.push(json!({"_index": ix,"_id": id,- "_version": 1,+ "_version": version,"_seq_no": seq_no,"_primary_term": 1,"found": true,diff --git a/engine/crates/xerj-api/src/responses.rs b/engine/crates/xerj-api/src/responses.rsindex 0eef97d01..3130c3e75 100644--- a/engine/crates/xerj-api/src/responses.rs+++ b/engine/crates/xerj-api/src/responses.rs@@ -129,11 +129,19 @@ pub struct EsDocResponse {}impl EsDocResponse {- pub fn created(index: impl Into<String>, id: impl Into<String>, seq_no: u64) -> Self {+ /// `result: "created"` — `version` is NOT always 1: a re-index after a+ /// delete reports `created` while `_version` continues from the+ /// tombstone (ES 8.13.4 live-verified).+ pub fn created(+ index: impl Into<String>,+ id: impl Into<String>,+ version: u64,+ seq_no: u64,+ ) -> Self {Self {index: index.into(),id: id.into(),- version: 1,+ version,result: "created".to_string(),shards: EsShards::single_success(),seq_no,@@ -160,18 +168,23 @@ impl EsDocResponse {}/// Response to `GET /{index}/_doc/{id}`.+///+/// The metadata fields are `Option` because ES omits `_version` /+/// `_seq_no` / `_primary_term` entirely on a not-found GET — the 404 body+/// is exactly `{"_index": ..., "_id": ..., "found": false}` (live-verified+/// against ES 8.13.4).#[derive(Debug, Serialize, Deserialize)]pub struct EsGetResponse {#[serde(rename = "_index")]pub index: String,#[serde(rename = "_id")]pub id: String,- #[serde(rename = "_version")]- pub version: u64,- #[serde(rename = "_seq_no")]- pub seq_no: u64,- #[serde(rename = "_primary_term")]- pub primary_term: u64,+ #[serde(rename = "_version", skip_serializing_if = "Option::is_none")]+ pub version: Option<u64>,+ #[serde(rename = "_seq_no", skip_serializing_if = "Option::is_none")]+ pub seq_no: Option<u64>,+ #[serde(rename = "_primary_term", skip_serializing_if = "Option::is_none")]+ pub primary_term: Option<u64>,pub found: bool,#[serde(rename = "_source", skip_serializing_if = "Option::is_none")]pub source: Option<Value>,@@ -188,9 +201,9 @@ impl EsGetResponse {Self {index: index.into(),id: id.into(),- version,- seq_no,- primary_term: 1,+ version: Some(version),+ seq_no: Some(seq_no),+ primary_term: Some(1),found: true,source: Some(source),}@@ -200,9 +213,9 @@ impl EsGetResponse {Self {index: index.into(),id: id.into(),- version: 0,- seq_no: 0,- primary_term: 0,+ version: None,+ seq_no: None,+ primary_term: None,found: false,source: None,}@@ -228,17 +241,42 @@ pub struct EsDeleteDocResponse {}impl EsDeleteDocResponse {- pub fn deleted(index: impl Into<String>, id: impl Into<String>, seq_no: u64) -> Self {+ pub fn deleted(+ index: impl Into<String>,+ id: impl Into<String>,+ version: u64,+ seq_no: u64,+ ) -> Self {Self {index: index.into(),id: id.into(),- version: 1,+ version,result: "deleted".to_string(),shards: EsShards::single_success(),seq_no,primary_term: 1,}}++ /// ES 404 body for a delete that found nothing: same shape as a+ /// successful delete but `result: "not_found"` (live-verified against+ /// ES 8.13.4 — the version bookkeeping still advances).+ pub fn not_found(+ index: impl Into<String>,+ id: impl Into<String>,+ version: u64,+ seq_no: u64,+ ) -> Self {+ Self {+ index: index.into(),+ id: id.into(),+ version,+ result: "not_found".to_string(),+ shards: EsShards::single_success(),+ seq_no,+ primary_term: 1,+ }+ }}// ═════════════════════════════════════════════════════════════════════════════diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex 080a76aaf..76623052d 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -454,7 +454,15 @@ pub async fn process_bulk_with_opts(.get("pipeline").and_then(Value::as_str).map(str::to_owned);- if_seq_no_per_item = meta.get("if_seq_no").and_then(Value::as_u64);+ // Wire → internal seq translation: ES numbers seq_no+ // from 0 on the wire, the internal WAL counter from 1.+ // The CAS compare (index_document_with_version) runs on+ // internal seqs, so a `_seq_no` captured from any read+ // or write response round-trips through bulk CAS.+ if_seq_no_per_item = meta+ .get("if_seq_no")+ .and_then(Value::as_u64)+ .map(|s| s.saturating_add(1));if_primary_term_per_item = meta.get("if_primary_term").and_then(Value::as_u64);routing_per_item = meta.get("routing")diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 160363ed3..1830fb468 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -149,6 +149,26 @@ pub struct IndexResponse {pub result: String,}+/// Outcome of a single-document delete (ES `DELETE /{index}/_doc/{id}`).+///+/// `found == true` → the doc was live and is now tombstoned; `seq_no` is+/// the tombstone's internal (1-based) WAL seq and `version` the bumped+/// per-doc version (ES semantics: deletes increment `_version`).+///+/// `found == false` → nothing was deleted (`result: not_found`). `version`+/// still reports the ES-visible version: an existing tombstone is bumped+/// and persisted (matching ES, where a not-found delete still versions the+/// tombstone), an unknown id reports 1. `seq_no` carries the tombstone's+/// seq when one exists, else the index's latest assigned seq (xerj does+/// not consume a new seq_no for a not-found delete — divergence from ES+/// in the reported VALUE only).+#[derive(Debug, Clone)]+pub struct DeleteDocOutcome {+ pub found: bool,+ pub seq_no: u64,+ pub version: u64,+}+/// Per-field encoding statistics, derived from `FieldAnalyzer`.#[derive(Debug, Clone, Serialize, Deserialize)]pub struct FieldEncodingInfo {@@ -1237,8 +1257,16 @@ impl Index {// Detect whether this is an overwrite (existing doc) vs a new// insert BEFORE writing to storage. Used to set the response// `result` field to "updated" / "created" per ES semantics so- // HTTP-layer 200/201 bulk status mapping works.- let existed_before = self.store.version_map.get(&doc_id).is_some();+ // HTTP-layer 200/201 bulk status mapping works. A tombstoned doc+ // counts as NOT existing: ES reports re-index-after-delete as+ // `created`/201 (while its `_version` still continues from the+ // tombstone).+ let existed_before = self+ .store+ .version_map+ .get(&doc_id)+ .map(|e| !e.deleted)+ .unwrap_or(false);// Write to storage WAL.let seq_no = self.store.index(&doc_id, source.clone())?;@@ -1251,8 +1279,22 @@ impl Index {mem.insert(doc_id.clone(), &source, &schema_guard.schema, seq_no);}- // Bump counter.- let version = self.doc_count.fetch_add(1, Ordering::Relaxed) + 1;+ // Bump counter (flush thresholds / stats bookkeeping only — NOT the+ // response `_version`, which is per-doc).+ self.doc_count.fetch_add(1, Ordering::Relaxed);++ // Real per-doc `_version`: `store.index` just recorded this write+ // in the version map (1 on first index, +1 per overwrite, deletes+ // included). The previous code reported the index-global doc_count+ // here, so any write to doc B inflated the next reported version+ // of doc A (live-verified: three PUTs to the same id after other+ // writes reported 4, 5, 6).+ let version = self+ .store+ .version_map+ .get(&doc_id)+ .map(|e| e.version)+ .unwrap_or(1);// Auto-evolve schema for new fields.// Fast path: skip the schema read lock every doc when schema is stable.@@ -4935,6 +4977,28 @@ impl Index {})}+ /// Look up the ES `_version` for a live document by id.+ ///+ /// The external-version map wins (docs written with+ /// `version_type=external[_gte]` must echo the caller's version, per+ /// ES), falling back to the engine-tracked per-doc write counter.+ /// Returns `None` when the doc is unknown or tombstoned.+ pub fn lookup_version(&self, id: &str) -> Option<u64> {+ let tracked = self.store.version_map.get(id).and_then(|entry| {+ if entry.deleted {+ None+ } else {+ Some(entry.version)+ }+ })?;+ Some(+ self.external_versions+ .get(id)+ .map(|v| *v)+ .unwrap_or(tracked),+ )+ }+/// Retrieve a document by its string ID.////// Checks the memtable first, then searches on-disk segments.@@ -5059,7 +5123,34 @@ impl Index {}/// Delete a document by ID.+ ///+ /// Thin compatibility wrapper over [`Self::delete_document_versioned`]+ /// (no optimistic-concurrency check). Returns whether a live document+ /// was found and tombstoned.pub async fn delete_document(&self, id: &str) -> Result<bool> {+ self.delete_document_versioned(id, None, None)+ .await+ .map(|outcome| outcome.found)+ }++ /// Delete a document by ID with ES delete semantics.+ ///+ /// When `if_seq_no` is supplied (internal 1-based form), the delete is+ /// conditional: the doc's current seq_no must match or a+ /// `VersionConflict` is returned — including when the doc does not+ /// exist (ES returns 409 for a conditional delete of a missing doc).+ /// xerj is single-shard with `primary_term` fixed at 1, so a supplied+ /// `if_primary_term` is accepted but only `if_seq_no` is compared+ /// (same treatment as `index_document_with_version`).+ ///+ /// Unconditional deletes of a missing doc return `found: false` with+ /// the ES-visible version bookkeeping described on [`DeleteDocOutcome`].+ pub async fn delete_document_versioned(+ &self,+ id: &str,+ if_seq_no: Option<u64>,+ if_primary_term: Option<u64>,+ ) -> Result<DeleteDocOutcome> {// Check write block.if self.is_write_blocked().await {return Err(EngineError::Common(xerj_common::XerjError::index_blocked(@@ -5069,6 +5160,32 @@ impl Index {}self.dataset_version.fetch_add(1, Ordering::Release);+ // Optimistic concurrency check (see index_document_with_version for+ // the if_primary_term rationale).+ let _ = if_primary_term;+ if let Some(expected_seq) = if_seq_no {+ match self.store.version_map.get(id) {+ Some(entry) if !entry.deleted => {+ if entry.seq_no != expected_seq {+ return Err(EngineError::Common(+ xerj_common::XerjError::version_conflict(+ id,+ expected_seq,+ entry.seq_no,+ ),+ ));+ }+ }+ // Missing or tombstoned doc + conditional delete → conflict+ // (ES: "required seqNo [N] ... but no document was found").+ _ => {+ return Err(EngineError::Common(+ xerj_common::XerjError::version_conflict(id, expected_seq, 0),+ ));+ }+ }+ }+// Check whether the document actually exists and is not already deleted.let is_live = self.store@@ -5087,10 +5204,28 @@ impl Index {let should_delete = is_live || in_memtable;if !should_delete {- return Ok(false);+ // ES `result: not_found` bookkeeping: a repeat delete of a+ // tombstoned doc still bumps (and persists) the tombstone's+ // version; a never-seen id reports version 1. No WAL entry or+ // seq_no is consumed for a not-found delete.+ let existing = self.store.version_map.get(id);+ let version = self+ .store+ .version_map+ .bump_tombstone_version(id)+ .unwrap_or(1);+ let seq_no = existing+ .map(|e| e.seq_no)+ .unwrap_or_else(|| self.store.current_seq_no().saturating_sub(1));+ return Ok(DeleteDocOutcome {+ found: false,+ seq_no,+ version,+ });}- let existed = self.store.delete(id)?.is_some();+ let deleted_seq = self.store.delete(id)?;+ let existed = deleted_seq.is_some();// Remove from memtable.{@@ -5123,7 +5258,36 @@ impl Index {if existed {self.doc_count.fetch_sub(1, Ordering::Relaxed);}- Ok(existed || in_memtable)++ let found = existed || in_memtable;+ if !found {+ // Raced away between the liveness check and store.delete —+ // report as not_found (same bookkeeping as the early return).+ let version = self+ .store+ .version_map+ .bump_tombstone_version(id)+ .unwrap_or(1);+ return Ok(DeleteDocOutcome {+ found: false,+ seq_no: self.store.current_seq_no().saturating_sub(1),+ version,+ });+ }++ // The tombstone entry now carries the bumped per-doc version.+ let version = self+ .store+ .version_map+ .get(id)+ .map(|e| e.version)+ .unwrap_or(1);+ let seq_no = deleted_seq.unwrap_or_else(|| self.store.current_seq_no().saturating_sub(1));+ Ok(DeleteDocOutcome {+ found: true,+ seq_no,+ version,+ })}/// Update a document by merging a partial document into the existing source.diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rsindex 20f7e6a56..de5685775 100644--- a/engine/crates/xerj-storage/src/index_store.rs+++ b/engine/crates/xerj-storage/src/index_store.rs@@ -1502,16 +1502,26 @@ impl IndexStore {post_finish(&meta)?;let prof_pf_us = t_pf.elapsed().as_micros();- // Update version map: point live docs at the new segment+ // Update version map: point live docs at the new segment.+ //+ // `repoint` (not `set`): the drained entries include superseded+ // duplicates (a doc overwritten within the same memtable+ // generation drains once per write), and a doc can be updated or+ // deleted concurrently after the drain. The old unconditional+ // `set` transiently clobbered the newer entry with the stale+ // copy's seq_no (and, with per-doc `_version` tracking, would+ // spuriously bump the version once per duplicate). `repoint`+ // only swaps the segment id when the entry still carries exactly+ // this seq_no — same-generation duplicates and post-drain+ // updates/deletes are left untouched.let t_vm = std::time::Instant::now();let segment_id_arc: std::sync::Arc<str> = std::sync::Arc::from(segment_id.as_str());for entry in &entries {if entry.source.is_some() {- self.version_map.set(+ self.version_map.repoint(&entry.doc_id,entry.seq_no,std::sync::Arc::clone(&segment_id_arc),- false,);}}diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rsindex 709dddb38..b3cbf82c2 100644--- a/engine/crates/xerj-storage/src/version_map.rs+++ b/engine/crates/xerj-storage/src/version_map.rs@@ -64,6 +64,20 @@ pub struct VersionEntry {pub segment_id: Arc<str>,/// `true` if the document has been deleted (tombstoned).pub deleted: bool,+ /// ES `_version`: per-document write counter, starting at 1 on first+ /// index and incremented by every subsequent index or delete of the+ /// same `doc_id` (deletes bump it too — the tombstone carries the+ /// bumped version, matching ES delete responses).+ ///+ /// Physical repoints (flush moving a doc from the memtable to its+ /// segment, merges repointing surviving docs) do NOT bump it: they+ /// re-record the same logical write.+ ///+ /// Restart caveat: the map is rebuilt from segments + WAL tail on+ /// open, so the counter restarts from the recovered history rather+ /// than the true lifetime write count (CAS is unaffected — it+ /// compares `seq_no`, which is durable).+ pub version: u64,}// ── VersionMap ───────────────────────────────────────────────────────────────@@ -145,28 +159,55 @@ impl VersionMap {seq_no: SeqNo,segment_id: impl Into<Arc<str>>,deleted: bool,- ) {+ ) -> u64 {let doc_id = doc_id.into();debug!(?doc_id, seq_no, "version_map::set");- let old = self.inner.insert(- doc_id,- VersionEntry {- seq_no,- segment_id: segment_id.into(),- deleted,- },- );+ use dashmap::mapref::entry::Entry;+ // Per-doc `_version`: a STRICTLY newer seq is a new logical write+ // (bump); an equal seq is a physical repoint of the same write+ // (keep); an older seq only occurs on replay/recovery re-sets of+ // an already-superseded copy (keep — the newer write already+ // counted itself). Computed and written under the dashmap shard+ // guard so racing writers on the same key each derive from the+ // exact entry they displace.+ let (version, was_overwrite, old_was_live) = match self.inner.entry(doc_id) {+ Entry::Occupied(mut occ) => {+ let old_seq = occ.get().seq_no;+ let old_live = !occ.get().deleted;+ let version = if seq_no > old_seq {+ occ.get().version + 1+ } else {+ occ.get().version+ };+ occ.insert(VersionEntry {+ seq_no,+ segment_id: segment_id.into(),+ deleted,+ version,+ });+ (version, old_seq != seq_no, old_live)+ }+ Entry::Vacant(vac) => {+ vac.insert(VersionEntry {+ seq_no,+ segment_id: segment_id.into(),+ deleted,+ version: 1,+ });+ (1, false, false)+ }+ };// Overwrite or delete — a superseded/tombstoned physical copy now// exists somewhere until merged away. A same-seq_no re-set is NOT// an overwrite: the flush path repoints every drained doc's entry// from `__memtable__` to its segment id with the seq_no unchanged// (live-verified: counting those flipped the gate ON for every// flushed doc and re-disabled all fast paths under pure appends).- let is_overwrite = old.as_ref().is_some_and(|e| e.seq_no != seq_no);- if is_overwrite || deleted {+ if was_overwrite || deleted {self.ghost_events.fetch_add(1, Ordering::Relaxed);}- self.live_delta(old.is_some_and(|e| !e.deleted), !deleted);+ self.live_delta(old_was_live, !deleted);+ version}/// Record `doc_id → (seq_no, segment_id)` only if `seq_no` is >= the@@ -196,10 +237,14 @@ impl VersionMap {match self.inner.entry(doc_id) {Entry::Occupied(mut occ) => {if seq_no >= occ.get().seq_no {+ // Physical repoint of an already-counted write — the+ // per-doc `_version` is carried over unchanged.+ let version = occ.get().version;occ.insert(VersionEntry {seq_no,segment_id: segment_id.into(),deleted,+ version,});}}@@ -208,6 +253,7 @@ impl VersionMap {seq_no,segment_id: segment_id.into(),deleted,+ version: 1,});}}@@ -260,10 +306,13 @@ impl VersionMap {}}let was_live = !occ.get().deleted;+ // A CAS write is a genuine new logical write.+ let version = occ.get().version + 1;occ.insert(VersionEntry {seq_no: new_seq_no,segment_id: new_segment_id,deleted,+ version,});self.ghost_events.fetch_add(1, Ordering::Relaxed);self.live_delta(was_live, !deleted);@@ -282,6 +331,7 @@ impl VersionMap {seq_no: new_seq_no,segment_id: new_segment_id,deleted,+ version: 1,});if deleted {self.ghost_events.fetch_add(1, Ordering::Relaxed);@@ -307,10 +357,15 @@ impl VersionMap {if entry.deleted {return Ok(false); // already deleted}+ // ES bumps `_version` on delete: the tombstone carries the+ // bumped value (surfaced in the delete response, and a+ // subsequent re-index continues from it).+ let version = entry.version + 1;*entry = VersionEntry {seq_no,segment_id: segment_id.into(),deleted: true,+ version,};self.ghost_events.fetch_add(1, Ordering::Relaxed);self.live_delta(true, false);@@ -320,6 +375,53 @@ impl VersionMap {}}+ /// Repoint a document's entry at a new segment WITHOUT counting a new+ /// logical write. Used by the flush path: a drained memtable doc keeps+ /// its seq_no and `_version`, only its physical home changes.+ ///+ /// - Entry present with the SAME `seq_no` → swap `segment_id`, keep+ /// `version` / `deleted` / counters untouched.+ /// - Entry present with a DIFFERENT `seq_no` → no-op. Covers both a+ /// superseded duplicate in the drained batch (an older copy of a doc+ /// that was overwritten in the same memtable generation) and a doc+ /// that was updated or deleted after the drain — clobbering either+ /// would resurrect a stale version (the old unconditional `set` here+ /// transiently did exactly that).+ /// - No entry → record it (recovery parity with the old `set` call).+ pub fn repoint(&self, doc_id: &str, seq_no: SeqNo, segment_id: impl Into<Arc<str>>) {+ use dashmap::mapref::entry::Entry;+ match self.inner.entry(doc_id.to_owned()) {+ Entry::Occupied(mut occ) => {+ if occ.get().seq_no == seq_no {+ occ.get_mut().segment_id = segment_id.into();+ }+ }+ Entry::Vacant(vac) => {+ vac.insert(VersionEntry {+ seq_no,+ segment_id: segment_id.into(),+ deleted: false,+ version: 1,+ });+ self.live_delta(false, true);+ }+ }+ }++ /// Bump the `_version` of an EXISTING tombstone and return the bumped+ /// value. ES semantics for `DELETE` of an already-deleted document:+ /// the delete reports `result: not_found` (404) but still increments+ /// the document's version, and a later re-index continues from it.+ /// Returns `None` when the doc is unknown or live.+ pub fn bump_tombstone_version(&self, doc_id: &str) -> Option<u64> {+ let mut entry = self.inner.get_mut(doc_id)?;+ if !entry.deleted {+ return None;+ }+ entry.version += 1;+ Some(entry.version)+ }+/// Remove all entries whose `segment_id` is in `stale_segment_ids`.////// Called after a merge to clean up references to segments that no longer@@ -354,9 +456,9 @@ impl VersionMap {{for (doc_id, seq_no, segment_id, deleted) in entries {// Only update if the incoming entry is newer- let should_insert = match self.inner.get(&doc_id) {- Some(existing) => seq_no > existing.seq_no,- None => true,+ let (should_insert, version) = match self.inner.get(&doc_id) {+ Some(existing) => (seq_no > existing.seq_no, existing.version + 1),+ None => (true, 1),};if should_insert {let old = self.inner.insert(@@ -365,6 +467,7 @@ impl VersionMap {seq_no,segment_id: Arc::from(segment_id),deleted,+ version,},);self.live_delta(old.is_some_and(|e| !e.deleted), !deleted);@@ -428,6 +531,70 @@ mod tests {assert_eq!(e.seq_no, 10);assert_eq!(&*e.segment_id, "seg-a");assert!(!e.deleted);+ assert_eq!(e.version, 1);+ }++ #[test]+ fn version_bumps_on_newer_seq_only() {+ let vm = VersionMap::new();+ assert_eq!(vm.set("d", 1, "__memtable__", false), 1);+ assert_eq!(vm.set("d", 5, "__memtable__", false), 2); // overwrite+ // Same-seq repoint (flush) keeps the version.+ assert_eq!(vm.set("d", 5, "seg-a", false), 2);+ // Stale replay of an older copy keeps the version.+ assert_eq!(vm.set("d", 1, "seg-a", false), 2);+ }++ #[test]+ fn delete_bumps_version_and_reindex_continues() {+ let vm = VersionMap::new();+ vm.set("d", 1, "s", false); // v1+ vm.set("d", 2, "s", false); // v2+ vm.delete("d", 3, "s").unwrap(); // tombstone carries v3+ assert_eq!(vm.get("d").unwrap().version, 3);+ assert_eq!(vm.set("d", 4, "s", false), 4); // recreate → v4+ }++ #[test]+ fn bump_tombstone_version_only_on_tombstones() {+ let vm = VersionMap::new();+ assert_eq!(vm.bump_tombstone_version("ghost"), None);+ vm.set("d", 1, "s", false);+ assert_eq!(vm.bump_tombstone_version("d"), None); // live+ vm.delete("d", 2, "s").unwrap(); // v2 tombstone+ assert_eq!(vm.bump_tombstone_version("d"), Some(3));+ assert_eq!(vm.bump_tombstone_version("d"), Some(4));+ // A recreate continues from the bumped tombstone version.+ assert_eq!(vm.set("d", 3, "s", false), 5);+ }++ #[test]+ fn repoint_swaps_segment_without_bumping() {+ let vm = VersionMap::new();+ vm.set("d", 1, "__memtable__", false);+ vm.set("d", 2, "__memtable__", false); // v2 (overwrite in memtable)+ let ghosts_before = vm.ghost_events();++ // Flush replays BOTH drained copies; only the live seq repoints.+ vm.repoint("d", 1, "seg-a"); // superseded duplicate → no-op+ let e = vm.get("d").unwrap();+ assert_eq!((&*e.segment_id, e.seq_no, e.version), ("__memtable__", 2, 2));+ vm.repoint("d", 2, "seg-a");+ let e = vm.get("d").unwrap();+ assert_eq!((&*e.segment_id, e.seq_no, e.version), ("seg-a", 2, 2));+ assert_eq!(vm.ghost_events(), ghosts_before);++ // A doc deleted after the drain is NOT resurrected by the repoint.+ vm.delete("d", 3, "__memtable__").unwrap();+ vm.repoint("d", 2, "seg-b");+ assert!(vm.get("d").unwrap().deleted);+ assert_eq!(vm.get("d").unwrap().seq_no, 3);++ // Unknown doc → recorded live (recovery parity with the old set()).+ vm.repoint("x", 7, "seg-a");+ let e = vm.get("x").unwrap();+ assert_eq!((e.seq_no, e.version, e.deleted), (7, 1, false));+ assert_eq!(vm.live_count(), 1); // "x" live, "d" tombstoned}#[test]