Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(es-compat): real per-doc _version + ES seq_no wire convention for doc CRUD (RC4-W2 A1-A3)

xerj-org8cb41ae44b87
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 7254f6185..a825ced00 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -1572,7 +1572,12 @@ pub async fn index_doc_auto(
match idx.index_document(None, doc).await {
Ok(resp) => {
state.metrics.record_doc_indexed(&index);
- let er = EsDocResponse::created(&index, &resp.id, resp.seq_no);
+ let er = EsDocResponse::created(
+ &index,
+ &resp.id,
+ resp.version,
+ resp.seq_no.saturating_sub(1),
+ );
(StatusCode::CREATED, Json(er)).into_response()
}
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
@@ -1689,11 +1694,22 @@ pub async fn index_doc(
// op_type=create means fail if document already exists.
let is_create_op = params.op_type.as_deref() == Some("create");
+ // The wire (ES) numbers seq_no from 0; the internal WAL counter from 1.
+ // `if_seq_no` arrives in ES form → +1 before the engine CAS compare
+ // (which runs on internal seqs), so a value read from GET/_search/_mget
+ // or a previous write response round-trips correctly.
+ let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));
+
if is_create_op {
match idx.create_document(id.clone(), doc).await {
Ok(resp) => {
state.metrics.record_doc_indexed(&index);
- let er = EsDocResponse::created(&index, &resp.id, resp.seq_no);
+ let er = EsDocResponse::created(
+ &index,
+ &resp.id,
+ resp.version,
+ resp.seq_no.saturating_sub(1),
+ );
(StatusCode::CREATED, Json(er)).into_response()
}
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
@@ -1708,7 +1724,7 @@ pub async fn index_doc(
idx.index_document_with_version(
Some(id.clone()),
doc,
- params.if_seq_no,
+ if_seq_no_internal,
params.if_primary_term,
)
.await
@@ -1717,7 +1733,7 @@ pub async fn index_doc(
idx.index_document_with_version(
Some(id.clone()),
doc,
- params.if_seq_no,
+ if_seq_no_internal,
params.if_primary_term,
)
.await
@@ -1726,12 +1742,17 @@ pub async fn index_doc(
Ok(resp) => {
state.metrics.record_doc_indexed(&index);
let is_update = resp.result == "updated";
- let mut er = EsDocResponse::created(&index, &resp.id, resp.seq_no);
- er.version = resp.version;
- let status = if is_update {
- StatusCode::OK
+ let wire_seq = resp.seq_no.saturating_sub(1);
+ let (er, status) = if is_update {
+ (
+ EsDocResponse::updated(&index, &resp.id, resp.version, wire_seq),
+ StatusCode::OK,
+ )
} else {
- StatusCode::CREATED
+ (
+ EsDocResponse::created(&index, &resp.id, resp.version, wire_seq),
+ StatusCode::CREATED,
+ )
};
(status, Json(er)).into_response()
}
@@ -1793,7 +1814,12 @@ pub async fn create_doc(
match idx.create_document(id.clone(), doc).await {
Ok(resp) => {
state.metrics.record_doc_indexed(&index);
- let er = EsDocResponse::created(&index, &resp.id, resp.seq_no);
+ let er = EsDocResponse::created(
+ &index,
+ &resp.id,
+ resp.version,
+ resp.seq_no.saturating_sub(1),
+ );
(StatusCode::CREATED, Json(er)).into_response()
}
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
@@ -1837,7 +1863,14 @@ pub async fn get_doc(
Ok(Some(source)) => {
// Apply _source filtering based on query params.
let filtered = apply_get_doc_source_filter(source, &params);
- let resp = EsGetResponse::found(&index, &id, 1, 1, filtered);
+ // Real per-doc metadata (was hardcoded `_version: 1` /
+ // `_seq_no: 1`, which broke read-then-CAS flows): same
+ // resolvers `_mget` and `_search` hits use. `lookup_seq_no`
+ // yields the ES wire form (0-based); `lookup_version` prefers
+ // an external version when one was supplied.
+ let version = idx.lookup_version(&id).unwrap_or(1);
+ let seq_no = idx.lookup_seq_no(&id).unwrap_or(0);
+ let resp = EsGetResponse::found(&index, &id, version, seq_no, filtered);
Json(resp).into_response()
}
Ok(None) => {
@@ -1913,24 +1946,55 @@ fn source_field_matches(field: &str, pattern: &str) -> bool {
pub struct DeleteDocParams {
/// `refresh=true|wait_for` — accepted without error; memtable is always visible.
pub refresh: Option<String>,
+ /// Optimistic concurrency: expected sequence number (ES wire form, 0-based).
+ pub if_seq_no: Option<u64>,
+ /// Optimistic concurrency: expected primary term (must accompany if_seq_no).
+ pub if_primary_term: Option<u64>,
}
pub async fn delete_doc(
State(state): State<AppState>,
Path((index, id)): Path<(String, String)>,
- Query(_params): Query<DeleteDocParams>,
+ Query(params): Query<DeleteDocParams>,
) -> impl IntoResponse {
let idx = match state.engine.get_index(&index) {
Ok(i) => i,
Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),
};
- match idx.delete_document(&id).await {
- Ok(_) => {
- let seq_no = current_timestamp_micros();
- let resp = EsDeleteDocResponse::deleted(&index, &id, seq_no);
+ // Wire → internal seq translation, same convention as the index path.
+ let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));
+
+ match idx
+ .delete_document_versioned(&id, if_seq_no_internal, params.if_primary_term)
+ .await
+ {
+ // Live doc tombstoned → 200 `deleted` with the doc's REAL bumped
+ // version and the tombstone's seq (ES wire form). Was: always
+ // 200 `deleted` `_version: 1` with a timestamp-garbage seq_no,
+ // even for missing docs and ignored CAS params.
+ Ok(outcome) if outcome.found => {
+ let resp = EsDeleteDocResponse::deleted(
+ &index,
+ &id,
+ outcome.version,
+ outcome.seq_no.saturating_sub(1),
+ );
Json(resp).into_response()
}
+ // Nothing to delete → ES answers 404 with a RESULT body (not an
+ // error body): `result: "not_found"`, version bookkeeping still
+ // advancing (live-verified against ES 8.13.4).
+ Ok(outcome) => {
+ let resp = EsDeleteDocResponse::not_found(
+ &index,
+ &id,
+ outcome.version,
+ outcome.seq_no.saturating_sub(1),
+ );
+ (StatusCode::NOT_FOUND, Json(resp)).into_response()
+ }
+ // CAS mismatch surfaces as 409 version_conflict_engine_exception.
Err(e) => ApiError::new(xerj_common::XerjError::from(e)).into_response(),
}
}
@@ -7965,17 +8029,14 @@ pub async fn search(
continue;
}
"_version" => {
- // Echo the caller-supplied external `_version`
- // (`version_type=external[_gte]`) from the same
- // map the top-level and inner_hits `_version`
- // paths use. Internal per-doc version increments
- // aren't tracked yet, so freshly-indexed docs
- // default to 1.
+ // Real per-doc `_version` via the same
+ // resolver GET/_mget use (external-version
+ // map wins, then the engine write counter).
let v = state
.engine
.get_index(&idx_name)
.ok()
- .and_then(|idx| idx.external_versions.get(&h.id).map(|v| *v))
+ .and_then(|idx| idx.lookup_version(&h.id))
.unwrap_or(1);
fmap.insert("_version".to_string(), Value::Array(vec![json!(v)]));
continue;
@@ -9045,15 +9106,14 @@ pub async fn search(
hit_obj.insert("_score".to_string(), score);
}
if emit_version_ih {
- // Look up the external version map so
- // inner_hits reflect the caller-supplied
- // `_version` just like top-level hits.
+ // Real per-doc `_version` (external map
+ // wins) so inner_hits match top-level hits.
let id_str = m.get("_id").and_then(Value::as_str).unwrap_or("");
let v = state
.engine
.get_index(&idx_name)
.ok()
- .and_then(|idx| idx.external_versions.get(id_str).map(|v| *v))
+ .and_then(|idx| idx.lookup_version(id_str))
.unwrap_or(1);
hit_obj.insert("_version".to_string(), json!(v));
}
@@ -9176,15 +9236,16 @@ pub async fn search(
None
},
version: if emit_version {
- // Prefer the external-version map (set by
- // `version_type=external[_gte]`) so reindexes with
- // explicit versions echo the caller's value.
- let ext = state
+ // Real per-doc `_version` via the shared resolver
+ // (external-version map wins so reindexes with
+ // `version_type=external[_gte]` echo the caller's
+ // value, then the engine write counter).
+ state
.engine
.get_index(&idx_name)
.ok()
- .and_then(|idx| idx.external_versions.get(&h.id).map(|v| *v));
- ext.or(Some(1))
+ .and_then(|idx| idx.lookup_version(&h.id))
+ .or(Some(1))
} else {
None
},
@@ -11524,7 +11585,7 @@ pub async fn update_doc(
&index,
&resp.id,
resp.version,
- resp.seq_no,
+ resp.seq_no.saturating_sub(1),
);
Json(er).into_response()
}
@@ -11550,7 +11611,8 @@ pub async fn update_doc(
let er = crate::responses::EsDocResponse::created(
&index,
&resp.id,
- resp.seq_no,
+ resp.version,
+ resp.seq_no.saturating_sub(1),
);
Json(er).into_response()
}
@@ -11574,7 +11636,7 @@ pub async fn update_doc(
&index,
&resp.id,
resp.version,
- resp.seq_no,
+ resp.seq_no.saturating_sub(1),
);
Json(er).into_response()
}
@@ -12104,13 +12166,15 @@ pub async fn mget(
match idx.get_document(&req_doc.id).await {
Ok(Some(source)) => {
- // Real per-doc `_seq_no` — same source the `_search` hit
- // metadata uses (`lookup_seq_no`); was hardcoded 0.
+ // Real per-doc `_seq_no` / `_version` — same resolvers the
+ // `_search` hit metadata and GET `_doc` use; were hardcoded
+ // 0 / 1.
let seq_no = idx.lookup_seq_no(&req_doc.id).unwrap_or(0);
+ let version = idx.lookup_version(&req_doc.id).unwrap_or(1);
docs.push(json!({
"_index": req_doc.index,
"_id": req_doc.id,
- "_version": 1,
+ "_version": version,
"_seq_no": seq_no,
"_primary_term": 1,
"found": true,
@@ -12189,13 +12253,14 @@ pub async fn mget_index(
match state.engine.get_index(ix) {
Ok(idx) => match idx.get_document(id).await {
Ok(Some(source)) => {
- // Real per-doc `_seq_no` (was hardcoded 0) — matches the
- // `_search` hit metadata path.
+ // Real per-doc `_seq_no` / `_version` (were hardcoded
+ // 0 / 1) — matches the `_search` hit metadata path.
let seq_no = idx.lookup_seq_no(id).unwrap_or(0);
+ let version = idx.lookup_version(id).unwrap_or(1);
docs.push(json!({
"_index": ix,
"_id": id,
- "_version": 1,
+ "_version": version,
"_seq_no": seq_no,
"_primary_term": 1,
"found": true,
diff --git a/engine/crates/xerj-api/src/responses.rs b/engine/crates/xerj-api/src/responses.rs
index 0eef97d01..3130c3e75 100644
--- a/engine/crates/xerj-api/src/responses.rs
+++ b/engine/crates/xerj-api/src/responses.rs
@@ -129,11 +129,19 @@ pub struct EsDocResponse {
}
impl EsDocResponse {
- pub fn created(index: impl Into<String>, id: impl Into<String>, seq_no: u64) -> Self {
+ /// `result: "created"` — `version` is NOT always 1: a re-index after a
+ /// delete reports `created` while `_version` continues from the
+ /// tombstone (ES 8.13.4 live-verified).
+ pub fn created(
+ index: impl Into<String>,
+ id: impl Into<String>,
+ version: u64,
+ seq_no: u64,
+ ) -> Self {
Self {
index: index.into(),
id: id.into(),
- version: 1,
+ version,
result: "created".to_string(),
shards: EsShards::single_success(),
seq_no,
@@ -160,18 +168,23 @@ impl EsDocResponse {
}
/// Response to `GET /{index}/_doc/{id}`.
+///
+/// The metadata fields are `Option` because ES omits `_version` /
+/// `_seq_no` / `_primary_term` entirely on a not-found GET — the 404 body
+/// is exactly `{"_index": ..., "_id": ..., "found": false}` (live-verified
+/// against ES 8.13.4).
#[derive(Debug, Serialize, Deserialize)]
pub struct EsGetResponse {
#[serde(rename = "_index")]
pub index: String,
#[serde(rename = "_id")]
pub id: String,
- #[serde(rename = "_version")]
- pub version: u64,
- #[serde(rename = "_seq_no")]
- pub seq_no: u64,
- #[serde(rename = "_primary_term")]
- pub primary_term: u64,
+ #[serde(rename = "_version", skip_serializing_if = "Option::is_none")]
+ pub version: Option<u64>,
+ #[serde(rename = "_seq_no", skip_serializing_if = "Option::is_none")]
+ pub seq_no: Option<u64>,
+ #[serde(rename = "_primary_term", skip_serializing_if = "Option::is_none")]
+ pub primary_term: Option<u64>,
pub found: bool,
#[serde(rename = "_source", skip_serializing_if = "Option::is_none")]
pub source: Option<Value>,
@@ -188,9 +201,9 @@ impl EsGetResponse {
Self {
index: index.into(),
id: id.into(),
- version,
- seq_no,
- primary_term: 1,
+ version: Some(version),
+ seq_no: Some(seq_no),
+ primary_term: Some(1),
found: true,
source: Some(source),
}
@@ -200,9 +213,9 @@ impl EsGetResponse {
Self {
index: index.into(),
id: id.into(),
- version: 0,
- seq_no: 0,
- primary_term: 0,
+ version: None,
+ seq_no: None,
+ primary_term: None,
found: false,
source: None,
}
@@ -228,17 +241,42 @@ pub struct EsDeleteDocResponse {
}
impl EsDeleteDocResponse {
- pub fn deleted(index: impl Into<String>, id: impl Into<String>, seq_no: u64) -> Self {
+ pub fn deleted(
+ index: impl Into<String>,
+ id: impl Into<String>,
+ version: u64,
+ seq_no: u64,
+ ) -> Self {
Self {
index: index.into(),
id: id.into(),
- version: 1,
+ version,
result: "deleted".to_string(),
shards: EsShards::single_success(),
seq_no,
primary_term: 1,
}
}
+
+ /// ES 404 body for a delete that found nothing: same shape as a
+ /// successful delete but `result: "not_found"` (live-verified against
+ /// ES 8.13.4 — the version bookkeeping still advances).
+ pub fn not_found(
+ index: impl Into<String>,
+ id: impl Into<String>,
+ version: u64,
+ seq_no: u64,
+ ) -> Self {
+ Self {
+ index: index.into(),
+ id: id.into(),
+ version,
+ result: "not_found".to_string(),
+ shards: EsShards::single_success(),
+ seq_no,
+ primary_term: 1,
+ }
+ }
}
// ═════════════════════════════════════════════════════════════════════════════
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index 080a76aaf..76623052d 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -454,7 +454,15 @@ pub async fn process_bulk_with_opts(
.get("pipeline")
.and_then(Value::as_str)
.map(str::to_owned);
- if_seq_no_per_item = meta.get("if_seq_no").and_then(Value::as_u64);
+ // Wire → internal seq translation: ES numbers seq_no
+ // from 0 on the wire, the internal WAL counter from 1.
+ // The CAS compare (index_document_with_version) runs on
+ // internal seqs, so a `_seq_no` captured from any read
+ // or write response round-trips through bulk CAS.
+ if_seq_no_per_item = meta
+ .get("if_seq_no")
+ .and_then(Value::as_u64)
+ .map(|s| s.saturating_add(1));
if_primary_term_per_item = meta.get("if_primary_term").and_then(Value::as_u64);
routing_per_item = meta
.get("routing")
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 160363ed3..1830fb468 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -149,6 +149,26 @@ pub struct IndexResponse {
pub result: String,
}
+/// Outcome of a single-document delete (ES `DELETE /{index}/_doc/{id}`).
+///
+/// `found == true` → the doc was live and is now tombstoned; `seq_no` is
+/// the tombstone's internal (1-based) WAL seq and `version` the bumped
+/// per-doc version (ES semantics: deletes increment `_version`).
+///
+/// `found == false` → nothing was deleted (`result: not_found`). `version`
+/// still reports the ES-visible version: an existing tombstone is bumped
+/// and persisted (matching ES, where a not-found delete still versions the
+/// tombstone), an unknown id reports 1. `seq_no` carries the tombstone's
+/// seq when one exists, else the index's latest assigned seq (xerj does
+/// not consume a new seq_no for a not-found delete — divergence from ES
+/// in the reported VALUE only).
+#[derive(Debug, Clone)]
+pub struct DeleteDocOutcome {
+ pub found: bool,
+ pub seq_no: u64,
+ pub version: u64,
+}
+
/// Per-field encoding statistics, derived from `FieldAnalyzer`.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FieldEncodingInfo {
@@ -1237,8 +1257,16 @@ impl Index {
// Detect whether this is an overwrite (existing doc) vs a new
// insert BEFORE writing to storage. Used to set the response
// `result` field to "updated" / "created" per ES semantics so
- // HTTP-layer 200/201 bulk status mapping works.
- let existed_before = self.store.version_map.get(&doc_id).is_some();
+ // HTTP-layer 200/201 bulk status mapping works. A tombstoned doc
+ // counts as NOT existing: ES reports re-index-after-delete as
+ // `created`/201 (while its `_version` still continues from the
+ // tombstone).
+ let existed_before = self
+ .store
+ .version_map
+ .get(&doc_id)
+ .map(|e| !e.deleted)
+ .unwrap_or(false);
// Write to storage WAL.
let seq_no = self.store.index(&doc_id, source.clone())?;
@@ -1251,8 +1279,22 @@ impl Index {
mem.insert(doc_id.clone(), &source, &schema_guard.schema, seq_no);
}
- // Bump counter.
- let version = self.doc_count.fetch_add(1, Ordering::Relaxed) + 1;
+ // Bump counter (flush thresholds / stats bookkeeping only — NOT the
+ // response `_version`, which is per-doc).
+ self.doc_count.fetch_add(1, Ordering::Relaxed);
+
+ // Real per-doc `_version`: `store.index` just recorded this write
+ // in the version map (1 on first index, +1 per overwrite, deletes
+ // included). The previous code reported the index-global doc_count
+ // here, so any write to doc B inflated the next reported version
+ // of doc A (live-verified: three PUTs to the same id after other
+ // writes reported 4, 5, 6).
+ let version = self
+ .store
+ .version_map
+ .get(&doc_id)
+ .map(|e| e.version)
+ .unwrap_or(1);
// Auto-evolve schema for new fields.
// Fast path: skip the schema read lock every doc when schema is stable.
@@ -4935,6 +4977,28 @@ impl Index {
})
}
+ /// Look up the ES `_version` for a live document by id.
+ ///
+ /// The external-version map wins (docs written with
+ /// `version_type=external[_gte]` must echo the caller's version, per
+ /// ES), falling back to the engine-tracked per-doc write counter.
+ /// Returns `None` when the doc is unknown or tombstoned.
+ pub fn lookup_version(&self, id: &str) -> Option<u64> {
+ let tracked = self.store.version_map.get(id).and_then(|entry| {
+ if entry.deleted {
+ None
+ } else {
+ Some(entry.version)
+ }
+ })?;
+ Some(
+ self.external_versions
+ .get(id)
+ .map(|v| *v)
+ .unwrap_or(tracked),
+ )
+ }
+
/// Retrieve a document by its string ID.
///
/// Checks the memtable first, then searches on-disk segments.
@@ -5059,7 +5123,34 @@ impl Index {
}
/// Delete a document by ID.
+ ///
+ /// Thin compatibility wrapper over [`Self::delete_document_versioned`]
+ /// (no optimistic-concurrency check). Returns whether a live document
+ /// was found and tombstoned.
pub async fn delete_document(&self, id: &str) -> Result<bool> {
+ self.delete_document_versioned(id, None, None)
+ .await
+ .map(|outcome| outcome.found)
+ }
+
+ /// Delete a document by ID with ES delete semantics.
+ ///
+ /// When `if_seq_no` is supplied (internal 1-based form), the delete is
+ /// conditional: the doc's current seq_no must match or a
+ /// `VersionConflict` is returned — including when the doc does not
+ /// exist (ES returns 409 for a conditional delete of a missing doc).
+ /// xerj is single-shard with `primary_term` fixed at 1, so a supplied
+ /// `if_primary_term` is accepted but only `if_seq_no` is compared
+ /// (same treatment as `index_document_with_version`).
+ ///
+ /// Unconditional deletes of a missing doc return `found: false` with
+ /// the ES-visible version bookkeeping described on [`DeleteDocOutcome`].
+ pub async fn delete_document_versioned(
+ &self,
+ id: &str,
+ if_seq_no: Option<u64>,
+ if_primary_term: Option<u64>,
+ ) -> Result<DeleteDocOutcome> {
// Check write block.
if self.is_write_blocked().await {
return Err(EngineError::Common(xerj_common::XerjError::index_blocked(
@@ -5069,6 +5160,32 @@ impl Index {
}
self.dataset_version.fetch_add(1, Ordering::Release);
+ // Optimistic concurrency check (see index_document_with_version for
+ // the if_primary_term rationale).
+ let _ = if_primary_term;
+ if let Some(expected_seq) = if_seq_no {
+ match self.store.version_map.get(id) {
+ Some(entry) if !entry.deleted => {
+ if entry.seq_no != expected_seq {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::version_conflict(
+ id,
+ expected_seq,
+ entry.seq_no,
+ ),
+ ));
+ }
+ }
+ // Missing or tombstoned doc + conditional delete → conflict
+ // (ES: "required seqNo [N] ... but no document was found").
+ _ => {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::version_conflict(id, expected_seq, 0),
+ ));
+ }
+ }
+ }
+
// Check whether the document actually exists and is not already deleted.
let is_live = self
.store
@@ -5087,10 +5204,28 @@ impl Index {
let should_delete = is_live || in_memtable;
if !should_delete {
- return Ok(false);
+ // ES `result: not_found` bookkeeping: a repeat delete of a
+ // tombstoned doc still bumps (and persists) the tombstone's
+ // version; a never-seen id reports version 1. No WAL entry or
+ // seq_no is consumed for a not-found delete.
+ let existing = self.store.version_map.get(id);
+ let version = self
+ .store
+ .version_map
+ .bump_tombstone_version(id)
+ .unwrap_or(1);
+ let seq_no = existing
+ .map(|e| e.seq_no)
+ .unwrap_or_else(|| self.store.current_seq_no().saturating_sub(1));
+ return Ok(DeleteDocOutcome {
+ found: false,
+ seq_no,
+ version,
+ });
}
- let existed = self.store.delete(id)?.is_some();
+ let deleted_seq = self.store.delete(id)?;
+ let existed = deleted_seq.is_some();
// Remove from memtable.
{
@@ -5123,7 +5258,36 @@ impl Index {
if existed {
self.doc_count.fetch_sub(1, Ordering::Relaxed);
}
- Ok(existed || in_memtable)
+
+ let found = existed || in_memtable;
+ if !found {
+ // Raced away between the liveness check and store.delete —
+ // report as not_found (same bookkeeping as the early return).
+ let version = self
+ .store
+ .version_map
+ .bump_tombstone_version(id)
+ .unwrap_or(1);
+ return Ok(DeleteDocOutcome {
+ found: false,
+ seq_no: self.store.current_seq_no().saturating_sub(1),
+ version,
+ });
+ }
+
+ // The tombstone entry now carries the bumped per-doc version.
+ let version = self
+ .store
+ .version_map
+ .get(id)
+ .map(|e| e.version)
+ .unwrap_or(1);
+ let seq_no = deleted_seq.unwrap_or_else(|| self.store.current_seq_no().saturating_sub(1));
+ Ok(DeleteDocOutcome {
+ found: true,
+ seq_no,
+ version,
+ })
}
/// Update a document by merging a partial document into the existing source.
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 20f7e6a56..de5685775 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -1502,16 +1502,26 @@ impl IndexStore {
post_finish(&meta)?;
let prof_pf_us = t_pf.elapsed().as_micros();
- // Update version map: point live docs at the new segment
+ // Update version map: point live docs at the new segment.
+ //
+ // `repoint` (not `set`): the drained entries include superseded
+ // duplicates (a doc overwritten within the same memtable
+ // generation drains once per write), and a doc can be updated or
+ // deleted concurrently after the drain. The old unconditional
+ // `set` transiently clobbered the newer entry with the stale
+ // copy's seq_no (and, with per-doc `_version` tracking, would
+ // spuriously bump the version once per duplicate). `repoint`
+ // only swaps the segment id when the entry still carries exactly
+ // this seq_no — same-generation duplicates and post-drain
+ // updates/deletes are left untouched.
let t_vm = std::time::Instant::now();
let segment_id_arc: std::sync::Arc<str> = std::sync::Arc::from(segment_id.as_str());
for entry in &entries {
if entry.source.is_some() {
- self.version_map.set(
+ self.version_map.repoint(
&entry.doc_id,
entry.seq_no,
std::sync::Arc::clone(&segment_id_arc),
- false,
);
}
}
diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rs
index 709dddb38..b3cbf82c2 100644
--- a/engine/crates/xerj-storage/src/version_map.rs
+++ b/engine/crates/xerj-storage/src/version_map.rs
@@ -64,6 +64,20 @@ pub struct VersionEntry {
pub segment_id: Arc<str>,
/// `true` if the document has been deleted (tombstoned).
pub deleted: bool,
+ /// ES `_version`: per-document write counter, starting at 1 on first
+ /// index and incremented by every subsequent index or delete of the
+ /// same `doc_id` (deletes bump it too — the tombstone carries the
+ /// bumped version, matching ES delete responses).
+ ///
+ /// Physical repoints (flush moving a doc from the memtable to its
+ /// segment, merges repointing surviving docs) do NOT bump it: they
+ /// re-record the same logical write.
+ ///
+ /// Restart caveat: the map is rebuilt from segments + WAL tail on
+ /// open, so the counter restarts from the recovered history rather
+ /// than the true lifetime write count (CAS is unaffected — it
+ /// compares `seq_no`, which is durable).
+ pub version: u64,
}
// ── VersionMap ───────────────────────────────────────────────────────────────
@@ -145,28 +159,55 @@ impl VersionMap {
seq_no: SeqNo,
segment_id: impl Into<Arc<str>>,
deleted: bool,
- ) {
+ ) -> u64 {
let doc_id = doc_id.into();
debug!(?doc_id, seq_no, "version_map::set");
- let old = self.inner.insert(
- doc_id,
- VersionEntry {
- seq_no,
- segment_id: segment_id.into(),
- deleted,
- },
- );
+ use dashmap::mapref::entry::Entry;
+ // Per-doc `_version`: a STRICTLY newer seq is a new logical write
+ // (bump); an equal seq is a physical repoint of the same write
+ // (keep); an older seq only occurs on replay/recovery re-sets of
+ // an already-superseded copy (keep — the newer write already
+ // counted itself). Computed and written under the dashmap shard
+ // guard so racing writers on the same key each derive from the
+ // exact entry they displace.
+ let (version, was_overwrite, old_was_live) = match self.inner.entry(doc_id) {
+ Entry::Occupied(mut occ) => {
+ let old_seq = occ.get().seq_no;
+ let old_live = !occ.get().deleted;
+ let version = if seq_no > old_seq {
+ occ.get().version + 1
+ } else {
+ occ.get().version
+ };
+ occ.insert(VersionEntry {
+ seq_no,
+ segment_id: segment_id.into(),
+ deleted,
+ version,
+ });
+ (version, old_seq != seq_no, old_live)
+ }
+ Entry::Vacant(vac) => {
+ vac.insert(VersionEntry {
+ seq_no,
+ segment_id: segment_id.into(),
+ deleted,
+ version: 1,
+ });
+ (1, false, false)
+ }
+ };
// Overwrite or delete — a superseded/tombstoned physical copy now
// exists somewhere until merged away. A same-seq_no re-set is NOT
// an overwrite: the flush path repoints every drained doc's entry
// from `__memtable__` to its segment id with the seq_no unchanged
// (live-verified: counting those flipped the gate ON for every
// flushed doc and re-disabled all fast paths under pure appends).
- let is_overwrite = old.as_ref().is_some_and(|e| e.seq_no != seq_no);
- if is_overwrite || deleted {
+ if was_overwrite || deleted {
self.ghost_events.fetch_add(1, Ordering::Relaxed);
}
- self.live_delta(old.is_some_and(|e| !e.deleted), !deleted);
+ self.live_delta(old_was_live, !deleted);
+ version
}
/// Record `doc_id → (seq_no, segment_id)` only if `seq_no` is >= the
@@ -196,10 +237,14 @@ impl VersionMap {
match self.inner.entry(doc_id) {
Entry::Occupied(mut occ) => {
if seq_no >= occ.get().seq_no {
+ // Physical repoint of an already-counted write — the
+ // per-doc `_version` is carried over unchanged.
+ let version = occ.get().version;
occ.insert(VersionEntry {
seq_no,
segment_id: segment_id.into(),
deleted,
+ version,
});
}
}
@@ -208,6 +253,7 @@ impl VersionMap {
seq_no,
segment_id: segment_id.into(),
deleted,
+ version: 1,
});
}
}
@@ -260,10 +306,13 @@ impl VersionMap {
}
}
let was_live = !occ.get().deleted;
+ // A CAS write is a genuine new logical write.
+ let version = occ.get().version + 1;
occ.insert(VersionEntry {
seq_no: new_seq_no,
segment_id: new_segment_id,
deleted,
+ version,
});
self.ghost_events.fetch_add(1, Ordering::Relaxed);
self.live_delta(was_live, !deleted);
@@ -282,6 +331,7 @@ impl VersionMap {
seq_no: new_seq_no,
segment_id: new_segment_id,
deleted,
+ version: 1,
});
if deleted {
self.ghost_events.fetch_add(1, Ordering::Relaxed);
@@ -307,10 +357,15 @@ impl VersionMap {
if entry.deleted {
return Ok(false); // already deleted
}
+ // ES bumps `_version` on delete: the tombstone carries the
+ // bumped value (surfaced in the delete response, and a
+ // subsequent re-index continues from it).
+ let version = entry.version + 1;
*entry = VersionEntry {
seq_no,
segment_id: segment_id.into(),
deleted: true,
+ version,
};
self.ghost_events.fetch_add(1, Ordering::Relaxed);
self.live_delta(true, false);
@@ -320,6 +375,53 @@ impl VersionMap {
}
}
+ /// Repoint a document's entry at a new segment WITHOUT counting a new
+ /// logical write. Used by the flush path: a drained memtable doc keeps
+ /// its seq_no and `_version`, only its physical home changes.
+ ///
+ /// - Entry present with the SAME `seq_no` → swap `segment_id`, keep
+ /// `version` / `deleted` / counters untouched.
+ /// - Entry present with a DIFFERENT `seq_no` → no-op. Covers both a
+ /// superseded duplicate in the drained batch (an older copy of a doc
+ /// that was overwritten in the same memtable generation) and a doc
+ /// that was updated or deleted after the drain — clobbering either
+ /// would resurrect a stale version (the old unconditional `set` here
+ /// transiently did exactly that).
+ /// - No entry → record it (recovery parity with the old `set` call).
+ pub fn repoint(&self, doc_id: &str, seq_no: SeqNo, segment_id: impl Into<Arc<str>>) {
+ use dashmap::mapref::entry::Entry;
+ match self.inner.entry(doc_id.to_owned()) {
+ Entry::Occupied(mut occ) => {
+ if occ.get().seq_no == seq_no {
+ occ.get_mut().segment_id = segment_id.into();
+ }
+ }
+ Entry::Vacant(vac) => {
+ vac.insert(VersionEntry {
+ seq_no,
+ segment_id: segment_id.into(),
+ deleted: false,
+ version: 1,
+ });
+ self.live_delta(false, true);
+ }
+ }
+ }
+
+ /// Bump the `_version` of an EXISTING tombstone and return the bumped
+ /// value. ES semantics for `DELETE` of an already-deleted document:
+ /// the delete reports `result: not_found` (404) but still increments
+ /// the document's version, and a later re-index continues from it.
+ /// Returns `None` when the doc is unknown or live.
+ pub fn bump_tombstone_version(&self, doc_id: &str) -> Option<u64> {
+ let mut entry = self.inner.get_mut(doc_id)?;
+ if !entry.deleted {
+ return None;
+ }
+ entry.version += 1;
+ Some(entry.version)
+ }
+
/// Remove all entries whose `segment_id` is in `stale_segment_ids`.
///
/// Called after a merge to clean up references to segments that no longer
@@ -354,9 +456,9 @@ impl VersionMap {
{
for (doc_id, seq_no, segment_id, deleted) in entries {
// Only update if the incoming entry is newer
- let should_insert = match self.inner.get(&doc_id) {
- Some(existing) => seq_no > existing.seq_no,
- None => true,
+ let (should_insert, version) = match self.inner.get(&doc_id) {
+ Some(existing) => (seq_no > existing.seq_no, existing.version + 1),
+ None => (true, 1),
};
if should_insert {
let old = self.inner.insert(
@@ -365,6 +467,7 @@ impl VersionMap {
seq_no,
segment_id: Arc::from(segment_id),
deleted,
+ version,
},
);
self.live_delta(old.is_some_and(|e| !e.deleted), !deleted);
@@ -428,6 +531,70 @@ mod tests {
assert_eq!(e.seq_no, 10);
assert_eq!(&*e.segment_id, "seg-a");
assert!(!e.deleted);
+ assert_eq!(e.version, 1);
+ }
+
+ #[test]
+ fn version_bumps_on_newer_seq_only() {
+ let vm = VersionMap::new();
+ assert_eq!(vm.set("d", 1, "__memtable__", false), 1);
+ assert_eq!(vm.set("d", 5, "__memtable__", false), 2); // overwrite
+ // Same-seq repoint (flush) keeps the version.
+ assert_eq!(vm.set("d", 5, "seg-a", false), 2);
+ // Stale replay of an older copy keeps the version.
+ assert_eq!(vm.set("d", 1, "seg-a", false), 2);
+ }
+
+ #[test]
+ fn delete_bumps_version_and_reindex_continues() {
+ let vm = VersionMap::new();
+ vm.set("d", 1, "s", false); // v1
+ vm.set("d", 2, "s", false); // v2
+ vm.delete("d", 3, "s").unwrap(); // tombstone carries v3
+ assert_eq!(vm.get("d").unwrap().version, 3);
+ assert_eq!(vm.set("d", 4, "s", false), 4); // recreate → v4
+ }
+
+ #[test]
+ fn bump_tombstone_version_only_on_tombstones() {
+ let vm = VersionMap::new();
+ assert_eq!(vm.bump_tombstone_version("ghost"), None);
+ vm.set("d", 1, "s", false);
+ assert_eq!(vm.bump_tombstone_version("d"), None); // live
+ vm.delete("d", 2, "s").unwrap(); // v2 tombstone
+ assert_eq!(vm.bump_tombstone_version("d"), Some(3));
+ assert_eq!(vm.bump_tombstone_version("d"), Some(4));
+ // A recreate continues from the bumped tombstone version.
+ assert_eq!(vm.set("d", 3, "s", false), 5);
+ }
+
+ #[test]
+ fn repoint_swaps_segment_without_bumping() {
+ let vm = VersionMap::new();
+ vm.set("d", 1, "__memtable__", false);
+ vm.set("d", 2, "__memtable__", false); // v2 (overwrite in memtable)
+ let ghosts_before = vm.ghost_events();
+
+ // Flush replays BOTH drained copies; only the live seq repoints.
+ vm.repoint("d", 1, "seg-a"); // superseded duplicate → no-op
+ let e = vm.get("d").unwrap();
+ assert_eq!((&*e.segment_id, e.seq_no, e.version), ("__memtable__", 2, 2));
+ vm.repoint("d", 2, "seg-a");
+ let e = vm.get("d").unwrap();
+ assert_eq!((&*e.segment_id, e.seq_no, e.version), ("seg-a", 2, 2));
+ assert_eq!(vm.ghost_events(), ghosts_before);
+
+ // A doc deleted after the drain is NOT resurrected by the repoint.
+ vm.delete("d", 3, "__memtable__").unwrap();
+ vm.repoint("d", 2, "seg-b");
+ assert!(vm.get("d").unwrap().deleted);
+ assert_eq!(vm.get("d").unwrap().seq_no, 3);
+
+ // Unknown doc → recorded live (recovery parity with the old set()).
+ vm.repoint("x", 7, "seg-a");
+ let e = vm.get("x").unwrap();
+ assert_eq!((e.seq_no, e.version, e.deleted), (7, 1, false));
+ assert_eq!(vm.live_count(), 1); // "x" live, "d" tombstoned
}
#[test]