AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): transactional merge publication and rollback (#140)
xerj-orgba7bd23975e6
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 7b158b543..efbf19b08 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -996,6 +996,324 @@ impl<'a> Drop for MergeFlagClear<'a> {}}+#[cfg(test)]+mod merge_publication_transaction_tests {+ use super::*;+ use crate::Engine;+ use tempfile::TempDir;++ fn config(dir: &TempDir) -> xerj_common::config::Config {+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.merge.min_segments = 2;+ config.merge.min_merge_count = 2;+ config.merge.max_merge_count = 2;+ config+ }++ async fn fixture(dir: &TempDir) -> (Engine, Arc<Index>) {+ let engine = Engine::new(config(dir)).unwrap();+ engine+ .create_index("post-save-panic", Schema::empty())+ .unwrap();+ let index = engine.get_index("post-save-panic").unwrap();+ index+ .index_document(Some("a".into()), serde_json::json!({"value": "old-a"}))+ .await+ .unwrap();+ index.flush().await.unwrap();+ index+ .index_document(Some("b".into()), serde_json::json!({"value": "old-b"}))+ .await+ .unwrap();+ index.flush().await.unwrap();+ assert_eq!(index.store.snapshot().segments.len(), 2);+ (engine, index)+ }++ async fn wait_for_repoint(index: &Index) {+ tokio::time::timeout(std::time::Duration::from_secs(10), async {+ while !index.test_merge_repoint_ready.load(Ordering::Acquire) {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("merge did not reach the pre-apply gate");+ }++ #[tokio::test]+ async fn apply_failure_restores_sources_survives_restart_and_retry() {+ let dir = TempDir::new().unwrap();+ let (engine, index) = fixture(&dir).await;+ let before_ids: std::collections::HashSet<_> = index+ .store+ .snapshot()+ .segments+ .iter()+ .map(|meta| meta.id.clone())+ .collect();+ index+ .test_fail_merge_before_apply+ .store(true, Ordering::Release);+ assert!(index.run_merge_once().await.is_err());+ let after_ids: std::collections::HashSet<_> = index+ .store+ .snapshot()+ .segments+ .iter()+ .map(|meta| meta.id.clone())+ .collect();+ assert_eq!(after_ids, before_ids);+ for (id, expected) in [("a", "old-a"), ("b", "old-b")] {+ assert_eq!(+ index.get_document(id).await.unwrap().unwrap()["value"],+ expected+ );+ }+ assert_eq!(+ index+ .search(&SearchRequest::default())+ .await+ .unwrap()+ .total+ .value,+ 2+ );+ drop(index);+ drop(engine);++ let reopened = Engine::new(config(&dir)).unwrap();+ let reopened_index = reopened.get_index("post-save-panic").unwrap();+ for (id, expected) in [("a", "old-a"), ("b", "old-b")] {+ assert_eq!(+ reopened_index.get_document(id).await.unwrap().unwrap()["value"],+ expected+ );+ }+ assert_eq!(reopened_index.run_merge_once().await.unwrap(), 1);+ assert_eq!(reopened_index.store.snapshot().segments.len(), 1);+ assert_eq!(+ reopened_index+ .search(&SearchRequest::default())+ .await+ .unwrap()+ .total+ .value,+ 2+ );+ }++ #[tokio::test]+ async fn rollback_preserves_concurrent_put_and_delete() {+ let dir = TempDir::new().unwrap();+ let (engine, index) = fixture(&dir).await;+ index+ .test_pause_merge_before_apply+ .store(true, Ordering::Release);+ index+ .test_fail_merge_before_apply+ .store(true, Ordering::Release);+ let merge_index = Arc::clone(&index);+ let merge = tokio::spawn(async move { merge_index.run_merge_once().await });+ wait_for_repoint(&index).await;+ index+ .index_document(Some("a".into()), serde_json::json!({"value": "new-a"}))+ .await+ .unwrap();+ assert!(index.delete_document("b").await.unwrap());+ index+ .test_pause_merge_before_apply+ .store(false, Ordering::Release);+ assert!(merge.await.unwrap().is_err());+ assert_eq!(+ index.get_document("a").await.unwrap().unwrap()["value"],+ "new-a"+ );+ assert!(index.get_document("b").await.unwrap().is_none());+ assert!(index.store.version_map.get("b").unwrap().deleted);+ assert_eq!(+ index+ .search(&SearchRequest::default())+ .await+ .unwrap()+ .total+ .value,+ 1+ );+ drop(index);+ drop(engine);+ let reopened = Engine::new(config(&dir)).unwrap();+ let reopened_index = reopened.get_index("post-save-panic").unwrap();+ assert_eq!(+ reopened_index.get_document("a").await.unwrap().unwrap()["value"],+ "new-a"+ );+ assert!(reopened_index.get_document("b").await.unwrap().is_none());+ assert!(reopened_index.store.version_map.get("b").unwrap().deleted);+ assert_eq!(+ reopened_index+ .search(&SearchRequest::default())+ .await+ .unwrap()+ .total+ .value,+ 1+ );+ }++ #[tokio::test]+ async fn cancellation_after_repoint_restores_inputs() {+ let dir = TempDir::new().unwrap();+ let (_engine, index) = fixture(&dir).await;+ let before_ids: std::collections::HashSet<_> = index+ .store+ .snapshot()+ .segments+ .iter()+ .map(|meta| meta.id.clone())+ .collect();+ index+ .test_pause_merge_before_apply+ .store(true, Ordering::Release);+ let merge_index = Arc::clone(&index);+ let merge = tokio::spawn(async move { merge_index.run_merge_once().await });+ wait_for_repoint(&index).await;+ merge.abort();+ assert!(merge.await.unwrap_err().is_cancelled());+ index+ .test_pause_merge_before_apply+ .store(false, Ordering::Release);+ let after_ids: std::collections::HashSet<_> = index+ .store+ .snapshot()+ .segments+ .iter()+ .map(|meta| meta.id.clone())+ .collect();+ assert_eq!(after_ids, before_ids);+ for id in ["a", "b"] {+ assert!(index.get_document(id).await.unwrap().is_some());+ assert!(+ before_ids.contains(index.store.version_map.get(id).unwrap().segment_id.as_ref())+ );+ }+ }++ #[tokio::test]+ async fn partial_multi_batch_failure_is_reported_without_losing_either_batch() {+ let dir = TempDir::new().unwrap();+ let engine = Engine::new(config(&dir)).unwrap();+ engine.create_index("multi-batch", Schema::empty()).unwrap();+ let index = engine.get_index("multi-batch").unwrap();+ for id in ["a", "b", "c", "d"] {+ index+ .index_document(+ Some(id.into()),+ serde_json::json!({"value": format!("source-{id}")}),+ )+ .await+ .unwrap();+ index.flush().await.unwrap();+ }+ assert_eq!(index.store.snapshot().segments.len(), 4);+ index+ .test_fail_merge_before_apply+ .store(true, Ordering::Release);+ assert!(index.run_merge_once().await.is_err());+ assert_eq!(index.store.snapshot().segments.len(), 3);+ for id in ["a", "b", "c", "d"] {+ assert_eq!(+ index.get_document(id).await.unwrap().unwrap()["value"],+ format!("source-{id}")+ );+ }+ assert_eq!(+ index+ .search(&SearchRequest::default())+ .await+ .unwrap()+ .total+ .value,+ 4+ );+ drop(index);+ drop(engine);++ let reopened = Engine::new(config(&dir)).unwrap();+ let reopened_index = reopened.get_index("multi-batch").unwrap();+ for id in ["a", "b", "c", "d"] {+ assert_eq!(+ reopened_index.get_document(id).await.unwrap().unwrap()["value"],+ format!("source-{id}")+ );+ }+ assert_eq!(+ reopened_index+ .search(&SearchRequest::default())+ .await+ .unwrap()+ .total+ .value,+ 4+ );+ }++ #[tokio::test]+ async fn panic_after_durable_snapshot_keeps_output_repoints_and_survives_restart() {+ let dir = TempDir::new().unwrap();+ let (engine, index) = fixture(&dir).await;+ index+ .test_panic_after_merge_publish+ .store(true, Ordering::Release);+ let merge_index = Arc::clone(&index);+ let join = tokio::spawn(async move { merge_index.run_merge_once().await }).await;+ assert!(join.unwrap_err().is_panic());++ let snapshot = index.store.snapshot();+ assert_eq!(snapshot.segments.len(), 1);+ let output_id = snapshot.segments[0].id.clone();+ drop(snapshot);+ for (id, expected) in [("a", "old-a"), ("b", "old-b")] {+ let entry = index.store.version_map.get(id).unwrap();+ assert_eq!(entry.segment_id.as_ref(), output_id);+ assert_eq!(+ index.get_document(id).await.unwrap().unwrap()["value"],+ expected+ );+ }+ assert_eq!(+ index+ .search(&SearchRequest::default())+ .await+ .unwrap()+ .total+ .value,+ 2+ );+ drop(index);+ drop(engine);++ let reopened = Engine::new(config(&dir)).unwrap();+ let reopened_index = reopened.get_index("post-save-panic").unwrap();+ assert_eq!(reopened_index.store.snapshot().segments.len(), 1);+ for (id, expected) in [("a", "old-a"), ("b", "old-b")] {+ assert_eq!(+ reopened_index.get_document(id).await.unwrap().unwrap()["value"],+ expected+ );+ }+ assert_eq!(+ reopened_index+ .search(&SearchRequest::default())+ .await+ .unwrap()+ .total+ .value,+ 2+ );+ }+}+#[cfg(test)]mod semantic_deadline_regression_tests {use super::*;@@ -3466,6 +3784,14 @@ pub struct Index {write_publication: Arc<crate::write_publication::WritePublicationCoordinator>,#[cfg(test)]publication_test_hook: Arc<parking_lot::RwLock<Option<PublicationTestHook>>>,+ #[cfg(test)]+ test_panic_after_merge_publish: Arc<AtomicBool>,+ #[cfg(test)]+ test_fail_merge_before_apply: Arc<AtomicBool>,+ #[cfg(test)]+ test_pause_merge_before_apply: Arc<AtomicBool>,+ #[cfg(test)]+ test_merge_repoint_ready: Arc<AtomicBool>,doc_count: Arc<AtomicU64>,/// Counter for `update` operations that detected no change to the/// existing source — surfaced via `indices.stats` as@@ -4096,6 +4422,14 @@ impl Index {write_publication: crate::write_publication::WritePublicationCoordinator::new(),#[cfg(test)]publication_test_hook: Arc::new(parking_lot::RwLock::new(None)),+ #[cfg(test)]+ test_panic_after_merge_publish: Arc::new(AtomicBool::new(false)),+ #[cfg(test)]+ test_fail_merge_before_apply: Arc::new(AtomicBool::new(false)),+ #[cfg(test)]+ test_pause_merge_before_apply: Arc::new(AtomicBool::new(false)),+ #[cfg(test)]+ test_merge_repoint_ready: Arc::new(AtomicBool::new(false)),doc_count: Arc::new(AtomicU64::new(0)),noop_update_count: Arc::new(AtomicU64::new(0)),request_cache_seen: Arc::new(RwLock::new(RequestCacheSeen::with_capacity(65_536))),@@ -4469,6 +4803,14 @@ impl Index {write_publication: crate::write_publication::WritePublicationCoordinator::new(),#[cfg(test)]publication_test_hook: Arc::new(parking_lot::RwLock::new(None)),+ #[cfg(test)]+ test_panic_after_merge_publish: Arc::new(AtomicBool::new(false)),+ #[cfg(test)]+ test_fail_merge_before_apply: Arc::new(AtomicBool::new(false)),+ #[cfg(test)]+ test_pause_merge_before_apply: Arc::new(AtomicBool::new(false)),+ #[cfg(test)]+ test_merge_repoint_ready: Arc::new(AtomicBool::new(false)),doc_count: Arc::new(AtomicU64::new(total_doc_count)),noop_update_count: Arc::new(AtomicU64::new(0)),request_cache_seen: Arc::new(RwLock::new(RequestCacheSeen::with_capacity(65_536))),@@ -6592,6 +6934,9 @@ impl Index {// never got one, forcing the slow decode-stored fallback in// `rebuild_version_map_from_segments` on every reopen).Vec<(u64, String)>,+ // Rolls pre-publication VersionMap repoints back unless the+ // async driver commits them immediately after apply_merge.+ xerj_storage::version_map::VersionRepointTransaction,);// Build the list of (batch, metas) pairs we'll launch.@@ -7080,10 +7425,15 @@ impl Index {// `set`: a doc updated while the merge ran already has a// newer entry that must not be clobbered by the merged// (older) copy.+ let mut version_repoints =+ xerj_storage::version_map::VersionRepointTransaction::with_capacity(+ Arc::clone(&store_for_task.version_map),+ ids_pairs.len().saturating_add(tomb_carry.len()),+ );{let merged_arc: Arc<str> = Arc::from(merged_meta.id.as_str());for (seq_no, id) in &ids_pairs {- store_for_task.version_map.set_if_latest(+ version_repoints.repoint_if_latest(id.as_str(),*seq_no,Arc::clone(&merged_arc),@@ -7098,7 +7448,7 @@ impl Index {// (`set_if_latest`): a doc re-indexed while the// merge ran keeps its newer live entry.for (seq, id) in &tomb_carry {- store_for_task.version_map.set_if_latest(+ version_repoints.repoint_if_latest(id.as_str(),*seq,Arc::clone(&merged_arc),@@ -7128,6 +7478,7 @@ impl Index {merged_meta,live_doc_count as usize,ids_pairs,+ version_repoints,))})})@@ -7152,7 +7503,13 @@ impl Index {// order.let handle = in_flight.remove(0);match handle.await {- Ok(Some((batch_slice, merged_meta, live_doc_count, ids_pairs))) => {+ Ok(Some((+ batch_slice,+ merged_meta,+ live_doc_count,+ ids_pairs,+ version_repoints,+ ))) => {// Warm the merged segment's stored slices BEFORE the// swap makes it visible: the first sorted-candidates// queries after a merge otherwise each pay the fresh@@ -7241,13 +7598,66 @@ impl Index {}}}- if let Err(e) = self.store.apply_merge(&batch_slice, merged_meta.clone()) {- tracing::warn!("merge: apply_merge failed: {e}");+ #[cfg(test)]+ {+ self.test_merge_repoint_ready.store(true, Ordering::Release);+ while self.test_pause_merge_before_apply.load(Ordering::Acquire) {+ tokio::task::yield_now().await;+ }+ self.test_merge_repoint_ready+ .store(false, Ordering::Release);+ }+ #[cfg(test)]+ let apply_result = if self+ .test_fail_merge_before_apply+ .swap(false, Ordering::AcqRel)+ {+ Err(+ xerj_storage::index_store::MergePublicationError::NotPublished(+ xerj_storage::StorageError::MergeAborted(+ "injected merge apply failure before snapshot publication"+ .into(),+ ),+ ),+ )+ } else if self+ .test_panic_after_merge_publish+ .swap(false, Ordering::AcqRel)+ {+ self.store.apply_merge_with_repoints_and_post_publish(+ &batch_slice,+ merged_meta.clone(),+ version_repoints,+ || panic!("injected panic after durable merge publication"),+ )+ } else {+ self.store.apply_merge_with_repoints(+ &batch_slice,+ merged_meta.clone(),+ version_repoints,+ )+ };+ #[cfg(not(test))]+ let apply_result = self.store.apply_merge_with_repoints(+ &batch_slice,+ merged_meta.clone(),+ version_repoints,+ );+ if let Err(e) = apply_result {+ tracing::warn!("merge: apply_merge failed: {e:?}");+ failed_batches.fetch_add(1, std::sync::atomic::Ordering::Relaxed);// Seeded shortcut counts for a segment that never// became visible — drop them. The armed lease removes// all seven hydration owners when this scope exits.self.shortcut_count_cache.retain(|(seg, _), _| seg != merged_meta.id.as_str());+ // `continue` bypasses the common top-up below. Launch+ // the next disjoint batch first so one failed apply+ // does not silently suppress the rest of this pass.+ if let Some((batch, metas)) = pending.take() {+ in_flight.push(spawn_one(batch, metas));+ pending = queue_iter.next();+ }continue;}// `apply_merge` has made the output authoritative. Nodiff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rsindex 567e84d18..e9d46950d 100644--- a/engine/crates/xerj-storage/src/index_store.rs+++ b/engine/crates/xerj-storage/src/index_store.rs@@ -31,7 +31,7 @@ use uuid::Uuid;use crate::backend::StorageBackend;use crate::segment::{SectionType, SegmentId, SegmentMeta, SegmentReader, SegmentWriter};-use crate::version_map::{VersionMap, IN_MEMORY_SEGMENT_ID};+use crate::version_map::{VersionMap, VersionRepointTransaction, IN_MEMORY_SEGMENT_ID};use crate::wal::{SyncMode, WalEntry, WalWriter};use crate::{Result, SeqNo, StorageError};@@ -546,6 +546,10 @@ pub struct IndexStore {#[cfg(test)]fail_next_snapshot_save: std::sync::atomic::AtomicBool,#[cfg(test)]+ fail_snapshot_save_after: std::sync::atomic::AtomicUsize,+ #[cfg(test)]+ snapshot_save_failures_remaining: std::sync::atomic::AtomicUsize,+ #[cfg(test)]fail_next_wal_maintenance: std::sync::atomic::AtomicBool,#[cfg(test)]publication_failpoint: std::sync::atomic::AtomicU8,@@ -553,6 +557,22 @@ pub struct IndexStore {orphan_disarm_fail_after: std::sync::atomic::AtomicUsize,}+/// Authoritative publication state returned by transactional merge apply.+#[derive(Debug)]+pub enum MergePublicationOutcome {+ Published { maintenance_deferred: bool },+}++/// Failure classification for transactional merge publication.+#[derive(Debug)]+pub enum MergePublicationError {+ NotPublished(StorageError),+ Indeterminate {+ publication: StorageError,+ rollback: StorageError,+ },+}+/// Cached verification state of one rotated WAL generation.enum WalGenVerdict {/// Every entry proven durable-or-superseded — prunable now.@@ -637,6 +657,10 @@ impl IndexStore {#[cfg(test)]fail_next_snapshot_save: std::sync::atomic::AtomicBool::new(false),#[cfg(test)]+ fail_snapshot_save_after: std::sync::atomic::AtomicUsize::new(usize::MAX),+ #[cfg(test)]+ snapshot_save_failures_remaining: std::sync::atomic::AtomicUsize::new(1),+ #[cfg(test)]fail_next_wal_maintenance: std::sync::atomic::AtomicBool::new(false),#[cfg(test)]publication_failpoint: std::sync::atomic::AtomicU8::new(0),@@ -2881,6 +2905,30 @@ impl IndexStore {"injected post-publication snapshot save failure",)));}+ #[cfg(test)]+ {+ let remaining = self.fail_snapshot_save_after.load(Ordering::Acquire);+ if remaining != usize::MAX {+ if remaining == 0 {+ let failures = self+ .snapshot_save_failures_remaining+ .fetch_sub(1, Ordering::AcqRel);+ if failures > 0 {+ if failures == 1 {+ self.fail_snapshot_save_after+ .store(usize::MAX, Ordering::Release);+ self.snapshot_save_failures_remaining+ .store(1, Ordering::Release);+ }+ return Err(StorageError::Io(std::io::Error::other(+ "injected delayed snapshot save failure",+ )));+ }+ }+ self.fail_snapshot_save_after+ .store(remaining - 1, Ordering::Release);+ }+ }let snap = self.snapshot.load();// P2.3 — `to_vec` (compact) not `to_vec_pretty`: the snapshot is a// machine-read manifest (loaded via `from_slice`), never@@ -3332,21 +3380,69 @@ impl IndexStore {/// atomically replace merged segments with the merged result and update/// the version map.pub fn apply_merge(&self, merged_ids: &[SegmentId], new_meta: SegmentMeta) -> Result<()> {+ self.apply_merge_inner(merged_ids, new_meta, None, || {})+ .map(|_| ())+ .map_err(|error| match error {+ MergePublicationError::NotPublished(error) => error,+ MergePublicationError::Indeterminate {+ publication,+ rollback,+ } => StorageError::MergeAborted(format!(+ "merge publication is indeterminate: publication failed ({publication}); rollback persistence failed ({rollback})"+ )),+ })+ }++ /// Publish a merge and take ownership of its provisional version-map+ /// repoints. The transaction commits at the exact durable manifest+ /// boundary, before any fallible or panic-capable post-publication work.+ pub fn apply_merge_with_repoints(+ &self,+ merged_ids: &[SegmentId],+ new_meta: SegmentMeta,+ version_repoints: VersionRepointTransaction,+ ) -> std::result::Result<MergePublicationOutcome, MergePublicationError> {+ self.apply_merge_inner(merged_ids, new_meta, Some(version_repoints), || {})+ }++ /// Test seam for exercising cancellation/panic immediately after durable+ /// publication. Production callers use [`Self::apply_merge_with_repoints`].+ #[doc(hidden)]+ pub fn apply_merge_with_repoints_and_post_publish<F: FnOnce()>(+ &self,+ merged_ids: &[SegmentId],+ new_meta: SegmentMeta,+ version_repoints: VersionRepointTransaction,+ post_publish: F,+ ) -> std::result::Result<MergePublicationOutcome, MergePublicationError> {+ self.apply_merge_inner(merged_ids, new_meta, Some(version_repoints), post_publish)+ }++ fn apply_merge_inner<F: FnOnce()>(+ &self,+ merged_ids: &[SegmentId],+ new_meta: SegmentMeta,+ version_repoints: Option<VersionRepointTransaction>,+ post_publish: F,+ ) -> std::result::Result<MergePublicationOutcome, MergePublicationError> {// Inputs still belong to the authoritative snapshot at this point.// Durably remove their ZID3 completion markers before committing the// replacement, so no crash after snapshot publication can resurrect// a partially retired input. The ids sidecars stay readable if this// step fails and the merge is aborted.- self.disarm_orphan_recovery_for_segments(merged_ids)?;+ self.disarm_orphan_recovery_for_segments(merged_ids)+ .map_err(MergePublicationError::NotPublished)?;// Sum the doc counts of the segments we're about to replace, so we can// tell whether this merge actually dropped any documents.- let merged_total: u64 = {+ let (merged_total, previous_metas): (u64, Vec<SegmentMeta>) = {let snap = self.snapshot.load();- snap.segments+ let previous: Vec<_> = snap+ .segments.iter().filter(|s| merged_ids.contains(&s.id))- .map(|s| s.doc_count)- .sum()+ .cloned()+ .collect();+ (previous.iter().map(|meta| meta.doc_count).sum(), previous)};// Atomic replace via rcu — same race as `with_new_segment` in// `finalize_flush_with_publisher`: a concurrent flush appending@@ -3354,6 +3450,54 @@ impl IndexStore {// segment swap. rcu retries on contention.self.snapshot.rcu(|old| Arc::new(old.replace_segments(merged_ids, new_meta.clone())));+ if let Err(error) = self.save_snapshot() {+ // Publication changed memory before manifest persistence. Restore+ // the exact inputs only while this output remains authoritative;+ // the RCU closure preserves any concurrent flush append.+ let output_id = new_meta.id.clone();+ self.snapshot.rcu(|current| {+ let output_is_current = current.segments.iter().any(|meta| meta.id == output_id)+ && previous_metas.iter().all(|previous| {+ !current.segments.iter().any(|meta| meta.id == previous.id)+ });+ if !output_is_current {+ return Arc::clone(current);+ }+ let mut segments: Vec<_> = current+ .segments+ .iter()+ .filter(|meta| meta.id != output_id)+ .cloned()+ .collect();+ segments.extend(previous_metas.iter().cloned());+ let max_seq_no = segments+ .iter()+ .map(|meta| meta.max_seq_no)+ .max()+ .unwrap_or(0);+ Arc::new(IndexSnapshot {+ segments,+ generation: current.generation + 1,+ max_seq_no,+ })+ });+ // The initial error can occur after rename. Best-effort persist+ // the restored snapshot so memory and restart state converge.+ return match self.save_snapshot() {+ Ok(()) => Err(MergePublicationError::NotPublished(error)),+ Err(rollback) => Err(MergePublicationError::Indeterminate {+ publication: error,+ rollback,+ }),+ };+ }+ // The replacement is now durable. Commit provisional repoints here,+ // in the same synchronous ownership scope, so cancellation or panic+ // after publication cannot roll them back to retired inputs.+ if let Some(version_repoints) = version_repoints {+ version_repoints.commit();+ }+ post_publish();// `remove_segment` does a full O(N) `DashMap::retain` over the ENTIRE// version map, holding each shard's write lock — a >1s read-collapse// under merge pressure once the map holds millions of entries (reads@@ -3371,9 +3515,10 @@ impl IndexStore {if new_meta.doc_count < merged_total {self.version_map.remove_segment(merged_ids);}- self.save_snapshot()?;info!(merged = merged_ids.len(), "merge applied");- Ok(())+ Ok(MergePublicationOutcome::Published {+ maintenance_deferred: false,+ })}/// Returns stats useful for triggering merges.@@ -5565,4 +5710,74 @@ mod tests {store.index("x", serde_json::json!({"a": 1})).unwrap();assert!(store.version_map.get("x").is_some());}++ #[test]+ fn apply_merge_manifest_failure_restores_exact_input_snapshot() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ store.index("a", serde_json::json!({"value": "a"})).unwrap();+ store.flush().unwrap().unwrap();+ store.index("b", serde_json::json!({"value": "b"})).unwrap();+ store.flush().unwrap().unwrap();+ let before = store.snapshot();+ assert_eq!(before.segments.len(), 2);+ let before_ids: std::collections::HashSet<_> =+ before.segments.iter().map(|meta| meta.id.clone()).collect();++ let mut output = before.segments[0].clone();+ output.id = "injected-unpublished-output".into();+ output.seg_path = "segments/injected-unpublished-output.seg".into();+ output.sidx_path = "segments/injected-unpublished-output.sidx".into();+ let input_ids: Vec<_> = before.segments.iter().map(|meta| meta.id.clone()).collect();+ // apply_merge first persists the authoritative inputs before it+ // disarms recovery markers; fail the following publication save.+ store.fail_snapshot_save_after.store(1, Ordering::Release);+ assert!(store.apply_merge(&input_ids, output).is_err());++ let after_ids: std::collections::HashSet<_> = store+ .snapshot()+ .segments+ .iter()+ .map(|meta| meta.id.clone())+ .collect();+ assert_eq!(after_ids, before_ids);+ drop(before);+ drop(store);++ let reopened = open_test_store(dir.path());+ let reopened_ids: std::collections::HashSet<_> = reopened+ .snapshot()+ .segments+ .iter()+ .map(|meta| meta.id.clone())+ .collect();+ assert_eq!(reopened_ids, before_ids);+ assert_eq!(reopened.version_map.live_count(), 2);+ }++ #[test]+ fn apply_merge_reports_indeterminate_when_publication_and_rollback_saves_fail() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ store.index("a", serde_json::json!({"value": "a"})).unwrap();+ store.flush().unwrap().unwrap();+ store.index("b", serde_json::json!({"value": "b"})).unwrap();+ store.flush().unwrap().unwrap();+ let before = store.snapshot();+ let mut output = before.segments[0].clone();+ output.id = "indeterminate-output".into();+ let input_ids: Vec<_> = before.segments.iter().map(|meta| meta.id.clone()).collect();+ let transaction =+ VersionRepointTransaction::with_capacity(Arc::clone(&store.version_map), 0);+ // The pre-disarm input save succeeds; publication and rollback saves+ // then both fail. The API must not claim exact rollback durability.+ store.fail_snapshot_save_after.store(1, Ordering::Release);+ store+ .snapshot_save_failures_remaining+ .store(2, Ordering::Release);+ let error = store+ .apply_merge_with_repoints(&input_ids, output, transaction)+ .unwrap_err();+ assert!(matches!(error, MergePublicationError::Indeterminate { .. }));+ }}diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rsindex 05a0511a5..90e0e81ee 100644--- a/engine/crates/xerj-storage/src/version_map.rs+++ b/engine/crates/xerj-storage/src/version_map.rs@@ -119,6 +119,127 @@ pub struct VersionMap {ghost_events: std::sync::atomic::AtomicU64,}+#[derive(Debug)]+struct VersionRepointChange {+ doc_id: String,+ previous: Option<VersionEntry>,+ installed: VersionEntry,+}++/// Panic- and cancellation-safe group of physical version-map repoints.+///+/// Merge output identities are repointed before snapshot publication. Unless+/// publication commits this transaction, drop restores only entries that+/// still exactly match values installed by the transaction. A concurrent+/// newer write or delete therefore always wins over rollback.+pub struct VersionRepointTransaction {+ map: Arc<VersionMap>,+ changes: Vec<VersionRepointChange>,+ committed: bool,+}++impl VersionRepointTransaction {+ /// Reserve the complete rollback log before mutating the map.+ pub fn with_capacity(map: Arc<VersionMap>, capacity: usize) -> Self {+ Self {+ map,+ changes: Vec::with_capacity(capacity),+ committed: false,+ }+ }++ /// Conditionally repoint an entry and remember the exact displaced value.+ pub fn repoint_if_latest(+ &mut self,+ doc_id: &str,+ seq_no: SeqNo,+ segment_id: Arc<str>,+ deleted: bool,+ ) {+ debug_assert!(+ self.changes.len() < self.changes.capacity(),+ "merge rollback log must be reserved before repoint"+ );+ use dashmap::mapref::entry::Entry;+ let (previous, installed) = match self.map.inner.entry(doc_id.to_owned()) {+ Entry::Occupied(mut occupied) => {+ if seq_no < occupied.get().seq_no {+ return;+ }+ let previous = occupied.get().clone();+ let installed = VersionEntry {+ seq_no,+ segment_id,+ deleted,+ version: previous.version,+ };+ occupied.insert(installed.clone());+ if previous.seq_no != seq_no || (deleted && !previous.deleted) {+ self.map.ghost_events.fetch_add(1, Ordering::Relaxed);+ }+ self.map.live_delta(!previous.deleted, !deleted);+ (Some(previous), installed)+ }+ Entry::Vacant(vacant) => {+ let installed = VersionEntry {+ seq_no,+ segment_id,+ deleted,+ version: 1,+ };+ vacant.insert(installed.clone());+ if deleted {+ self.map.ghost_events.fetch_add(1, Ordering::Relaxed);+ }+ self.map.live_delta(false, !deleted);+ (None, installed)+ }+ };+ self.changes.push(VersionRepointChange {+ doc_id: doc_id.to_owned(),+ previous,+ installed,+ });+ }++ /// Keep all repoints after the snapshot publication succeeds.+ pub fn commit(mut self) {+ self.committed = true;+ }++ fn rollback(&mut self) {+ use dashmap::mapref::entry::Entry;+ for change in self.changes.drain(..).rev() {+ let Entry::Occupied(mut occupied) = self.map.inner.entry(change.doc_id) else {+ continue;+ };+ if occupied.get() != &change.installed {+ continue;+ }+ let current_live = !occupied.get().deleted;+ match change.previous {+ Some(previous) => {+ let previous_live = !previous.deleted;+ occupied.insert(previous);+ self.map.live_delta(current_live, previous_live);+ }+ None => {+ occupied.remove();+ self.map.live_delta(current_live, false);+ }+ }+ }+ }+}++impl Drop for VersionRepointTransaction {+ fn drop(&mut self) {+ if !self.committed {+ self.rollback();+ }+ }+}+impl VersionMap {/// Create an empty version map.pub fn new() -> Self {@@ -655,6 +776,71 @@ mod tests {assert_eq!(vm.live_count(), 1); // "x" live, "d" tombstoned}+ #[test]+ fn merge_repoint_transaction_rolls_back_exact_previous_entries() {+ let vm = Arc::new(VersionMap::new());+ vm.set("live", 7, "input-a", false);+ vm.set("tomb", 8, "input-b", true);+ let live_before = vm.live_count();+ {+ let mut transaction = VersionRepointTransaction::with_capacity(Arc::clone(&vm), 3);+ transaction.repoint_if_latest("live", 7, Arc::from("output"), false);+ transaction.repoint_if_latest("tomb", 8, Arc::from("output"), true);+ transaction.repoint_if_latest("new-tomb", 9, Arc::from("output"), true);+ assert_eq!(&*vm.get("live").unwrap().segment_id, "output");+ assert_eq!(&*vm.get("tomb").unwrap().segment_id, "output");+ }+ assert_eq!(&*vm.get("live").unwrap().segment_id, "input-a");+ assert_eq!(&*vm.get("tomb").unwrap().segment_id, "input-b");+ assert!(vm.get("new-tomb").is_none());+ assert_eq!(vm.live_count(), live_before);+ }++ #[test]+ fn merge_repoint_rollback_preserves_concurrent_update_and_delete() {+ let vm = Arc::new(VersionMap::new());+ vm.set("updated", 1, "input", false);+ vm.set("deleted", 2, "input", false);+ {+ let mut transaction = VersionRepointTransaction::with_capacity(Arc::clone(&vm), 2);+ transaction.repoint_if_latest("updated", 1, Arc::from("output"), false);+ transaction.repoint_if_latest("deleted", 2, Arc::from("output"), false);+ vm.set("updated", 10, IN_MEMORY_SEGMENT_ID, false);+ vm.delete("deleted", 11, IN_MEMORY_SEGMENT_ID).unwrap();+ }+ let updated = vm.get("updated").unwrap();+ assert_eq!(+ (updated.seq_no, &*updated.segment_id, updated.deleted),+ (10, IN_MEMORY_SEGMENT_ID, false)+ );+ let deleted = vm.get("deleted").unwrap();+ assert_eq!(+ (deleted.seq_no, &*deleted.segment_id, deleted.deleted),+ (11, IN_MEMORY_SEGMENT_ID, true)+ );+ }++ #[test]+ fn merge_repoint_transaction_is_panic_safe_and_commit_is_final() {+ let vm = Arc::new(VersionMap::new());+ vm.set("doc", 1, "input", false);+ let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe({+ let vm = Arc::clone(&vm);+ move || {+ let mut transaction = VersionRepointTransaction::with_capacity(Arc::clone(&vm), 1);+ transaction.repoint_if_latest("doc", 1, Arc::from("failed-output"), false);+ panic!("injected merge worker panic after repoint");+ }+ }));+ assert!(panic_result.is_err());+ assert_eq!(&*vm.get("doc").unwrap().segment_id, "input");++ let mut transaction = VersionRepointTransaction::with_capacity(Arc::clone(&vm), 1);+ transaction.repoint_if_latest("doc", 1, Arc::from("committed-output"), false);+ transaction.commit();+ assert_eq!(&*vm.get("doc").unwrap().segment_id, "committed-output");+ }+#[test]fn check_and_set_success() {let vm = VersionMap::new();