Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): transactional merge publication and rollback (#140)

xerj-orgba7bd23975e6
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 7b158b543..efbf19b08 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -996,6 +996,324 @@ impl<'a> Drop for MergeFlagClear<'a> {
}
}
+#[cfg(test)]
+mod merge_publication_transaction_tests {
+ use super::*;
+ use crate::Engine;
+ use tempfile::TempDir;
+
+ fn config(dir: &TempDir) -> xerj_common::config::Config {
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.merge.min_segments = 2;
+ config.merge.min_merge_count = 2;
+ config.merge.max_merge_count = 2;
+ config
+ }
+
+ async fn fixture(dir: &TempDir) -> (Engine, Arc<Index>) {
+ let engine = Engine::new(config(dir)).unwrap();
+ engine
+ .create_index("post-save-panic", Schema::empty())
+ .unwrap();
+ let index = engine.get_index("post-save-panic").unwrap();
+ index
+ .index_document(Some("a".into()), serde_json::json!({"value": "old-a"}))
+ .await
+ .unwrap();
+ index.flush().await.unwrap();
+ index
+ .index_document(Some("b".into()), serde_json::json!({"value": "old-b"}))
+ .await
+ .unwrap();
+ index.flush().await.unwrap();
+ assert_eq!(index.store.snapshot().segments.len(), 2);
+ (engine, index)
+ }
+
+ async fn wait_for_repoint(index: &Index) {
+ tokio::time::timeout(std::time::Duration::from_secs(10), async {
+ while !index.test_merge_repoint_ready.load(Ordering::Acquire) {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("merge did not reach the pre-apply gate");
+ }
+
+ #[tokio::test]
+ async fn apply_failure_restores_sources_survives_restart_and_retry() {
+ let dir = TempDir::new().unwrap();
+ let (engine, index) = fixture(&dir).await;
+ let before_ids: std::collections::HashSet<_> = index
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|meta| meta.id.clone())
+ .collect();
+ index
+ .test_fail_merge_before_apply
+ .store(true, Ordering::Release);
+ assert!(index.run_merge_once().await.is_err());
+ let after_ids: std::collections::HashSet<_> = index
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|meta| meta.id.clone())
+ .collect();
+ assert_eq!(after_ids, before_ids);
+ for (id, expected) in [("a", "old-a"), ("b", "old-b")] {
+ assert_eq!(
+ index.get_document(id).await.unwrap().unwrap()["value"],
+ expected
+ );
+ }
+ assert_eq!(
+ index
+ .search(&SearchRequest::default())
+ .await
+ .unwrap()
+ .total
+ .value,
+ 2
+ );
+ drop(index);
+ drop(engine);
+
+ let reopened = Engine::new(config(&dir)).unwrap();
+ let reopened_index = reopened.get_index("post-save-panic").unwrap();
+ for (id, expected) in [("a", "old-a"), ("b", "old-b")] {
+ assert_eq!(
+ reopened_index.get_document(id).await.unwrap().unwrap()["value"],
+ expected
+ );
+ }
+ assert_eq!(reopened_index.run_merge_once().await.unwrap(), 1);
+ assert_eq!(reopened_index.store.snapshot().segments.len(), 1);
+ assert_eq!(
+ reopened_index
+ .search(&SearchRequest::default())
+ .await
+ .unwrap()
+ .total
+ .value,
+ 2
+ );
+ }
+
+ #[tokio::test]
+ async fn rollback_preserves_concurrent_put_and_delete() {
+ let dir = TempDir::new().unwrap();
+ let (engine, index) = fixture(&dir).await;
+ index
+ .test_pause_merge_before_apply
+ .store(true, Ordering::Release);
+ index
+ .test_fail_merge_before_apply
+ .store(true, Ordering::Release);
+ let merge_index = Arc::clone(&index);
+ let merge = tokio::spawn(async move { merge_index.run_merge_once().await });
+ wait_for_repoint(&index).await;
+ index
+ .index_document(Some("a".into()), serde_json::json!({"value": "new-a"}))
+ .await
+ .unwrap();
+ assert!(index.delete_document("b").await.unwrap());
+ index
+ .test_pause_merge_before_apply
+ .store(false, Ordering::Release);
+ assert!(merge.await.unwrap().is_err());
+ assert_eq!(
+ index.get_document("a").await.unwrap().unwrap()["value"],
+ "new-a"
+ );
+ assert!(index.get_document("b").await.unwrap().is_none());
+ assert!(index.store.version_map.get("b").unwrap().deleted);
+ assert_eq!(
+ index
+ .search(&SearchRequest::default())
+ .await
+ .unwrap()
+ .total
+ .value,
+ 1
+ );
+ drop(index);
+ drop(engine);
+ let reopened = Engine::new(config(&dir)).unwrap();
+ let reopened_index = reopened.get_index("post-save-panic").unwrap();
+ assert_eq!(
+ reopened_index.get_document("a").await.unwrap().unwrap()["value"],
+ "new-a"
+ );
+ assert!(reopened_index.get_document("b").await.unwrap().is_none());
+ assert!(reopened_index.store.version_map.get("b").unwrap().deleted);
+ assert_eq!(
+ reopened_index
+ .search(&SearchRequest::default())
+ .await
+ .unwrap()
+ .total
+ .value,
+ 1
+ );
+ }
+
+ #[tokio::test]
+ async fn cancellation_after_repoint_restores_inputs() {
+ let dir = TempDir::new().unwrap();
+ let (_engine, index) = fixture(&dir).await;
+ let before_ids: std::collections::HashSet<_> = index
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|meta| meta.id.clone())
+ .collect();
+ index
+ .test_pause_merge_before_apply
+ .store(true, Ordering::Release);
+ let merge_index = Arc::clone(&index);
+ let merge = tokio::spawn(async move { merge_index.run_merge_once().await });
+ wait_for_repoint(&index).await;
+ merge.abort();
+ assert!(merge.await.unwrap_err().is_cancelled());
+ index
+ .test_pause_merge_before_apply
+ .store(false, Ordering::Release);
+ let after_ids: std::collections::HashSet<_> = index
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|meta| meta.id.clone())
+ .collect();
+ assert_eq!(after_ids, before_ids);
+ for id in ["a", "b"] {
+ assert!(index.get_document(id).await.unwrap().is_some());
+ assert!(
+ before_ids.contains(index.store.version_map.get(id).unwrap().segment_id.as_ref())
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn partial_multi_batch_failure_is_reported_without_losing_either_batch() {
+ let dir = TempDir::new().unwrap();
+ let engine = Engine::new(config(&dir)).unwrap();
+ engine.create_index("multi-batch", Schema::empty()).unwrap();
+ let index = engine.get_index("multi-batch").unwrap();
+ for id in ["a", "b", "c", "d"] {
+ index
+ .index_document(
+ Some(id.into()),
+ serde_json::json!({"value": format!("source-{id}")}),
+ )
+ .await
+ .unwrap();
+ index.flush().await.unwrap();
+ }
+ assert_eq!(index.store.snapshot().segments.len(), 4);
+ index
+ .test_fail_merge_before_apply
+ .store(true, Ordering::Release);
+ assert!(index.run_merge_once().await.is_err());
+ assert_eq!(index.store.snapshot().segments.len(), 3);
+ for id in ["a", "b", "c", "d"] {
+ assert_eq!(
+ index.get_document(id).await.unwrap().unwrap()["value"],
+ format!("source-{id}")
+ );
+ }
+ assert_eq!(
+ index
+ .search(&SearchRequest::default())
+ .await
+ .unwrap()
+ .total
+ .value,
+ 4
+ );
+ drop(index);
+ drop(engine);
+
+ let reopened = Engine::new(config(&dir)).unwrap();
+ let reopened_index = reopened.get_index("multi-batch").unwrap();
+ for id in ["a", "b", "c", "d"] {
+ assert_eq!(
+ reopened_index.get_document(id).await.unwrap().unwrap()["value"],
+ format!("source-{id}")
+ );
+ }
+ assert_eq!(
+ reopened_index
+ .search(&SearchRequest::default())
+ .await
+ .unwrap()
+ .total
+ .value,
+ 4
+ );
+ }
+
+ #[tokio::test]
+ async fn panic_after_durable_snapshot_keeps_output_repoints_and_survives_restart() {
+ let dir = TempDir::new().unwrap();
+ let (engine, index) = fixture(&dir).await;
+ index
+ .test_panic_after_merge_publish
+ .store(true, Ordering::Release);
+ let merge_index = Arc::clone(&index);
+ let join = tokio::spawn(async move { merge_index.run_merge_once().await }).await;
+ assert!(join.unwrap_err().is_panic());
+
+ let snapshot = index.store.snapshot();
+ assert_eq!(snapshot.segments.len(), 1);
+ let output_id = snapshot.segments[0].id.clone();
+ drop(snapshot);
+ for (id, expected) in [("a", "old-a"), ("b", "old-b")] {
+ let entry = index.store.version_map.get(id).unwrap();
+ assert_eq!(entry.segment_id.as_ref(), output_id);
+ assert_eq!(
+ index.get_document(id).await.unwrap().unwrap()["value"],
+ expected
+ );
+ }
+ assert_eq!(
+ index
+ .search(&SearchRequest::default())
+ .await
+ .unwrap()
+ .total
+ .value,
+ 2
+ );
+ drop(index);
+ drop(engine);
+
+ let reopened = Engine::new(config(&dir)).unwrap();
+ let reopened_index = reopened.get_index("post-save-panic").unwrap();
+ assert_eq!(reopened_index.store.snapshot().segments.len(), 1);
+ for (id, expected) in [("a", "old-a"), ("b", "old-b")] {
+ assert_eq!(
+ reopened_index.get_document(id).await.unwrap().unwrap()["value"],
+ expected
+ );
+ }
+ assert_eq!(
+ reopened_index
+ .search(&SearchRequest::default())
+ .await
+ .unwrap()
+ .total
+ .value,
+ 2
+ );
+ }
+}
+
#[cfg(test)]
mod semantic_deadline_regression_tests {
use super::*;
@@ -3466,6 +3784,14 @@ pub struct Index {
write_publication: Arc<crate::write_publication::WritePublicationCoordinator>,
#[cfg(test)]
publication_test_hook: Arc<parking_lot::RwLock<Option<PublicationTestHook>>>,
+ #[cfg(test)]
+ test_panic_after_merge_publish: Arc<AtomicBool>,
+ #[cfg(test)]
+ test_fail_merge_before_apply: Arc<AtomicBool>,
+ #[cfg(test)]
+ test_pause_merge_before_apply: Arc<AtomicBool>,
+ #[cfg(test)]
+ test_merge_repoint_ready: Arc<AtomicBool>,
doc_count: Arc<AtomicU64>,
/// Counter for `update` operations that detected no change to the
/// existing source — surfaced via `indices.stats` as
@@ -4096,6 +4422,14 @@ impl Index {
write_publication: crate::write_publication::WritePublicationCoordinator::new(),
#[cfg(test)]
publication_test_hook: Arc::new(parking_lot::RwLock::new(None)),
+ #[cfg(test)]
+ test_panic_after_merge_publish: Arc::new(AtomicBool::new(false)),
+ #[cfg(test)]
+ test_fail_merge_before_apply: Arc::new(AtomicBool::new(false)),
+ #[cfg(test)]
+ test_pause_merge_before_apply: Arc::new(AtomicBool::new(false)),
+ #[cfg(test)]
+ test_merge_repoint_ready: Arc::new(AtomicBool::new(false)),
doc_count: Arc::new(AtomicU64::new(0)),
noop_update_count: Arc::new(AtomicU64::new(0)),
request_cache_seen: Arc::new(RwLock::new(RequestCacheSeen::with_capacity(65_536))),
@@ -4469,6 +4803,14 @@ impl Index {
write_publication: crate::write_publication::WritePublicationCoordinator::new(),
#[cfg(test)]
publication_test_hook: Arc::new(parking_lot::RwLock::new(None)),
+ #[cfg(test)]
+ test_panic_after_merge_publish: Arc::new(AtomicBool::new(false)),
+ #[cfg(test)]
+ test_fail_merge_before_apply: Arc::new(AtomicBool::new(false)),
+ #[cfg(test)]
+ test_pause_merge_before_apply: Arc::new(AtomicBool::new(false)),
+ #[cfg(test)]
+ test_merge_repoint_ready: Arc::new(AtomicBool::new(false)),
doc_count: Arc::new(AtomicU64::new(total_doc_count)),
noop_update_count: Arc::new(AtomicU64::new(0)),
request_cache_seen: Arc::new(RwLock::new(RequestCacheSeen::with_capacity(65_536))),
@@ -6592,6 +6934,9 @@ impl Index {
// never got one, forcing the slow decode-stored fallback in
// `rebuild_version_map_from_segments` on every reopen).
Vec<(u64, String)>,
+ // Rolls pre-publication VersionMap repoints back unless the
+ // async driver commits them immediately after apply_merge.
+ xerj_storage::version_map::VersionRepointTransaction,
);
// Build the list of (batch, metas) pairs we'll launch.
@@ -7080,10 +7425,15 @@ impl Index {
// `set`: a doc updated while the merge ran already has a
// newer entry that must not be clobbered by the merged
// (older) copy.
+ let mut version_repoints =
+ xerj_storage::version_map::VersionRepointTransaction::with_capacity(
+ Arc::clone(&store_for_task.version_map),
+ ids_pairs.len().saturating_add(tomb_carry.len()),
+ );
{
let merged_arc: Arc<str> = Arc::from(merged_meta.id.as_str());
for (seq_no, id) in &ids_pairs {
- store_for_task.version_map.set_if_latest(
+ version_repoints.repoint_if_latest(
id.as_str(),
*seq_no,
Arc::clone(&merged_arc),
@@ -7098,7 +7448,7 @@ impl Index {
// (`set_if_latest`): a doc re-indexed while the
// merge ran keeps its newer live entry.
for (seq, id) in &tomb_carry {
- store_for_task.version_map.set_if_latest(
+ version_repoints.repoint_if_latest(
id.as_str(),
*seq,
Arc::clone(&merged_arc),
@@ -7128,6 +7478,7 @@ impl Index {
merged_meta,
live_doc_count as usize,
ids_pairs,
+ version_repoints,
))
})
})
@@ -7152,7 +7503,13 @@ impl Index {
// order.
let handle = in_flight.remove(0);
match handle.await {
- Ok(Some((batch_slice, merged_meta, live_doc_count, ids_pairs))) => {
+ Ok(Some((
+ batch_slice,
+ merged_meta,
+ live_doc_count,
+ ids_pairs,
+ version_repoints,
+ ))) => {
// Warm the merged segment's stored slices BEFORE the
// swap makes it visible: the first sorted-candidates
// queries after a merge otherwise each pay the fresh
@@ -7241,13 +7598,66 @@ impl Index {
}
}
}
- if let Err(e) = self.store.apply_merge(&batch_slice, merged_meta.clone()) {
- tracing::warn!("merge: apply_merge failed: {e}");
+ #[cfg(test)]
+ {
+ self.test_merge_repoint_ready.store(true, Ordering::Release);
+ while self.test_pause_merge_before_apply.load(Ordering::Acquire) {
+ tokio::task::yield_now().await;
+ }
+ self.test_merge_repoint_ready
+ .store(false, Ordering::Release);
+ }
+ #[cfg(test)]
+ let apply_result = if self
+ .test_fail_merge_before_apply
+ .swap(false, Ordering::AcqRel)
+ {
+ Err(
+ xerj_storage::index_store::MergePublicationError::NotPublished(
+ xerj_storage::StorageError::MergeAborted(
+ "injected merge apply failure before snapshot publication"
+ .into(),
+ ),
+ ),
+ )
+ } else if self
+ .test_panic_after_merge_publish
+ .swap(false, Ordering::AcqRel)
+ {
+ self.store.apply_merge_with_repoints_and_post_publish(
+ &batch_slice,
+ merged_meta.clone(),
+ version_repoints,
+ || panic!("injected panic after durable merge publication"),
+ )
+ } else {
+ self.store.apply_merge_with_repoints(
+ &batch_slice,
+ merged_meta.clone(),
+ version_repoints,
+ )
+ };
+ #[cfg(not(test))]
+ let apply_result = self.store.apply_merge_with_repoints(
+ &batch_slice,
+ merged_meta.clone(),
+ version_repoints,
+ );
+ if let Err(e) = apply_result {
+ tracing::warn!("merge: apply_merge failed: {e:?}");
+ failed_batches.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
// Seeded shortcut counts for a segment that never
// became visible — drop them. The armed lease removes
// all seven hydration owners when this scope exits.
self.shortcut_count_cache
.retain(|(seg, _), _| seg != merged_meta.id.as_str());
+ // `continue` bypasses the common top-up below. Launch
+ // the next disjoint batch first so one failed apply
+ // does not silently suppress the rest of this pass.
+ if let Some((batch, metas)) = pending.take() {
+ in_flight.push(spawn_one(batch, metas));
+ pending = queue_iter.next();
+ }
continue;
}
// `apply_merge` has made the output authoritative. No
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 567e84d18..e9d46950d 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -31,7 +31,7 @@ use uuid::Uuid;
use crate::backend::StorageBackend;
use crate::segment::{SectionType, SegmentId, SegmentMeta, SegmentReader, SegmentWriter};
-use crate::version_map::{VersionMap, IN_MEMORY_SEGMENT_ID};
+use crate::version_map::{VersionMap, VersionRepointTransaction, IN_MEMORY_SEGMENT_ID};
use crate::wal::{SyncMode, WalEntry, WalWriter};
use crate::{Result, SeqNo, StorageError};
@@ -546,6 +546,10 @@ pub struct IndexStore {
#[cfg(test)]
fail_next_snapshot_save: std::sync::atomic::AtomicBool,
#[cfg(test)]
+ fail_snapshot_save_after: std::sync::atomic::AtomicUsize,
+ #[cfg(test)]
+ snapshot_save_failures_remaining: std::sync::atomic::AtomicUsize,
+ #[cfg(test)]
fail_next_wal_maintenance: std::sync::atomic::AtomicBool,
#[cfg(test)]
publication_failpoint: std::sync::atomic::AtomicU8,
@@ -553,6 +557,22 @@ pub struct IndexStore {
orphan_disarm_fail_after: std::sync::atomic::AtomicUsize,
}
+/// Authoritative publication state returned by transactional merge apply.
+#[derive(Debug)]
+pub enum MergePublicationOutcome {
+ Published { maintenance_deferred: bool },
+}
+
+/// Failure classification for transactional merge publication.
+#[derive(Debug)]
+pub enum MergePublicationError {
+ NotPublished(StorageError),
+ Indeterminate {
+ publication: StorageError,
+ rollback: StorageError,
+ },
+}
+
/// Cached verification state of one rotated WAL generation.
enum WalGenVerdict {
/// Every entry proven durable-or-superseded — prunable now.
@@ -637,6 +657,10 @@ impl IndexStore {
#[cfg(test)]
fail_next_snapshot_save: std::sync::atomic::AtomicBool::new(false),
#[cfg(test)]
+ fail_snapshot_save_after: std::sync::atomic::AtomicUsize::new(usize::MAX),
+ #[cfg(test)]
+ snapshot_save_failures_remaining: std::sync::atomic::AtomicUsize::new(1),
+ #[cfg(test)]
fail_next_wal_maintenance: std::sync::atomic::AtomicBool::new(false),
#[cfg(test)]
publication_failpoint: std::sync::atomic::AtomicU8::new(0),
@@ -2881,6 +2905,30 @@ impl IndexStore {
"injected post-publication snapshot save failure",
)));
}
+ #[cfg(test)]
+ {
+ let remaining = self.fail_snapshot_save_after.load(Ordering::Acquire);
+ if remaining != usize::MAX {
+ if remaining == 0 {
+ let failures = self
+ .snapshot_save_failures_remaining
+ .fetch_sub(1, Ordering::AcqRel);
+ if failures > 0 {
+ if failures == 1 {
+ self.fail_snapshot_save_after
+ .store(usize::MAX, Ordering::Release);
+ self.snapshot_save_failures_remaining
+ .store(1, Ordering::Release);
+ }
+ return Err(StorageError::Io(std::io::Error::other(
+ "injected delayed snapshot save failure",
+ )));
+ }
+ }
+ self.fail_snapshot_save_after
+ .store(remaining - 1, Ordering::Release);
+ }
+ }
let snap = self.snapshot.load();
// P2.3 — `to_vec` (compact) not `to_vec_pretty`: the snapshot is a
// machine-read manifest (loaded via `from_slice`), never
@@ -3332,21 +3380,69 @@ impl IndexStore {
/// atomically replace merged segments with the merged result and update
/// the version map.
pub fn apply_merge(&self, merged_ids: &[SegmentId], new_meta: SegmentMeta) -> Result<()> {
+ self.apply_merge_inner(merged_ids, new_meta, None, || {})
+ .map(|_| ())
+ .map_err(|error| match error {
+ MergePublicationError::NotPublished(error) => error,
+ MergePublicationError::Indeterminate {
+ publication,
+ rollback,
+ } => StorageError::MergeAborted(format!(
+ "merge publication is indeterminate: publication failed ({publication}); rollback persistence failed ({rollback})"
+ )),
+ })
+ }
+
+ /// Publish a merge and take ownership of its provisional version-map
+ /// repoints. The transaction commits at the exact durable manifest
+ /// boundary, before any fallible or panic-capable post-publication work.
+ pub fn apply_merge_with_repoints(
+ &self,
+ merged_ids: &[SegmentId],
+ new_meta: SegmentMeta,
+ version_repoints: VersionRepointTransaction,
+ ) -> std::result::Result<MergePublicationOutcome, MergePublicationError> {
+ self.apply_merge_inner(merged_ids, new_meta, Some(version_repoints), || {})
+ }
+
+ /// Test seam for exercising cancellation/panic immediately after durable
+ /// publication. Production callers use [`Self::apply_merge_with_repoints`].
+ #[doc(hidden)]
+ pub fn apply_merge_with_repoints_and_post_publish<F: FnOnce()>(
+ &self,
+ merged_ids: &[SegmentId],
+ new_meta: SegmentMeta,
+ version_repoints: VersionRepointTransaction,
+ post_publish: F,
+ ) -> std::result::Result<MergePublicationOutcome, MergePublicationError> {
+ self.apply_merge_inner(merged_ids, new_meta, Some(version_repoints), post_publish)
+ }
+
+ fn apply_merge_inner<F: FnOnce()>(
+ &self,
+ merged_ids: &[SegmentId],
+ new_meta: SegmentMeta,
+ version_repoints: Option<VersionRepointTransaction>,
+ post_publish: F,
+ ) -> std::result::Result<MergePublicationOutcome, MergePublicationError> {
// Inputs still belong to the authoritative snapshot at this point.
// Durably remove their ZID3 completion markers before committing the
// replacement, so no crash after snapshot publication can resurrect
// a partially retired input. The ids sidecars stay readable if this
// step fails and the merge is aborted.
- self.disarm_orphan_recovery_for_segments(merged_ids)?;
+ self.disarm_orphan_recovery_for_segments(merged_ids)
+ .map_err(MergePublicationError::NotPublished)?;
// Sum the doc counts of the segments we're about to replace, so we can
// tell whether this merge actually dropped any documents.
- let merged_total: u64 = {
+ let (merged_total, previous_metas): (u64, Vec<SegmentMeta>) = {
let snap = self.snapshot.load();
- snap.segments
+ let previous: Vec<_> = snap
+ .segments
.iter()
.filter(|s| merged_ids.contains(&s.id))
- .map(|s| s.doc_count)
- .sum()
+ .cloned()
+ .collect();
+ (previous.iter().map(|meta| meta.doc_count).sum(), previous)
};
// Atomic replace via rcu — same race as `with_new_segment` in
// `finalize_flush_with_publisher`: a concurrent flush appending
@@ -3354,6 +3450,54 @@ impl IndexStore {
// segment swap. rcu retries on contention.
self.snapshot
.rcu(|old| Arc::new(old.replace_segments(merged_ids, new_meta.clone())));
+ if let Err(error) = self.save_snapshot() {
+ // Publication changed memory before manifest persistence. Restore
+ // the exact inputs only while this output remains authoritative;
+ // the RCU closure preserves any concurrent flush append.
+ let output_id = new_meta.id.clone();
+ self.snapshot.rcu(|current| {
+ let output_is_current = current.segments.iter().any(|meta| meta.id == output_id)
+ && previous_metas.iter().all(|previous| {
+ !current.segments.iter().any(|meta| meta.id == previous.id)
+ });
+ if !output_is_current {
+ return Arc::clone(current);
+ }
+ let mut segments: Vec<_> = current
+ .segments
+ .iter()
+ .filter(|meta| meta.id != output_id)
+ .cloned()
+ .collect();
+ segments.extend(previous_metas.iter().cloned());
+ let max_seq_no = segments
+ .iter()
+ .map(|meta| meta.max_seq_no)
+ .max()
+ .unwrap_or(0);
+ Arc::new(IndexSnapshot {
+ segments,
+ generation: current.generation + 1,
+ max_seq_no,
+ })
+ });
+ // The initial error can occur after rename. Best-effort persist
+ // the restored snapshot so memory and restart state converge.
+ return match self.save_snapshot() {
+ Ok(()) => Err(MergePublicationError::NotPublished(error)),
+ Err(rollback) => Err(MergePublicationError::Indeterminate {
+ publication: error,
+ rollback,
+ }),
+ };
+ }
+ // The replacement is now durable. Commit provisional repoints here,
+ // in the same synchronous ownership scope, so cancellation or panic
+ // after publication cannot roll them back to retired inputs.
+ if let Some(version_repoints) = version_repoints {
+ version_repoints.commit();
+ }
+ post_publish();
// `remove_segment` does a full O(N) `DashMap::retain` over the ENTIRE
// version map, holding each shard's write lock — a >1s read-collapse
// under merge pressure once the map holds millions of entries (reads
@@ -3371,9 +3515,10 @@ impl IndexStore {
if new_meta.doc_count < merged_total {
self.version_map.remove_segment(merged_ids);
}
- self.save_snapshot()?;
info!(merged = merged_ids.len(), "merge applied");
- Ok(())
+ Ok(MergePublicationOutcome::Published {
+ maintenance_deferred: false,
+ })
}
/// Returns stats useful for triggering merges.
@@ -5565,4 +5710,74 @@ mod tests {
store.index("x", serde_json::json!({"a": 1})).unwrap();
assert!(store.version_map.get("x").is_some());
}
+
+ #[test]
+ fn apply_merge_manifest_failure_restores_exact_input_snapshot() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ store.index("a", serde_json::json!({"value": "a"})).unwrap();
+ store.flush().unwrap().unwrap();
+ store.index("b", serde_json::json!({"value": "b"})).unwrap();
+ store.flush().unwrap().unwrap();
+ let before = store.snapshot();
+ assert_eq!(before.segments.len(), 2);
+ let before_ids: std::collections::HashSet<_> =
+ before.segments.iter().map(|meta| meta.id.clone()).collect();
+
+ let mut output = before.segments[0].clone();
+ output.id = "injected-unpublished-output".into();
+ output.seg_path = "segments/injected-unpublished-output.seg".into();
+ output.sidx_path = "segments/injected-unpublished-output.sidx".into();
+ let input_ids: Vec<_> = before.segments.iter().map(|meta| meta.id.clone()).collect();
+ // apply_merge first persists the authoritative inputs before it
+ // disarms recovery markers; fail the following publication save.
+ store.fail_snapshot_save_after.store(1, Ordering::Release);
+ assert!(store.apply_merge(&input_ids, output).is_err());
+
+ let after_ids: std::collections::HashSet<_> = store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|meta| meta.id.clone())
+ .collect();
+ assert_eq!(after_ids, before_ids);
+ drop(before);
+ drop(store);
+
+ let reopened = open_test_store(dir.path());
+ let reopened_ids: std::collections::HashSet<_> = reopened
+ .snapshot()
+ .segments
+ .iter()
+ .map(|meta| meta.id.clone())
+ .collect();
+ assert_eq!(reopened_ids, before_ids);
+ assert_eq!(reopened.version_map.live_count(), 2);
+ }
+
+ #[test]
+ fn apply_merge_reports_indeterminate_when_publication_and_rollback_saves_fail() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ store.index("a", serde_json::json!({"value": "a"})).unwrap();
+ store.flush().unwrap().unwrap();
+ store.index("b", serde_json::json!({"value": "b"})).unwrap();
+ store.flush().unwrap().unwrap();
+ let before = store.snapshot();
+ let mut output = before.segments[0].clone();
+ output.id = "indeterminate-output".into();
+ let input_ids: Vec<_> = before.segments.iter().map(|meta| meta.id.clone()).collect();
+ let transaction =
+ VersionRepointTransaction::with_capacity(Arc::clone(&store.version_map), 0);
+ // The pre-disarm input save succeeds; publication and rollback saves
+ // then both fail. The API must not claim exact rollback durability.
+ store.fail_snapshot_save_after.store(1, Ordering::Release);
+ store
+ .snapshot_save_failures_remaining
+ .store(2, Ordering::Release);
+ let error = store
+ .apply_merge_with_repoints(&input_ids, output, transaction)
+ .unwrap_err();
+ assert!(matches!(error, MergePublicationError::Indeterminate { .. }));
+ }
}
diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rs
index 05a0511a5..90e0e81ee 100644
--- a/engine/crates/xerj-storage/src/version_map.rs
+++ b/engine/crates/xerj-storage/src/version_map.rs
@@ -119,6 +119,127 @@ pub struct VersionMap {
ghost_events: std::sync::atomic::AtomicU64,
}
+#[derive(Debug)]
+struct VersionRepointChange {
+ doc_id: String,
+ previous: Option<VersionEntry>,
+ installed: VersionEntry,
+}
+
+/// Panic- and cancellation-safe group of physical version-map repoints.
+///
+/// Merge output identities are repointed before snapshot publication. Unless
+/// publication commits this transaction, drop restores only entries that
+/// still exactly match values installed by the transaction. A concurrent
+/// newer write or delete therefore always wins over rollback.
+pub struct VersionRepointTransaction {
+ map: Arc<VersionMap>,
+ changes: Vec<VersionRepointChange>,
+ committed: bool,
+}
+
+impl VersionRepointTransaction {
+ /// Reserve the complete rollback log before mutating the map.
+ pub fn with_capacity(map: Arc<VersionMap>, capacity: usize) -> Self {
+ Self {
+ map,
+ changes: Vec::with_capacity(capacity),
+ committed: false,
+ }
+ }
+
+ /// Conditionally repoint an entry and remember the exact displaced value.
+ pub fn repoint_if_latest(
+ &mut self,
+ doc_id: &str,
+ seq_no: SeqNo,
+ segment_id: Arc<str>,
+ deleted: bool,
+ ) {
+ debug_assert!(
+ self.changes.len() < self.changes.capacity(),
+ "merge rollback log must be reserved before repoint"
+ );
+ use dashmap::mapref::entry::Entry;
+ let (previous, installed) = match self.map.inner.entry(doc_id.to_owned()) {
+ Entry::Occupied(mut occupied) => {
+ if seq_no < occupied.get().seq_no {
+ return;
+ }
+ let previous = occupied.get().clone();
+ let installed = VersionEntry {
+ seq_no,
+ segment_id,
+ deleted,
+ version: previous.version,
+ };
+ occupied.insert(installed.clone());
+ if previous.seq_no != seq_no || (deleted && !previous.deleted) {
+ self.map.ghost_events.fetch_add(1, Ordering::Relaxed);
+ }
+ self.map.live_delta(!previous.deleted, !deleted);
+ (Some(previous), installed)
+ }
+ Entry::Vacant(vacant) => {
+ let installed = VersionEntry {
+ seq_no,
+ segment_id,
+ deleted,
+ version: 1,
+ };
+ vacant.insert(installed.clone());
+ if deleted {
+ self.map.ghost_events.fetch_add(1, Ordering::Relaxed);
+ }
+ self.map.live_delta(false, !deleted);
+ (None, installed)
+ }
+ };
+ self.changes.push(VersionRepointChange {
+ doc_id: doc_id.to_owned(),
+ previous,
+ installed,
+ });
+ }
+
+ /// Keep all repoints after the snapshot publication succeeds.
+ pub fn commit(mut self) {
+ self.committed = true;
+ }
+
+ fn rollback(&mut self) {
+ use dashmap::mapref::entry::Entry;
+ for change in self.changes.drain(..).rev() {
+ let Entry::Occupied(mut occupied) = self.map.inner.entry(change.doc_id) else {
+ continue;
+ };
+ if occupied.get() != &change.installed {
+ continue;
+ }
+ let current_live = !occupied.get().deleted;
+ match change.previous {
+ Some(previous) => {
+ let previous_live = !previous.deleted;
+ occupied.insert(previous);
+ self.map.live_delta(current_live, previous_live);
+ }
+ None => {
+ occupied.remove();
+ self.map.live_delta(current_live, false);
+ }
+ }
+ }
+ }
+}
+
+impl Drop for VersionRepointTransaction {
+ fn drop(&mut self) {
+ if !self.committed {
+ self.rollback();
+ }
+ }
+}
+
impl VersionMap {
/// Create an empty version map.
pub fn new() -> Self {
@@ -655,6 +776,71 @@ mod tests {
assert_eq!(vm.live_count(), 1); // "x" live, "d" tombstoned
}
+ #[test]
+ fn merge_repoint_transaction_rolls_back_exact_previous_entries() {
+ let vm = Arc::new(VersionMap::new());
+ vm.set("live", 7, "input-a", false);
+ vm.set("tomb", 8, "input-b", true);
+ let live_before = vm.live_count();
+ {
+ let mut transaction = VersionRepointTransaction::with_capacity(Arc::clone(&vm), 3);
+ transaction.repoint_if_latest("live", 7, Arc::from("output"), false);
+ transaction.repoint_if_latest("tomb", 8, Arc::from("output"), true);
+ transaction.repoint_if_latest("new-tomb", 9, Arc::from("output"), true);
+ assert_eq!(&*vm.get("live").unwrap().segment_id, "output");
+ assert_eq!(&*vm.get("tomb").unwrap().segment_id, "output");
+ }
+ assert_eq!(&*vm.get("live").unwrap().segment_id, "input-a");
+ assert_eq!(&*vm.get("tomb").unwrap().segment_id, "input-b");
+ assert!(vm.get("new-tomb").is_none());
+ assert_eq!(vm.live_count(), live_before);
+ }
+
+ #[test]
+ fn merge_repoint_rollback_preserves_concurrent_update_and_delete() {
+ let vm = Arc::new(VersionMap::new());
+ vm.set("updated", 1, "input", false);
+ vm.set("deleted", 2, "input", false);
+ {
+ let mut transaction = VersionRepointTransaction::with_capacity(Arc::clone(&vm), 2);
+ transaction.repoint_if_latest("updated", 1, Arc::from("output"), false);
+ transaction.repoint_if_latest("deleted", 2, Arc::from("output"), false);
+ vm.set("updated", 10, IN_MEMORY_SEGMENT_ID, false);
+ vm.delete("deleted", 11, IN_MEMORY_SEGMENT_ID).unwrap();
+ }
+ let updated = vm.get("updated").unwrap();
+ assert_eq!(
+ (updated.seq_no, &*updated.segment_id, updated.deleted),
+ (10, IN_MEMORY_SEGMENT_ID, false)
+ );
+ let deleted = vm.get("deleted").unwrap();
+ assert_eq!(
+ (deleted.seq_no, &*deleted.segment_id, deleted.deleted),
+ (11, IN_MEMORY_SEGMENT_ID, true)
+ );
+ }
+
+ #[test]
+ fn merge_repoint_transaction_is_panic_safe_and_commit_is_final() {
+ let vm = Arc::new(VersionMap::new());
+ vm.set("doc", 1, "input", false);
+ let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe({
+ let vm = Arc::clone(&vm);
+ move || {
+ let mut transaction = VersionRepointTransaction::with_capacity(Arc::clone(&vm), 1);
+ transaction.repoint_if_latest("doc", 1, Arc::from("failed-output"), false);
+ panic!("injected merge worker panic after repoint");
+ }
+ }));
+ assert!(panic_result.is_err());
+ assert_eq!(&*vm.get("doc").unwrap().segment_id, "input");
+
+ let mut transaction = VersionRepointTransaction::with_capacity(Arc::clone(&vm), 1);
+ transaction.repoint_if_latest("doc", 1, Arc::from("committed-output"), false);
+ transaction.commit();
+ assert_eq!(&*vm.get("doc").unwrap().segment_id, "committed-output");
+ }
+
#[test]
fn check_and_set_success() {
let vm = VersionMap::new();