AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): isolate one-shot test failpoints by thread
buger07cc2870edf2
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 31ddd21c7..b2d59565f 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -248,15 +248,16 @@ jobs:## Scoped to the one crate whose lib tests have been *measured* green at# default parallelism (10 dev-profile and 5 ci-test-profile clean runs at- # this commit, 20 cores). `--workspace --lib` is the destination, not this- # commit: at default parallelism it is currently red in xerj-autoindex- # (5 failures in 10 runs), where the `incremental_reconcile_http_tests`- # module reaches the global `SNAPSHOT_FAILPOINT` without holding the- # `SNAPSHOT_TEST_LOCK` that guards it and steals another test's injected- # failure -- the same defect class as #372, in a different crate. That is- # issue #385; widening this step past xerj-engine is #384. A gate is- # worth nothing if it is merged red, so it goes in at the width it- # passes at.+ # this commit, 20 cores). `--workspace --lib` remains the destination,+ # but the broader gate is tracked separately in #384.+ #+ # Historical context for this scope: before #385, xerj-autoindex was red+ # at default parallelism because `incremental_reconcile_http_tests` could+ # reach the global `SNAPSHOT_FAILPOINT` without holding the+ # `SNAPSHOT_TEST_LOCK` and steal another test's injected failure. The+ # thread-local failpoint fix in this commit removes that defect; widening+ # this step past xerj-engine remains #384. A gate is worth nothing if it+ # is merged red, so it goes in at the width it passes at.## Reuses the binaries the step above already built: the added cost is test# runtime, not another compile.diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ebdcfadfb..4767b5939 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -94,6 +94,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- **Autoindex snapshot and replay one-shot failpoints can no longer be stolen by unrelated parallel tests.** ([#385](https://github.com/xerj-org/xerj/issues/385)).+ The two test-only failpoints are now owned by the thread that arms them, while+ retaining their existing one-shot behavior and serialization locks.+- **Console delete/upsert no longer reports success after a refused write.**`DELETE /_xerj-console/api/v1/views/{id}` swallowed `delete_document`errors and returned `204` while the view stayed on disk (the same classdiff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex e7080d826..48316b171 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -21,16 +21,18 @@ use std::path::Path;const SNAPSHOT_VERSION: u32 = 2;#[cfg(test)]-static SNAPSHOT_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);+thread_local! {+ // Test failpoints belong to the thread that arms them: an unrelated test+ // running in parallel must not consume another test's one-shot failure.+ static SNAPSHOT_FAILPOINT: std::cell::Cell<u8> = const { std::cell::Cell::new(0) };+ static REPLAY_FAIL_AFTER_APPLY: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };+}#[cfg(test)]static SNAPSHOT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());#[cfg(test)]static POST_SEAL_SOURCE_REPLACEMENT: std::sync::Mutex<Option<(std::path::PathBuf, Vec<u8>)>> =std::sync::Mutex::new(None);#[cfg(test)]-static REPLAY_FAIL_AFTER_APPLY: std::sync::atomic::AtomicBool =- std::sync::atomic::AtomicBool::new(false);-#[cfg(test)]pub(crate) static REPLAY_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());#[cfg(test)]static GC_FAIL_AFTER_RENAME: std::sync::atomic::AtomicBool =@@ -1744,18 +1746,20 @@ fn apply_post_seal_source_replacement(path: &Path) -> Result<()> {#[cfg(test)]fn fail_next_snapshot(boundary: u8) {- SNAPSHOT_FAILPOINT.store(boundary, std::sync::atomic::Ordering::SeqCst);+ SNAPSHOT_FAILPOINT.with(|failpoint| failpoint.set(boundary));}#[cfg(test)]fn snapshot_failpoint(boundary: u8) -> Result<()> {- if SNAPSHOT_FAILPOINT.compare_exchange(- boundary,- 0,- std::sync::atomic::Ordering::SeqCst,- std::sync::atomic::Ordering::SeqCst,- ) == Ok(boundary)- {+ let armed = SNAPSHOT_FAILPOINT.with(|failpoint| {+ if failpoint.get() == boundary {+ failpoint.set(0);+ true+ } else {+ false+ }+ });+ if armed {anyhow::bail!("injected snapshot failure at boundary {boundary}");}Ok(())@@ -1768,12 +1772,12 @@ fn snapshot_failpoint(_boundary: u8) -> Result<()> {#[cfg(test)]fn fail_replay_after_next_apply() {- REPLAY_FAIL_AFTER_APPLY.store(true, std::sync::atomic::Ordering::SeqCst);+ REPLAY_FAIL_AFTER_APPLY.with(|failpoint| failpoint.set(true));}#[cfg(test)]fn replay_fail_after_apply() -> Result<()> {- if REPLAY_FAIL_AFTER_APPLY.swap(false, std::sync::atomic::Ordering::SeqCst) {+ if REPLAY_FAIL_AFTER_APPLY.with(|failpoint| failpoint.replace(false)) {anyhow::bail!("injected crash after accepted operation");}Ok(())@@ -2484,6 +2488,80 @@ mod tests {}}+ #[test]+ fn snapshot_failpoint_cannot_be_stolen_by_an_unrelated_thread() {+ let _guard = SNAPSHOT_TEST_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ let (armed_tx, armed_rx) = std::sync::mpsc::channel();+ let (thief_done_tx, thief_done_rx) = std::sync::mpsc::channel();++ let owner = std::thread::spawn(move || {+ fail_next_snapshot(2);+ armed_tx.send(()).unwrap();+ thief_done_rx.recv().unwrap();+ (+ snapshot_failpoint(2).is_err(),+ snapshot_failpoint(2).is_ok(),+ )+ });++ armed_rx.recv().unwrap();+ let thief_result = snapshot_failpoint(2);+ thief_done_tx.send(()).unwrap();+ let (owner_consumed_once, owner_second_probe_succeeded) = owner.join().unwrap();++ assert!(+ thief_result.is_ok(),+ "a thread that did not arm the snapshot failpoint consumed it"+ );+ assert!(+ owner_consumed_once,+ "the thread that armed the snapshot failpoint did not consume it"+ );+ assert!(+ owner_second_probe_succeeded,+ "the snapshot failpoint must retain one-shot semantics"+ );+ }++ #[test]+ fn replay_failpoint_cannot_be_stolen_by_an_unrelated_thread() {+ let _guard = REPLAY_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ let (armed_tx, armed_rx) = std::sync::mpsc::channel();+ let (thief_done_tx, thief_done_rx) = std::sync::mpsc::channel();++ let owner = std::thread::spawn(move || {+ fail_replay_after_next_apply();+ armed_tx.send(()).unwrap();+ thief_done_rx.recv().unwrap();+ (+ replay_fail_after_apply().is_err(),+ replay_fail_after_apply().is_ok(),+ )+ });++ armed_rx.recv().unwrap();+ let thief_result = replay_fail_after_apply();+ thief_done_tx.send(()).unwrap();+ let (owner_consumed_once, owner_second_probe_succeeded) = owner.join().unwrap();++ assert!(+ thief_result.is_ok(),+ "a thread that did not arm the replay failpoint consumed it"+ );+ assert!(+ owner_consumed_once,+ "the thread that armed the replay failpoint did not consume it"+ );+ assert!(+ owner_second_probe_succeeded,+ "the replay failpoint must retain one-shot semantics"+ );+ }+#[test]fn pending_generation_is_bound_before_mutable_source_replanning() {let _guard = SNAPSHOT_TEST_LOCK