Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): isolate one-shot test failpoints by thread

buger07cc2870edf2
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 31ddd21c7..b2d59565f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -248,15 +248,16 @@ jobs:
#
# Scoped to the one crate whose lib tests have been *measured* green at
# default parallelism (10 dev-profile and 5 ci-test-profile clean runs at
- # this commit, 20 cores). `--workspace --lib` is the destination, not this
- # commit: at default parallelism it is currently red in xerj-autoindex
- # (5 failures in 10 runs), where the `incremental_reconcile_http_tests`
- # module reaches the global `SNAPSHOT_FAILPOINT` without holding the
- # `SNAPSHOT_TEST_LOCK` that guards it and steals another test's injected
- # failure -- the same defect class as #372, in a different crate. That is
- # issue #385; widening this step past xerj-engine is #384. A gate is
- # worth nothing if it is merged red, so it goes in at the width it
- # passes at.
+ # this commit, 20 cores). `--workspace --lib` remains the destination,
+ # but the broader gate is tracked separately in #384.
+ #
+ # Historical context for this scope: before #385, xerj-autoindex was red
+ # at default parallelism because `incremental_reconcile_http_tests` could
+ # reach the global `SNAPSHOT_FAILPOINT` without holding the
+ # `SNAPSHOT_TEST_LOCK` and steal another test's injected failure. The
+ # thread-local failpoint fix in this commit removes that defect; widening
+ # this step past xerj-engine remains #384. A gate is worth nothing if it
+ # is merged red, so it goes in at the width it passes at.
#
# Reuses the binaries the step above already built: the added cost is test
# runtime, not another compile.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ebdcfadfb..4767b5939 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -94,6 +94,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- **Autoindex snapshot and replay one-shot failpoints can no longer be stolen by unrelated parallel tests.** ([#385](https://github.com/xerj-org/xerj/issues/385)).
+ The two test-only failpoints are now owned by the thread that arms them, while
+ retaining their existing one-shot behavior and serialization locks.
+
- **Console delete/upsert no longer reports success after a refused write.**
`DELETE /_xerj-console/api/v1/views/{id}` swallowed `delete_document`
errors and returned `204` while the view stayed on disk (the same class
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index e7080d826..48316b171 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -21,16 +21,18 @@ use std::path::Path;
const SNAPSHOT_VERSION: u32 = 2;
#[cfg(test)]
-static SNAPSHOT_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
+thread_local! {
+ // Test failpoints belong to the thread that arms them: an unrelated test
+ // running in parallel must not consume another test's one-shot failure.
+ static SNAPSHOT_FAILPOINT: std::cell::Cell<u8> = const { std::cell::Cell::new(0) };
+ static REPLAY_FAIL_AFTER_APPLY: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
+}
#[cfg(test)]
static SNAPSHOT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
#[cfg(test)]
static POST_SEAL_SOURCE_REPLACEMENT: std::sync::Mutex<Option<(std::path::PathBuf, Vec<u8>)>> =
std::sync::Mutex::new(None);
#[cfg(test)]
-static REPLAY_FAIL_AFTER_APPLY: std::sync::atomic::AtomicBool =
- std::sync::atomic::AtomicBool::new(false);
-#[cfg(test)]
pub(crate) static REPLAY_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
#[cfg(test)]
static GC_FAIL_AFTER_RENAME: std::sync::atomic::AtomicBool =
@@ -1744,18 +1746,20 @@ fn apply_post_seal_source_replacement(path: &Path) -> Result<()> {
#[cfg(test)]
fn fail_next_snapshot(boundary: u8) {
- SNAPSHOT_FAILPOINT.store(boundary, std::sync::atomic::Ordering::SeqCst);
+ SNAPSHOT_FAILPOINT.with(|failpoint| failpoint.set(boundary));
}
#[cfg(test)]
fn snapshot_failpoint(boundary: u8) -> Result<()> {
- if SNAPSHOT_FAILPOINT.compare_exchange(
- boundary,
- 0,
- std::sync::atomic::Ordering::SeqCst,
- std::sync::atomic::Ordering::SeqCst,
- ) == Ok(boundary)
- {
+ let armed = SNAPSHOT_FAILPOINT.with(|failpoint| {
+ if failpoint.get() == boundary {
+ failpoint.set(0);
+ true
+ } else {
+ false
+ }
+ });
+ if armed {
anyhow::bail!("injected snapshot failure at boundary {boundary}");
}
Ok(())
@@ -1768,12 +1772,12 @@ fn snapshot_failpoint(_boundary: u8) -> Result<()> {
#[cfg(test)]
fn fail_replay_after_next_apply() {
- REPLAY_FAIL_AFTER_APPLY.store(true, std::sync::atomic::Ordering::SeqCst);
+ REPLAY_FAIL_AFTER_APPLY.with(|failpoint| failpoint.set(true));
}
#[cfg(test)]
fn replay_fail_after_apply() -> Result<()> {
- if REPLAY_FAIL_AFTER_APPLY.swap(false, std::sync::atomic::Ordering::SeqCst) {
+ if REPLAY_FAIL_AFTER_APPLY.with(|failpoint| failpoint.replace(false)) {
anyhow::bail!("injected crash after accepted operation");
}
Ok(())
@@ -2484,6 +2488,80 @@ mod tests {
}
}
+ #[test]
+ fn snapshot_failpoint_cannot_be_stolen_by_an_unrelated_thread() {
+ let _guard = SNAPSHOT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let (armed_tx, armed_rx) = std::sync::mpsc::channel();
+ let (thief_done_tx, thief_done_rx) = std::sync::mpsc::channel();
+
+ let owner = std::thread::spawn(move || {
+ fail_next_snapshot(2);
+ armed_tx.send(()).unwrap();
+ thief_done_rx.recv().unwrap();
+ (
+ snapshot_failpoint(2).is_err(),
+ snapshot_failpoint(2).is_ok(),
+ )
+ });
+
+ armed_rx.recv().unwrap();
+ let thief_result = snapshot_failpoint(2);
+ thief_done_tx.send(()).unwrap();
+ let (owner_consumed_once, owner_second_probe_succeeded) = owner.join().unwrap();
+
+ assert!(
+ thief_result.is_ok(),
+ "a thread that did not arm the snapshot failpoint consumed it"
+ );
+ assert!(
+ owner_consumed_once,
+ "the thread that armed the snapshot failpoint did not consume it"
+ );
+ assert!(
+ owner_second_probe_succeeded,
+ "the snapshot failpoint must retain one-shot semantics"
+ );
+ }
+
+ #[test]
+ fn replay_failpoint_cannot_be_stolen_by_an_unrelated_thread() {
+ let _guard = REPLAY_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let (armed_tx, armed_rx) = std::sync::mpsc::channel();
+ let (thief_done_tx, thief_done_rx) = std::sync::mpsc::channel();
+
+ let owner = std::thread::spawn(move || {
+ fail_replay_after_next_apply();
+ armed_tx.send(()).unwrap();
+ thief_done_rx.recv().unwrap();
+ (
+ replay_fail_after_apply().is_err(),
+ replay_fail_after_apply().is_ok(),
+ )
+ });
+
+ armed_rx.recv().unwrap();
+ let thief_result = replay_fail_after_apply();
+ thief_done_tx.send(()).unwrap();
+ let (owner_consumed_once, owner_second_probe_succeeded) = owner.join().unwrap();
+
+ assert!(
+ thief_result.is_ok(),
+ "a thread that did not arm the replay failpoint consumed it"
+ );
+ assert!(
+ owner_consumed_once,
+ "the thread that armed the replay failpoint did not consume it"
+ );
+ assert!(
+ owner_second_probe_succeeded,
+ "the replay failpoint must retain one-shot semantics"
+ );
+ }
+
#[test]
fn pending_generation_is_bound_before_mutable_source_replanning() {
let _guard = SNAPSHOT_TEST_LOCK