Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): exclusive node.lock on the data dir — second process fails fast (RC4 blocker 13)

xerj-orge788e148ee3c
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index ad2da0d88..46a285761 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -256,6 +256,18 @@ pub struct Engine {
/// `ShardRouter::update_from_metadata` whenever the Raft log
/// commits a new shard assignment.
pub shard_router: Arc<parking_lot::RwLock<xerj_cluster::router::ShardRouter>>,
+
+ /// Exclusive advisory lock on `<data_dir>/node.lock`, held for the
+ /// engine's whole lifetime (RC4 blocker 13). Acquired in
+ /// [`Engine::new`] BEFORE any index is opened (i.e. before any WAL
+ /// replay or segment flush can touch the directory), so a second
+ /// xerj process pointed at a live data dir fails fast instead of
+ /// replaying the WAL and flushing duplicate segments into it — the
+ /// classic systemd double-start corruption. The lock is an OS-level
+ /// `flock`-style lock (`std::fs::File::try_lock`), so it dies with
+ /// the process: a `kill -9` releases it automatically and a stale
+ /// `node.lock` file never blocks the next boot.
+ _node_lock: Arc<std::fs::File>,
}
impl Engine {
@@ -264,6 +276,12 @@ impl Engine {
let data_dir = PathBuf::from(&config.server.data_dir);
std::fs::create_dir_all(&data_dir)?;
+ // Data-dir exclusivity (RC4 blocker 13): take the node lock BEFORE
+ // scanning/opening any index below — Index::open replays the WAL
+ // and can flush segments, which must never happen while another
+ // process serves the same directory.
+ let node_lock = Arc::new(Self::acquire_node_lock(&data_dir)?);
+
// Apply operator-tunable aggregation bucket cap. Stored in a static
// AtomicUsize inside aggs.rs so all per-bucket-allocator hot loops
// can read it with no plumbing through every agg signature.
@@ -316,6 +334,7 @@ impl Engine {
shard_router: Arc::new(parking_lot::RwLock::new(
xerj_cluster::router::ShardRouter::new(1),
)),
+ _node_lock: node_lock,
};
// Scan data_dir for existing index directories.
@@ -368,6 +387,60 @@ impl Engine {
Ok(engine)
}
+ /// Acquire the exclusive `<data_dir>/node.lock` advisory lock (RC4
+ /// blocker 13 — data-dir exclusivity).
+ ///
+ /// Uses `std::fs::File::try_lock` (flock-style, non-blocking): if
+ /// another process already holds the lock we fail fast with the
+ /// holder's pid instead of replaying its WAL and flushing duplicate
+ /// segments into a live directory. On success our own pid is written
+ /// into the file purely as a diagnostic for the *next* contender —
+ /// exclusivity comes from the OS lock, never from the pid content,
+ /// so a stale file left by `kill -9` (lock auto-released at process
+ /// death) can never wedge a reboot.
+ fn acquire_node_lock(data_dir: &std::path::Path) -> Result<std::fs::File> {
+ use std::io::Write;
+ let lock_path = data_dir.join("node.lock");
+ // Never O_TRUNC here: truncation must only happen AFTER the lock
+ // is ours, or a losing contender would erase the holder's pid.
+ let file = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(&lock_path)?;
+ match file.try_lock() {
+ Ok(()) => {
+ let _ = file.set_len(0);
+ let _ = writeln!(&file, "{}", std::process::id());
+ let _ = file.sync_all();
+ Ok(file)
+ }
+ Err(std::fs::TryLockError::WouldBlock) => {
+ let holder = std::fs::read_to_string(&lock_path)
+ .ok()
+ .map(|s| s.trim().to_string())
+ .filter(|s| !s.is_empty())
+ .unwrap_or_else(|| "unknown".to_string());
+ Err(EngineError::Common(xerj_common::XerjError::config(format!(
+ "data dir '{}' is already in use by another running xerj process \
+ (pid {holder}, lock file '{}') — refusing to start. Two processes \
+ serving one data dir would replay each other's WAL and corrupt \
+ segments; stop the other process or point this one at its own \
+ server.data_dir.",
+ data_dir.display(),
+ lock_path.display(),
+ ))))
+ }
+ Err(std::fs::TryLockError::Error(e)) => Err(EngineError::Common(
+ xerj_common::XerjError::config(format!(
+ "failed to acquire node lock '{}': {e}",
+ lock_path.display()
+ )),
+ )),
+ }
+ }
+
/// Drop PIT contexts whose `expires_at` is in the past. Cheap
/// O(N) walk; runs on the background sweeper task and is also
/// invoked opportunistically inside `open_pit` so a tight
diff --git a/engine/crates/xerj-engine/tests/node_lock.rs b/engine/crates/xerj-engine/tests/node_lock.rs
new file mode 100644
index 000000000..68741fabf
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/node_lock.rs
@@ -0,0 +1,50 @@
+//! RC4 blocker 13 regression: data-dir exclusivity via `<data_dir>/node.lock`.
+//!
+//! Before the fix a second xerj process pointed at a live data dir booted
+//! happily: it re-opened every index, replayed the WAL, and flushed
+//! duplicate segments into the directory the first process was still
+//! serving (live-reproduced: duplicate `min_seq=1/max_seq=1` segments from
+//! the second boot). The engine now takes an exclusive OS-level lock on
+//! `<data_dir>/node.lock` before any index open / WAL replay, and fails
+//! fast with the holder's pid.
+
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_engine::Engine;
+
+fn config_for(dir: &TempDir) -> Config {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ config
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn second_engine_on_same_data_dir_fails_fast_with_holder_pid() {
+ let dir = TempDir::new().unwrap();
+ let engine = Engine::new(config_for(&dir)).expect("first engine must boot");
+
+ // Second engine on the SAME live dir must refuse to start…
+ let err = match Engine::new(config_for(&dir)) {
+ Ok(_) => panic!("second engine on a live data dir must fail fast"),
+ Err(e) => e.to_string(),
+ };
+ // …with a self-explanatory message carrying the holder's pid.
+ assert!(
+ err.contains("already in use"),
+ "unexpected lock error: {err}"
+ );
+ assert!(
+ err.contains(&std::process::id().to_string()),
+ "lock error should name the holding pid: {err}"
+ );
+
+ // Dropping the holder releases the OS lock; the stale node.lock FILE
+ // left behind (as after kill -9) must never block the next boot.
+ drop(engine);
+ let reopened = Engine::new(config_for(&dir));
+ assert!(
+ reopened.is_ok(),
+ "reopen after the holder exits must succeed: {:?}",
+ reopened.err()
+ );
+}