AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): exclusive node.lock on the data dir — second process fails fast (RC4 blocker 13)
xerj-orge788e148ee3c
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex ad2da0d88..46a285761 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -256,6 +256,18 @@ pub struct Engine {/// `ShardRouter::update_from_metadata` whenever the Raft log/// commits a new shard assignment.pub shard_router: Arc<parking_lot::RwLock<xerj_cluster::router::ShardRouter>>,++ /// Exclusive advisory lock on `<data_dir>/node.lock`, held for the+ /// engine's whole lifetime (RC4 blocker 13). Acquired in+ /// [`Engine::new`] BEFORE any index is opened (i.e. before any WAL+ /// replay or segment flush can touch the directory), so a second+ /// xerj process pointed at a live data dir fails fast instead of+ /// replaying the WAL and flushing duplicate segments into it — the+ /// classic systemd double-start corruption. The lock is an OS-level+ /// `flock`-style lock (`std::fs::File::try_lock`), so it dies with+ /// the process: a `kill -9` releases it automatically and a stale+ /// `node.lock` file never blocks the next boot.+ _node_lock: Arc<std::fs::File>,}impl Engine {@@ -264,6 +276,12 @@ impl Engine {let data_dir = PathBuf::from(&config.server.data_dir);std::fs::create_dir_all(&data_dir)?;+ // Data-dir exclusivity (RC4 blocker 13): take the node lock BEFORE+ // scanning/opening any index below — Index::open replays the WAL+ // and can flush segments, which must never happen while another+ // process serves the same directory.+ let node_lock = Arc::new(Self::acquire_node_lock(&data_dir)?);+// Apply operator-tunable aggregation bucket cap. Stored in a static// AtomicUsize inside aggs.rs so all per-bucket-allocator hot loops// can read it with no plumbing through every agg signature.@@ -316,6 +334,7 @@ impl Engine {shard_router: Arc::new(parking_lot::RwLock::new(xerj_cluster::router::ShardRouter::new(1),)),+ _node_lock: node_lock,};// Scan data_dir for existing index directories.@@ -368,6 +387,60 @@ impl Engine {Ok(engine)}+ /// Acquire the exclusive `<data_dir>/node.lock` advisory lock (RC4+ /// blocker 13 — data-dir exclusivity).+ ///+ /// Uses `std::fs::File::try_lock` (flock-style, non-blocking): if+ /// another process already holds the lock we fail fast with the+ /// holder's pid instead of replaying its WAL and flushing duplicate+ /// segments into a live directory. On success our own pid is written+ /// into the file purely as a diagnostic for the *next* contender —+ /// exclusivity comes from the OS lock, never from the pid content,+ /// so a stale file left by `kill -9` (lock auto-released at process+ /// death) can never wedge a reboot.+ fn acquire_node_lock(data_dir: &std::path::Path) -> Result<std::fs::File> {+ use std::io::Write;+ let lock_path = data_dir.join("node.lock");+ // Never O_TRUNC here: truncation must only happen AFTER the lock+ // is ours, or a losing contender would erase the holder's pid.+ let file = std::fs::OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .truncate(false)+ .open(&lock_path)?;+ match file.try_lock() {+ Ok(()) => {+ let _ = file.set_len(0);+ let _ = writeln!(&file, "{}", std::process::id());+ let _ = file.sync_all();+ Ok(file)+ }+ Err(std::fs::TryLockError::WouldBlock) => {+ let holder = std::fs::read_to_string(&lock_path)+ .ok()+ .map(|s| s.trim().to_string())+ .filter(|s| !s.is_empty())+ .unwrap_or_else(|| "unknown".to_string());+ Err(EngineError::Common(xerj_common::XerjError::config(format!(+ "data dir '{}' is already in use by another running xerj process \+ (pid {holder}, lock file '{}') — refusing to start. Two processes \+ serving one data dir would replay each other's WAL and corrupt \+ segments; stop the other process or point this one at its own \+ server.data_dir.",+ data_dir.display(),+ lock_path.display(),+ ))))+ }+ Err(std::fs::TryLockError::Error(e)) => Err(EngineError::Common(+ xerj_common::XerjError::config(format!(+ "failed to acquire node lock '{}': {e}",+ lock_path.display()+ )),+ )),+ }+ }+/// Drop PIT contexts whose `expires_at` is in the past. Cheap/// O(N) walk; runs on the background sweeper task and is also/// invoked opportunistically inside `open_pit` so a tightdiff --git a/engine/crates/xerj-engine/tests/node_lock.rs b/engine/crates/xerj-engine/tests/node_lock.rsnew file mode 100644index 000000000..68741fabf--- /dev/null+++ b/engine/crates/xerj-engine/tests/node_lock.rs@@ -0,0 +1,50 @@+//! RC4 blocker 13 regression: data-dir exclusivity via `<data_dir>/node.lock`.+//!+//! Before the fix a second xerj process pointed at a live data dir booted+//! happily: it re-opened every index, replayed the WAL, and flushed+//! duplicate segments into the directory the first process was still+//! serving (live-reproduced: duplicate `min_seq=1/max_seq=1` segments from+//! the second boot). The engine now takes an exclusive OS-level lock on+//! `<data_dir>/node.lock` before any index open / WAL replay, and fails+//! fast with the holder's pid.++use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_engine::Engine;++fn config_for(dir: &TempDir) -> Config {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ config+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn second_engine_on_same_data_dir_fails_fast_with_holder_pid() {+ let dir = TempDir::new().unwrap();+ let engine = Engine::new(config_for(&dir)).expect("first engine must boot");++ // Second engine on the SAME live dir must refuse to start…+ let err = match Engine::new(config_for(&dir)) {+ Ok(_) => panic!("second engine on a live data dir must fail fast"),+ Err(e) => e.to_string(),+ };+ // …with a self-explanatory message carrying the holder's pid.+ assert!(+ err.contains("already in use"),+ "unexpected lock error: {err}"+ );+ assert!(+ err.contains(&std::process::id().to_string()),+ "lock error should name the holding pid: {err}"+ );++ // Dropping the holder releases the OS lock; the stale node.lock FILE+ // left behind (as after kill -9) must never block the next boot.+ drop(engine);+ let reopened = Engine::new(config_for(&dir));+ assert!(+ reopened.is_ok(),+ "reopen after the holder exits must succeed: {:?}",+ reopened.err()+ );+}