AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(cluster): drain and handle inbound messages in ClusterRunner run loop
xerj-orgd5ba9e67ff21
diff --git a/engine/crates/xerj-cluster/src/node.rs b/engine/crates/xerj-cluster/src/node.rsindex 6f66605b1..f3d3b76fe 100644--- a/engine/crates/xerj-cluster/src/node.rs+++ b/engine/crates/xerj-cluster/src/node.rs@@ -82,34 +82,48 @@ impl ClusterNode {self.raft.propose(cmd)}- /// Run the node event loop until `shutdown` fires.+ /// Perform one iteration of the node event loop:+ /// 1. Wait for an inbound message, bounded by `tick_interval`.+ /// 2. If one arrives, deliver it to the Raft state machine.+ /// 3. Tick the state machine (advancing elections / heartbeats).///- /// Continuously:- /// 1. Waits for an inbound message (with a timeout equal to the tick interval).- /// 2. Delivers the message to the Raft state machine.- /// 3. Ticks the state machine regardless.+ /// This is the unit of work that [`crate::runner::ClusterRunner`] drives,+ /// so the caller can interleave it with a shutdown signal via+ /// `tokio::select!`. It is cancellation-safe: dropping the returned future+ /// before it resolves only abandons an in-flight `recv`/`send`, which Raft+ /// tolerates via retransmission.+ pub async fn step(&mut self, tick_interval: Duration) -> Result<()> {+ // Try to receive a message within the tick interval.+ let recv_result = tokio::time::timeout(tick_interval, self.transport.recv()).await;++ match recv_result {+ Ok(Ok((from, msg))) => {+ self.handle_message(&from, msg).await?;+ }+ Ok(Err(e)) => {+ warn!(node = %self.raft.id, error = %e, "Transport recv error");+ }+ Err(_timeout) => {+ // Timeout is normal — just tick.+ }+ }++ self.tick().await?;+ Ok(())+ }++ /// Run the node event loop forever (until an error is hit).+ ///+ /// Each iteration is a single [`ClusterNode::step`]: waits for an inbound+ /// message (with a timeout equal to the tick interval), delivers it to the+ /// Raft state machine, then ticks.////// In production you would wrap this with a `tokio::select!` that listens- /// for a shutdown signal and for the transport channel.+ /// for a shutdown signal — see [`crate::runner::ClusterRunner::run`].pub async fn run(&mut self, tick_interval: Duration) -> Result<()> {info!(node = %self.raft.id, "Starting cluster node run loop");loop {- // Try to receive a message within the tick interval- let recv_result = tokio::time::timeout(tick_interval, self.transport.recv()).await;-- match recv_result {- Ok(Ok((from, msg))) => {- self.handle_message(&from, msg).await?;- }- Ok(Err(e)) => {- warn!(node = %self.raft.id, error = %e, "Transport recv error");- }- Err(_timeout) => {- // Timeout is normal — just tick- }- }-- self.tick().await?;+ self.step(tick_interval).await?;}}@@ -329,4 +343,54 @@ mod tests {assert!(nodes[0].metadata.nodes.contains_key("n2"));assert_eq!(nodes[0].metadata.config["replicas"], "2");}++ /// End-to-end proof that [`crate::runner::ClusterRunner::run`] actually+ /// drains and handles inbound messages (not just sleeps and ticks).+ ///+ /// Three real [`ClusterNode`]s are wrapped in `ClusterRunner`s over a shared+ /// [`InMemoryBus`] and driven by their `run()` loops concurrently. A leader+ /// can only emerge if RequestVote / vote-response messages are received and+ /// handled during the run, so asserting exactly one leader within ~2s proves+ /// the run loop processes messages. With the old recv-less `sleep + tick`+ /// body, votes were sent but never received, so zero leaders emerged and+ /// this assertion fails.+ #[tokio::test]+ async fn test_cluster_runner_elects_single_leader() {+ use crate::runner::ClusterRunner;+ use tokio::sync::watch;++ let nodes = make_cluster(&["n1", "n2", "n3"]).await;++ // Keep the sender alive for the whole test: dropping it would resolve+ // `shutdown.changed()` with an error and stop the run loops early.+ let (_shutdown_tx, shutdown_rx) = watch::channel(false);++ let tick = Duration::from_millis(5);+ let mut runners = nodes+ .into_iter()+ .map(|n| ClusterRunner::new(n, tick, shutdown_rx.clone()));+ let mut r1 = runners.next().unwrap();+ let mut r2 = runners.next().unwrap();+ let mut r3 = runners.next().unwrap();++ // run() loops forever, so the 2s timeout branch is what ends the+ // select!. Because select! borrows (does not move) the runners, once+ // the timeout fires and the run() futures are cancelled we regain+ // access to inspect leadership.+ tokio::select! {+ _ = r1.run() => {}+ _ = r2.run() => {}+ _ = r3.run() => {}+ _ = tokio::time::sleep(Duration::from_secs(2)) => {}+ }++ let leaders = [r1.is_leader(), r2.is_leader(), r3.is_leader()]+ .into_iter()+ .filter(|&l| l)+ .count();+ assert_eq!(+ leaders, 1,+ "exactly one leader must be elected while ClusterRunner::run() drives the cluster"+ );+ }}diff --git a/engine/crates/xerj-cluster/src/runner.rs b/engine/crates/xerj-cluster/src/runner.rsindex 78c484126..b30a93c78 100644--- a/engine/crates/xerj-cluster/src/runner.rs+++ b/engine/crates/xerj-cluster/src/runner.rs@@ -1,13 +1,15 @@-//! Cluster runner — drives the Raft tick loop and message passing.+//! Cluster runner — drives the Raft consensus loop and message passing.//!-//! [`ClusterRunner`] owns a [`ClusterNode`] and two background tasks:-//! 1. A periodic **tick** (every 50 ms by default) that advances the Raft-//! state machine and dispatches heartbeats / log replication RPCs.-//! 2. A **receive loop** that delivers inbound messages from peers and routes-//! the responses.+//! [`ClusterRunner`] owns a [`ClusterNode`] and runs a single async loop that,+//! each iteration, drives one [`ClusterNode::step`]: it drains an inbound+//! message from a peer (bounded by the tick interval), delivers it to the Raft+//! state machine, and then ticks — advancing elections and dispatching+//! heartbeat / log-replication RPCs.//!//! Shutdown is controlled via a `tokio::sync::watch` channel. When the watch-//! value becomes `true`, the run loop exits cleanly.+//! value becomes `true` the run loop exits cleanly; the shutdown wait is raced+//! against `step` via `tokio::select!` so a shutdown is observed promptly+//! rather than only between ticks.use std::time::Duration;@@ -76,19 +78,16 @@ impl ClusterRunner {/// Run the Raft event loop until the shutdown signal fires.///- /// The loop alternates between:- /// - Ticking the Raft state machine at `tick_interval`.- /// - Receiving inbound messages (with a timeout equal to the tick interval)- /// via `ClusterNode::run`'s built-in timeout-based recv pattern.- /// - Checking the shutdown watch.+ /// Each iteration races two futures with `tokio::select!`:+ /// - [`ClusterNode::step`] — drain one inbound peer message (bounded by+ /// `tick_interval`), deliver it to the Raft state machine, then tick.+ /// - `shutdown.changed()` — resolves when the shutdown watch is set (or the+ /// sender is dropped), at which point the loop exits.///- /// Because `ClusterNode` owns the transport as a `Box<dyn ClusterTransport>`- /// and does not expose `recv()` directly to callers, we drive the node- /// forward by:- /// 1. Sleeping one tick.- /// 2. During that sleep, letting the node drain inbound messages via its- /// existing `run`-style timeout recv.- /// 3. Calling `node.tick()` after each sleep.+ /// This actually *handles* inbound messages, so leader election and log+ /// replication make progress. (The previous implementation only slept and+ /// ticked, never calling `recv()`, so a multi-node cluster could never+ /// elect a leader.)pub async fn run(&mut self) {info!(node = %self.node.raft.id,@@ -96,32 +95,32 @@ impl ClusterRunner {"ClusterRunner starting");+ let tick = self.tick_interval;loop {- // Check shutdown before each tick iteration.+ // Fast path: already asked to shut down before we start a step.if *self.shutdown.borrow() {info!(node = %self.node.raft.id, "ClusterRunner received shutdown signal");break;}- // Try to receive a message within the tick interval.- // This uses the same timeout pattern as ClusterNode::run.- // Sleep for one tick interval. During this window the Tokio- // runtime services other tasks (including any transport tasks).- tokio::time::sleep(self.tick_interval).await;-- // Tick the Raft state machine.- if let Err(e) = self.node.tick().await {- warn!(- node = %self.node.raft.id,- error = %e,- "Raft tick error"- );- }-- // Poll shutdown watch (non-blocking).- if self.shutdown.has_changed().unwrap_or(false) && *self.shutdown.borrow() {- info!(node = %self.node.raft.id, "ClusterRunner received shutdown signal");- break;+ tokio::select! {+ // Shutdown requested (value changed) or sender dropped (Err).+ changed = self.shutdown.changed() => {+ if changed.is_err() || *self.shutdown.borrow() {+ info!(node = %self.node.raft.id, "ClusterRunner received shutdown signal");+ break;+ }+ }+ // Drive one iteration of the Raft loop: recv → handle → tick.+ res = self.node.step(tick) => {+ if let Err(e) = res {+ warn!(+ node = %self.node.raft.id,+ error = %e,+ "Raft step error"+ );+ }+ }}}