Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(cluster): drain and handle inbound messages in ClusterRunner run loop

xerj-orgd5ba9e67ff21
diff --git a/engine/crates/xerj-cluster/src/node.rs b/engine/crates/xerj-cluster/src/node.rs
index 6f66605b1..f3d3b76fe 100644
--- a/engine/crates/xerj-cluster/src/node.rs
+++ b/engine/crates/xerj-cluster/src/node.rs
@@ -82,34 +82,48 @@ impl ClusterNode {
self.raft.propose(cmd)
}
- /// Run the node event loop until `shutdown` fires.
+ /// Perform one iteration of the node event loop:
+ /// 1. Wait for an inbound message, bounded by `tick_interval`.
+ /// 2. If one arrives, deliver it to the Raft state machine.
+ /// 3. Tick the state machine (advancing elections / heartbeats).
///
- /// Continuously:
- /// 1. Waits for an inbound message (with a timeout equal to the tick interval).
- /// 2. Delivers the message to the Raft state machine.
- /// 3. Ticks the state machine regardless.
+ /// This is the unit of work that [`crate::runner::ClusterRunner`] drives,
+ /// so the caller can interleave it with a shutdown signal via
+ /// `tokio::select!`. It is cancellation-safe: dropping the returned future
+ /// before it resolves only abandons an in-flight `recv`/`send`, which Raft
+ /// tolerates via retransmission.
+ pub async fn step(&mut self, tick_interval: Duration) -> Result<()> {
+ // Try to receive a message within the tick interval.
+ let recv_result = tokio::time::timeout(tick_interval, self.transport.recv()).await;
+
+ match recv_result {
+ Ok(Ok((from, msg))) => {
+ self.handle_message(&from, msg).await?;
+ }
+ Ok(Err(e)) => {
+ warn!(node = %self.raft.id, error = %e, "Transport recv error");
+ }
+ Err(_timeout) => {
+ // Timeout is normal — just tick.
+ }
+ }
+
+ self.tick().await?;
+ Ok(())
+ }
+
+ /// Run the node event loop forever (until an error is hit).
+ ///
+ /// Each iteration is a single [`ClusterNode::step`]: waits for an inbound
+ /// message (with a timeout equal to the tick interval), delivers it to the
+ /// Raft state machine, then ticks.
///
/// In production you would wrap this with a `tokio::select!` that listens
- /// for a shutdown signal and for the transport channel.
+ /// for a shutdown signal — see [`crate::runner::ClusterRunner::run`].
pub async fn run(&mut self, tick_interval: Duration) -> Result<()> {
info!(node = %self.raft.id, "Starting cluster node run loop");
loop {
- // Try to receive a message within the tick interval
- let recv_result = tokio::time::timeout(tick_interval, self.transport.recv()).await;
-
- match recv_result {
- Ok(Ok((from, msg))) => {
- self.handle_message(&from, msg).await?;
- }
- Ok(Err(e)) => {
- warn!(node = %self.raft.id, error = %e, "Transport recv error");
- }
- Err(_timeout) => {
- // Timeout is normal — just tick
- }
- }
-
- self.tick().await?;
+ self.step(tick_interval).await?;
}
}
@@ -329,4 +343,54 @@ mod tests {
assert!(nodes[0].metadata.nodes.contains_key("n2"));
assert_eq!(nodes[0].metadata.config["replicas"], "2");
}
+
+ /// End-to-end proof that [`crate::runner::ClusterRunner::run`] actually
+ /// drains and handles inbound messages (not just sleeps and ticks).
+ ///
+ /// Three real [`ClusterNode`]s are wrapped in `ClusterRunner`s over a shared
+ /// [`InMemoryBus`] and driven by their `run()` loops concurrently. A leader
+ /// can only emerge if RequestVote / vote-response messages are received and
+ /// handled during the run, so asserting exactly one leader within ~2s proves
+ /// the run loop processes messages. With the old recv-less `sleep + tick`
+ /// body, votes were sent but never received, so zero leaders emerged and
+ /// this assertion fails.
+ #[tokio::test]
+ async fn test_cluster_runner_elects_single_leader() {
+ use crate::runner::ClusterRunner;
+ use tokio::sync::watch;
+
+ let nodes = make_cluster(&["n1", "n2", "n3"]).await;
+
+ // Keep the sender alive for the whole test: dropping it would resolve
+ // `shutdown.changed()` with an error and stop the run loops early.
+ let (_shutdown_tx, shutdown_rx) = watch::channel(false);
+
+ let tick = Duration::from_millis(5);
+ let mut runners = nodes
+ .into_iter()
+ .map(|n| ClusterRunner::new(n, tick, shutdown_rx.clone()));
+ let mut r1 = runners.next().unwrap();
+ let mut r2 = runners.next().unwrap();
+ let mut r3 = runners.next().unwrap();
+
+ // run() loops forever, so the 2s timeout branch is what ends the
+ // select!. Because select! borrows (does not move) the runners, once
+ // the timeout fires and the run() futures are cancelled we regain
+ // access to inspect leadership.
+ tokio::select! {
+ _ = r1.run() => {}
+ _ = r2.run() => {}
+ _ = r3.run() => {}
+ _ = tokio::time::sleep(Duration::from_secs(2)) => {}
+ }
+
+ let leaders = [r1.is_leader(), r2.is_leader(), r3.is_leader()]
+ .into_iter()
+ .filter(|&l| l)
+ .count();
+ assert_eq!(
+ leaders, 1,
+ "exactly one leader must be elected while ClusterRunner::run() drives the cluster"
+ );
+ }
}
diff --git a/engine/crates/xerj-cluster/src/runner.rs b/engine/crates/xerj-cluster/src/runner.rs
index 78c484126..b30a93c78 100644
--- a/engine/crates/xerj-cluster/src/runner.rs
+++ b/engine/crates/xerj-cluster/src/runner.rs
@@ -1,13 +1,15 @@
-//! Cluster runner — drives the Raft tick loop and message passing.
+//! Cluster runner — drives the Raft consensus loop and message passing.
//!
-//! [`ClusterRunner`] owns a [`ClusterNode`] and two background tasks:
-//! 1. A periodic **tick** (every 50 ms by default) that advances the Raft
-//! state machine and dispatches heartbeats / log replication RPCs.
-//! 2. A **receive loop** that delivers inbound messages from peers and routes
-//! the responses.
+//! [`ClusterRunner`] owns a [`ClusterNode`] and runs a single async loop that,
+//! each iteration, drives one [`ClusterNode::step`]: it drains an inbound
+//! message from a peer (bounded by the tick interval), delivers it to the Raft
+//! state machine, and then ticks — advancing elections and dispatching
+//! heartbeat / log-replication RPCs.
//!
//! Shutdown is controlled via a `tokio::sync::watch` channel. When the watch
-//! value becomes `true`, the run loop exits cleanly.
+//! value becomes `true` the run loop exits cleanly; the shutdown wait is raced
+//! against `step` via `tokio::select!` so a shutdown is observed promptly
+//! rather than only between ticks.
use std::time::Duration;
@@ -76,19 +78,16 @@ impl ClusterRunner {
/// Run the Raft event loop until the shutdown signal fires.
///
- /// The loop alternates between:
- /// - Ticking the Raft state machine at `tick_interval`.
- /// - Receiving inbound messages (with a timeout equal to the tick interval)
- /// via `ClusterNode::run`'s built-in timeout-based recv pattern.
- /// - Checking the shutdown watch.
+ /// Each iteration races two futures with `tokio::select!`:
+ /// - [`ClusterNode::step`] — drain one inbound peer message (bounded by
+ /// `tick_interval`), deliver it to the Raft state machine, then tick.
+ /// - `shutdown.changed()` — resolves when the shutdown watch is set (or the
+ /// sender is dropped), at which point the loop exits.
///
- /// Because `ClusterNode` owns the transport as a `Box<dyn ClusterTransport>`
- /// and does not expose `recv()` directly to callers, we drive the node
- /// forward by:
- /// 1. Sleeping one tick.
- /// 2. During that sleep, letting the node drain inbound messages via its
- /// existing `run`-style timeout recv.
- /// 3. Calling `node.tick()` after each sleep.
+ /// This actually *handles* inbound messages, so leader election and log
+ /// replication make progress. (The previous implementation only slept and
+ /// ticked, never calling `recv()`, so a multi-node cluster could never
+ /// elect a leader.)
pub async fn run(&mut self) {
info!(
node = %self.node.raft.id,
@@ -96,32 +95,32 @@ impl ClusterRunner {
"ClusterRunner starting"
);
+ let tick = self.tick_interval;
loop {
- // Check shutdown before each tick iteration.
+ // Fast path: already asked to shut down before we start a step.
if *self.shutdown.borrow() {
info!(node = %self.node.raft.id, "ClusterRunner received shutdown signal");
break;
}
- // Try to receive a message within the tick interval.
- // This uses the same timeout pattern as ClusterNode::run.
- // Sleep for one tick interval. During this window the Tokio
- // runtime services other tasks (including any transport tasks).
- tokio::time::sleep(self.tick_interval).await;
-
- // Tick the Raft state machine.
- if let Err(e) = self.node.tick().await {
- warn!(
- node = %self.node.raft.id,
- error = %e,
- "Raft tick error"
- );
- }
-
- // Poll shutdown watch (non-blocking).
- if self.shutdown.has_changed().unwrap_or(false) && *self.shutdown.borrow() {
- info!(node = %self.node.raft.id, "ClusterRunner received shutdown signal");
- break;
+ tokio::select! {
+ // Shutdown requested (value changed) or sender dropped (Err).
+ changed = self.shutdown.changed() => {
+ if changed.is_err() || *self.shutdown.borrow() {
+ info!(node = %self.node.raft.id, "ClusterRunner received shutdown signal");
+ break;
+ }
+ }
+ // Drive one iteration of the Raft loop: recv → handle → tick.
+ res = self.node.step(tick) => {
+ if let Err(e) = res {
+ warn!(
+ node = %self.node.raft.id,
+ error = %e,
+ "Raft step error"
+ );
+ }
+ }
}
}