Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(xerj-cluster): per-peer outbound queues — a dead peer can no longer starve heartbeats to live peers (#1168)

xerj-orgacf3665aed42
diff --git a/engine/crates/xerj-cluster/src/transport.rs b/engine/crates/xerj-cluster/src/transport.rs
index b3f485b1a..d4f4b28a0 100644
--- a/engine/crates/xerj-cluster/src/transport.rs
+++ b/engine/crates/xerj-cluster/src/transport.rs
@@ -75,6 +75,28 @@ const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
/// post-connect exchange is bounded instead.
const SEND_TIMEOUT: Duration = Duration::from_secs(5);
+/// Outbound messages buffered per peer before new ones are dropped (#1168).
+///
+/// A peer that has not drained its queue in this many messages (hours of
+/// heartbeats at the 50 ms rate) is down; Raft retransmits by design — the
+/// leader's next AppendEntries resends everything from `next_index` — so
+/// dropping is safe where buffering is not.
+const OUTBOUND_QUEUE_CAPACITY: usize = 256;
+
+/// Backoff before the next attempt to a peer after `failures` consecutive
+/// failed sends: 100 ms, 200, 400, … capped at 6.4 s (#1168).
+///
+/// Public so the policy is testable from the integration tests: a dead peer
+/// costs one immediate attempt, a handful in the first second, then at most
+/// one per 6.4 s — never the 20 attempts/s of the pre-fix per-tick retry,
+/// and never a `WARN` per tick.
+pub fn backoff_after_failures(failures: u32) -> Duration {
+ const BASE_MS: u64 = 100;
+ const CAP_MS: u64 = 6_400;
+ let shift = failures.saturating_sub(1).min(6);
+ Duration::from_millis((BASE_MS << shift).min(CAP_MS))
+}
+
/// Length of the receiver's handshake: magic ‖ version ‖ challenge.
const HANDSHAKE_LEN: usize = WIRE_MAGIC.len() + 1 + CHALLENGE_LEN;
@@ -255,8 +277,21 @@ async fn read_frame(
/// TCP-based transport for inter-node communication.
///
/// Incoming messages arrive via a background listener task and are delivered
-/// through an mpsc channel. Outgoing messages open a fresh TCP connection per
-/// send (connection pooling is a future optimisation).
+/// through an mpsc channel. Outgoing messages are **enqueued** per peer — one
+/// bounded queue and one long-lived sender task per peer — so the Raft tick
+/// loop never awaits a peer's TCP I/O (#1168). Each attempt still opens a
+/// fresh connection (connection pooling is a future optimisation).
+///
+/// Failure semantics of the outbound path, by design: a send that fails or a
+/// queue that is full **drops the message** and backs off
+/// ([`backoff_after_failures`]); it never blocks, retries synchronously, or
+/// buffers unboundedly. Raft tolerates the drops — heartbeats and log
+/// replication are retransmitted from `next_index` on the next tick, and a
+/// vote that arrives late is simply ignored. The pre-fix behaviour (inline
+/// `await` per send, bounded only by [`SEND_TIMEOUT`]) let one dead peer
+/// consume the full 5 s of every heartbeat round while the election timeout
+/// is 150–300 ms: the live peers starved, and a 3-node ring killed at its
+/// leader churned through a new election every 1.5–11 s indefinitely.
///
/// Every connection is authenticated in both directions of setup: the receiver
/// proves nothing (it holds no identity beyond the secret) but issues a
@@ -273,6 +308,11 @@ pub struct TcpTransport {
secret: ClusterSecret,
/// Receives `(sender_node_id, msg)` from the background listener.
incoming: Arc<Mutex<mpsc::Receiver<(String, RaftMessage)>>>,
+ /// One outbound queue per peer, drained by [`peer_sender_task`]. Built
+ /// once in [`TcpTransport::new`] — the peer set is fixed for the
+ /// transport's lifetime (membership changes arrive via the Raft log and
+ /// will rebuild the transport, #1170).
+ outbound: HashMap<String, mpsc::Sender<RaftMessage>>,
// The sender half is kept alive so the channel is not closed when the
// background listener task terminates.
#[allow(dead_code)]
@@ -292,12 +332,28 @@ impl TcpTransport {
) -> Result<Self> {
let (tx, rx) = mpsc::channel::<(String, RaftMessage)>(1024);
+ // One sender task per peer (#1168). Spawned here, before the listener
+ // binds, so a send can be enqueued the moment the transport exists.
+ let mut outbound = HashMap::with_capacity(peers.len());
+ for (peer_id, addr) in peers.iter() {
+ let (peer_tx, peer_rx) = mpsc::channel::<RaftMessage>(OUTBOUND_QUEUE_CAPACITY);
+ tokio::spawn(peer_sender_task(
+ node_id.clone(),
+ peer_id.clone(),
+ *addr,
+ secret.clone(),
+ peer_rx,
+ ));
+ outbound.insert(peer_id.clone(), peer_tx);
+ }
+
let transport = TcpTransport {
node_id: node_id.clone(),
listen_addr,
peers: Arc::new(peers),
secret,
incoming: Arc::new(Mutex::new(rx)),
+ outbound,
sender: tx.clone(),
};
@@ -341,37 +397,133 @@ impl TcpTransport {
Ok(())
}
- /// Send a message to a specific peer by node ID.
+ /// Send a message to a specific peer by node ID, awaiting the full TCP
+ /// exchange.
///
/// Opens a fresh TCP connection, completes the authenticated handshake,
- /// writes the frame, then closes.
+ /// writes the frame, then closes. This is the **one-shot** path, bounded
+ /// by [`SEND_TIMEOUT`] — used by tests and anywhere a synchronous
+ /// delivery result is genuinely needed. The Raft loop does NOT use it:
+ /// [`ClusterTransport::send`] enqueues instead (see the struct doc).
pub async fn send_to(&self, peer_id: &str, msg: &RaftMessage) -> Result<()> {
let addr = self
.peers
.get(peer_id)
.ok_or_else(|| anyhow::anyhow!("unknown peer: {peer_id}"))?;
+ send_frame(&self.node_id, peer_id, *addr, &self.secret, msg).await
+ }
+}
- let mut stream = TcpStream::connect(addr)
- .await
- .with_context(|| format!("connect to peer {peer_id} at {addr}"))?;
-
- // The receiver speaks first: magic, version, challenge. Bound the whole
- // exchange so an accepting-but-silent peer cannot stall the Raft loop.
- tokio::time::timeout(SEND_TIMEOUT, async {
- let challenge = read_handshake(&mut stream).await?;
- write_hello(&mut stream, &self.secret, &challenge, &self.node_id).await?;
- write_frame(&mut stream, &self.secret, &challenge, &self.node_id, 0, msg).await
- })
+/// Open a connection to `addr` and deliver one authenticated frame.
+///
+/// The receiver speaks first: magic, version, challenge. The whole
+/// post-connect exchange is bounded by [`SEND_TIMEOUT`] so an
+/// accepting-but-silent peer cannot hold the caller indefinitely.
+async fn send_frame(
+ node_id: &str,
+ peer_id: &str,
+ addr: SocketAddr,
+ secret: &ClusterSecret,
+ msg: &RaftMessage,
+) -> Result<()> {
+ let mut stream = TcpStream::connect(addr)
.await
- .with_context(|| format!("send to peer {peer_id} at {addr} timed out"))?
- .with_context(|| format!("send to peer {peer_id} at {addr}"))
+ .with_context(|| format!("connect to peer {peer_id} at {addr}"))?;
+
+ tokio::time::timeout(SEND_TIMEOUT, async {
+ let challenge = read_handshake(&mut stream).await?;
+ write_hello(&mut stream, secret, &challenge, node_id).await?;
+ write_frame(&mut stream, secret, &challenge, node_id, 0, msg).await
+ })
+ .await
+ .with_context(|| format!("send to peer {peer_id} at {addr} timed out"))?
+ .with_context(|| format!("send to peer {peer_id} at {addr}"))
+}
+
+/// Drain one peer's outbound queue, one message per connection attempt,
+/// off the Raft loop's critical path (#1168).
+///
+/// On failure the message is dropped and the next attempt waits
+/// [`backoff_after_failures`] — Raft retransmits, so a dropped heartbeat or
+/// vote response costs nothing but a tick. Logging is per **state change**
+/// (`WARN` when a peer that was up goes down, `INFO` when it answers again),
+/// never per attempt: the pre-fix code logged a `WARN` every tick for every
+/// dead peer, which is how a 4-minute outage produced thousands of identical
+/// lines. The task exits when the queue's sender half is dropped with the
+/// transport.
+async fn peer_sender_task(
+ node_id: String,
+ peer_id: String,
+ addr: SocketAddr,
+ secret: ClusterSecret,
+ mut rx: mpsc::Receiver<RaftMessage>,
+) {
+ let mut failures: u32 = 0;
+ let mut down = false;
+ let mut retry_at: Option<std::time::Instant> = None;
+
+ while let Some(msg) = rx.recv().await {
+ if let Some(deadline) = retry_at {
+ if std::time::Instant::now() < deadline {
+ continue; // in backoff — drop, the next heartbeat replaces it
+ }
+ }
+ match send_frame(&node_id, &peer_id, addr, &secret, &msg).await {
+ Ok(()) => {
+ failures = 0;
+ retry_at = None;
+ if down {
+ down = false;
+ info!(node = %node_id, peer = %peer_id, %addr, "peer reachable again");
+ }
+ }
+ Err(e) => {
+ failures += 1;
+ retry_at = Some(std::time::Instant::now() + backoff_after_failures(failures));
+ if !down {
+ down = true;
+ warn!(
+ node = %node_id,
+ peer = %peer_id,
+ %addr,
+ error = %e,
+ backoff_ms = backoff_after_failures(failures).as_millis() as u64,
+ "peer unreachable — Raft messages to it are dropped until it answers"
+ );
+ } else {
+ debug!(node = %node_id, peer = %peer_id, %addr, error = %e, "send still failing");
+ }
+ }
+ }
}
}
#[async_trait]
impl ClusterTransport for TcpTransport {
+ /// Enqueue `msg` on the peer's outbound queue — returns as soon as the
+ /// message is accepted for delivery, never after the TCP exchange.
+ ///
+ /// Errors only for an unknown peer or a full queue (the message is
+ /// dropped — see the struct doc for why that is the safe failure mode).
+ /// Delivery failures are logged by the per-peer task, not returned here.
async fn send(&self, to: &str, msg: RaftMessage) -> Result<()> {
- self.send_to(to, &msg).await
+ let tx = self
+ .outbound
+ .get(to)
+ .ok_or_else(|| anyhow::anyhow!("unknown peer: {to}"))?;
+ match tx.try_send(msg) {
+ Ok(()) => Ok(()),
+ Err(mpsc::error::TrySendError::Full(_)) => {
+ // The peer has not drained OUTBOUND_QUEUE_CAPACITY messages —
+ // it is down with the queue still backing up before the
+ // backoff deadline. Drop and let retransmission cover it.
+ debug!(node = %self.node_id, peer = %to, "outbound queue full — dropping Raft message");
+ Ok(())
+ }
+ Err(mpsc::error::TrySendError::Closed(_)) => {
+ anyhow::bail!("peer {to} sender task is gone")
+ }
+ }
}
async fn recv(&self) -> Result<(String, RaftMessage)> {
@@ -427,3 +579,29 @@ async fn handle_connection(
Ok(())
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// The ladder documented on [`backoff_after_failures`]: 100 ms doubling to
+ /// a 6.4 s cap. Pinned here because the #1168 churn window (an election
+ /// every 1.5–11 s) brackets a 5 s synchronous send timeout — the ladder's
+ /// job is to make a dead peer cost ~1 attempt per 6.4 s instead of one
+ /// per tick, so a change to these numbers changes the failure story.
+ #[test]
+ fn backoff_ladder_doubles_then_caps() {
+ assert_eq!(backoff_after_failures(1), Duration::from_millis(100));
+ assert_eq!(backoff_after_failures(2), Duration::from_millis(200));
+ assert_eq!(backoff_after_failures(3), Duration::from_millis(400));
+ assert_eq!(backoff_after_failures(4), Duration::from_millis(800));
+ assert_eq!(backoff_after_failures(5), Duration::from_millis(1_600));
+ assert_eq!(backoff_after_failures(6), Duration::from_millis(3_200));
+ assert_eq!(backoff_after_failures(7), Duration::from_millis(6_400));
+ assert_eq!(backoff_after_failures(8), Duration::from_millis(6_400));
+ assert_eq!(backoff_after_failures(1_000), Duration::from_millis(6_400));
+ // failures = 0 is not a state the task reaches (the first failure
+ // counts as 1), but saturate rather than panic if it ever does.
+ assert_eq!(backoff_after_failures(0), Duration::from_millis(100));
+ }
+}
diff --git a/engine/crates/xerj-cluster/tests/transport_tests.rs b/engine/crates/xerj-cluster/tests/transport_tests.rs
index b139a2db9..656e7a52a 100644
--- a/engine/crates/xerj-cluster/tests/transport_tests.rs
+++ b/engine/crates/xerj-cluster/tests/transport_tests.rs
@@ -279,3 +279,170 @@ async fn test_single_node_cluster_via_runner() {
"index should be committed and applied"
);
}
+
+// ── #1168: outbound sends must never block the Raft loop on one peer ────────
+
+/// A listener that accepts connections (so `connect` succeeds) but never
+/// speaks the cluster handshake — the shape of a peer that is dying slowly
+/// (SIGTERM graceful shutdown with the listener still bound) or hung. The
+/// pre-fix send path blocked on exactly this peer for the full 5 s
+/// `SEND_TIMEOUT`, per heartbeat, while the election timeout is 150–300 ms.
+///
+/// Counts how many connections it accepted so tests can assert the *attempt
+/// rate*, not just delivery.
+async fn black_hole_peer() -> (SocketAddr, std::sync::Arc<std::sync::atomic::AtomicUsize>) {
+ use std::sync::atomic::AtomicUsize;
+ use std::sync::Arc;
+
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let accepts = Arc::new(AtomicUsize::new(0));
+
+ let counter = accepts.clone();
+ tokio::spawn(async move {
+ let mut held = Vec::new();
+ while let Ok((stream, _)) = listener.accept().await {
+ counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ // Hold the stream open without ever writing the handshake.
+ held.push(stream);
+ }
+ });
+ (addr, accepts)
+}
+
+fn request_vote(term: u64) -> RaftMessage {
+ RaftMessage::RequestVote {
+ term,
+ candidate_id: "node-a".to_string(),
+ last_log_index: 0,
+ last_log_term: 0,
+ }
+}
+
+/// `ClusterTransport::send` must return as soon as the message is *enqueued*,
+/// not after the TCP exchange: a stalled (accepting-but-silent) peer costs the
+/// Raft tick loop nothing. Pre-fix, this send awaited the full `SEND_TIMEOUT`
+/// and then failed.
+#[tokio::test]
+async fn send_to_a_stalled_peer_returns_without_awaiting_the_tcp_exchange() {
+ use xerj_cluster::node::ClusterTransport;
+
+ let (stalled_addr, _accepts) = black_hole_peer().await;
+ let addr_a = free_addr().await;
+ let mut peers_a = HashMap::new();
+ peers_a.insert("stalled".to_string(), stalled_addr);
+
+ let transport_a = TcpTransport::new(
+ "node-a".to_string(),
+ addr_a,
+ peers_a,
+ ClusterSecret::new("shared-cluster-secret-for-tests").unwrap(),
+ )
+ .await
+ .expect("create transport A");
+
+ // 2 s is generous (the pre-fix path took the full 5 s timeout and erred);
+ // generous is deliberate — the assertion is the ordering of magnitude,
+ // not microtiming, so a co-tenant-loaded runner cannot flake it.
+ tokio::time::timeout(Duration::from_secs(2), async {
+ transport_a.send("stalled", request_vote(1)).await
+ })
+ .await
+ .expect("send() must not await the stalled peer's TCP exchange")
+ .expect("enqueue must succeed");
+}
+
+/// The #1168 scenario in miniature: a leader dispatching to a stalled peer and
+/// a healthy peer in the same tick. The healthy peer must receive its message
+/// immediately — pre-fix the sequential inline awaits made the stalled peer's
+/// 5 s timeout gate every later send in the same dispatch batch, which is how
+/// killing one node starved heartbeats to the survivors.
+#[tokio::test]
+async fn a_stalled_peer_does_not_starve_a_healthy_peer() {
+ use xerj_cluster::node::ClusterTransport;
+
+ let (stalled_addr, _accepts) = black_hole_peer().await;
+ let addr_a = free_addr().await;
+ let addr_b = free_addr().await;
+
+ let mut peers_a = HashMap::new();
+ peers_a.insert("stalled".to_string(), stalled_addr);
+ peers_a.insert("node-b".to_string(), addr_b);
+ let mut peers_b = HashMap::new();
+ peers_b.insert("node-a".to_string(), addr_a);
+
+ let secret = ClusterSecret::new("shared-cluster-secret-for-tests").unwrap();
+ let transport_a = TcpTransport::new("node-a".to_string(), addr_a, peers_a, secret.clone())
+ .await
+ .expect("create transport A");
+ let transport_b = TcpTransport::new("node-b".to_string(), addr_b, peers_b, secret)
+ .await
+ .expect("create transport B");
+
+ tokio::time::sleep(Duration::from_millis(50)).await;
+
+ // Same dispatch order as ClusterNode::dispatch: stalled peer first.
+ transport_a
+ .send("stalled", request_vote(1))
+ .await
+ .expect("enqueue to stalled peer");
+ transport_a
+ .send("node-b", request_vote(2))
+ .await
+ .expect("enqueue to healthy peer");
+
+ // Pre-fix this took ≥ 5 s (the stalled send's timeout gated the healthy
+ // send); post-fix it is one loopback round-trip. 2 s to stay flake-safe.
+ let (from, msg) = tokio::time::timeout(Duration::from_secs(2), async {
+ transport_b.recv().await.expect("recv error")
+ })
+ .await
+ .expect("healthy peer starved by the stalled peer's send timeout");
+
+ assert_eq!(from, "node-a");
+ match msg {
+ RaftMessage::RequestVote { term, .. } => assert_eq!(term, 2),
+ other => panic!("expected RequestVote, got {other:?}"),
+ }
+}
+
+/// A dead peer is retried on the backoff ladder, not once per heartbeat: 100
+/// sends across 1 s (the pre-fix pattern was one attempt per 50 ms tick) must
+/// produce only a handful of connection attempts. The upper bound is generous
+/// (the ladder predicts ~4: 0 ms, 100 ms, 300 ms, 700 ms) so scheduler jitter
+/// under a loaded co-tenant cannot flake it — the pre-fix count was ~100.
+#[tokio::test]
+async fn a_dead_peer_is_backed_off_not_retried_every_message() {
+ use std::sync::atomic::Ordering;
+
+ use xerj_cluster::node::ClusterTransport;
+
+ let (stalled_addr, accepts) = black_hole_peer().await;
+ let addr_a = free_addr().await;
+ let mut peers_a = HashMap::new();
+ peers_a.insert("stalled".to_string(), stalled_addr);
+
+ let transport_a = TcpTransport::new(
+ "node-a".to_string(),
+ addr_a,
+ peers_a,
+ ClusterSecret::new("shared-cluster-secret-for-tests").unwrap(),
+ )
+ .await
+ .expect("create transport A");
+
+ for i in 0..100u64 {
+ transport_a
+ .send("stalled", request_vote(i))
+ .await
+ .expect("enqueue must keep succeeding while the peer is down");
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+
+ let attempts = accepts.load(Ordering::Relaxed);
+ assert!(
+ attempts <= 8,
+ "dead peer was attempted {attempts} times in ~1 s — backoff is not engaging \
+ (ladder predicts ~4, pre-fix behaviour was ~100)"
+ );
+}