AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(xerj-cluster): per-peer outbound queues — a dead peer can no longer starve heartbeats to live peers (#1168)
xerj-orgacf3665aed42
diff --git a/engine/crates/xerj-cluster/src/transport.rs b/engine/crates/xerj-cluster/src/transport.rsindex b3f485b1a..d4f4b28a0 100644--- a/engine/crates/xerj-cluster/src/transport.rs+++ b/engine/crates/xerj-cluster/src/transport.rs@@ -75,6 +75,28 @@ const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);/// post-connect exchange is bounded instead.const SEND_TIMEOUT: Duration = Duration::from_secs(5);+/// Outbound messages buffered per peer before new ones are dropped (#1168).+///+/// A peer that has not drained its queue in this many messages (hours of+/// heartbeats at the 50 ms rate) is down; Raft retransmits by design — the+/// leader's next AppendEntries resends everything from `next_index` — so+/// dropping is safe where buffering is not.+const OUTBOUND_QUEUE_CAPACITY: usize = 256;++/// Backoff before the next attempt to a peer after `failures` consecutive+/// failed sends: 100 ms, 200, 400, … capped at 6.4 s (#1168).+///+/// Public so the policy is testable from the integration tests: a dead peer+/// costs one immediate attempt, a handful in the first second, then at most+/// one per 6.4 s — never the 20 attempts/s of the pre-fix per-tick retry,+/// and never a `WARN` per tick.+pub fn backoff_after_failures(failures: u32) -> Duration {+ const BASE_MS: u64 = 100;+ const CAP_MS: u64 = 6_400;+ let shift = failures.saturating_sub(1).min(6);+ Duration::from_millis((BASE_MS << shift).min(CAP_MS))+}+/// Length of the receiver's handshake: magic ‖ version ‖ challenge.const HANDSHAKE_LEN: usize = WIRE_MAGIC.len() + 1 + CHALLENGE_LEN;@@ -255,8 +277,21 @@ async fn read_frame(/// TCP-based transport for inter-node communication.////// Incoming messages arrive via a background listener task and are delivered-/// through an mpsc channel. Outgoing messages open a fresh TCP connection per-/// send (connection pooling is a future optimisation).+/// through an mpsc channel. Outgoing messages are **enqueued** per peer — one+/// bounded queue and one long-lived sender task per peer — so the Raft tick+/// loop never awaits a peer's TCP I/O (#1168). Each attempt still opens a+/// fresh connection (connection pooling is a future optimisation).+///+/// Failure semantics of the outbound path, by design: a send that fails or a+/// queue that is full **drops the message** and backs off+/// ([`backoff_after_failures`]); it never blocks, retries synchronously, or+/// buffers unboundedly. Raft tolerates the drops — heartbeats and log+/// replication are retransmitted from `next_index` on the next tick, and a+/// vote that arrives late is simply ignored. The pre-fix behaviour (inline+/// `await` per send, bounded only by [`SEND_TIMEOUT`]) let one dead peer+/// consume the full 5 s of every heartbeat round while the election timeout+/// is 150–300 ms: the live peers starved, and a 3-node ring killed at its+/// leader churned through a new election every 1.5–11 s indefinitely.////// Every connection is authenticated in both directions of setup: the receiver/// proves nothing (it holds no identity beyond the secret) but issues a@@ -273,6 +308,11 @@ pub struct TcpTransport {secret: ClusterSecret,/// Receives `(sender_node_id, msg)` from the background listener.incoming: Arc<Mutex<mpsc::Receiver<(String, RaftMessage)>>>,+ /// One outbound queue per peer, drained by [`peer_sender_task`]. Built+ /// once in [`TcpTransport::new`] — the peer set is fixed for the+ /// transport's lifetime (membership changes arrive via the Raft log and+ /// will rebuild the transport, #1170).+ outbound: HashMap<String, mpsc::Sender<RaftMessage>>,// The sender half is kept alive so the channel is not closed when the// background listener task terminates.#[allow(dead_code)]@@ -292,12 +332,28 @@ impl TcpTransport {) -> Result<Self> {let (tx, rx) = mpsc::channel::<(String, RaftMessage)>(1024);+ // One sender task per peer (#1168). Spawned here, before the listener+ // binds, so a send can be enqueued the moment the transport exists.+ let mut outbound = HashMap::with_capacity(peers.len());+ for (peer_id, addr) in peers.iter() {+ let (peer_tx, peer_rx) = mpsc::channel::<RaftMessage>(OUTBOUND_QUEUE_CAPACITY);+ tokio::spawn(peer_sender_task(+ node_id.clone(),+ peer_id.clone(),+ *addr,+ secret.clone(),+ peer_rx,+ ));+ outbound.insert(peer_id.clone(), peer_tx);+ }+let transport = TcpTransport {node_id: node_id.clone(),listen_addr,peers: Arc::new(peers),secret,incoming: Arc::new(Mutex::new(rx)),+ outbound,sender: tx.clone(),};@@ -341,37 +397,133 @@ impl TcpTransport {Ok(())}- /// Send a message to a specific peer by node ID.+ /// Send a message to a specific peer by node ID, awaiting the full TCP+ /// exchange.////// Opens a fresh TCP connection, completes the authenticated handshake,- /// writes the frame, then closes.+ /// writes the frame, then closes. This is the **one-shot** path, bounded+ /// by [`SEND_TIMEOUT`] — used by tests and anywhere a synchronous+ /// delivery result is genuinely needed. The Raft loop does NOT use it:+ /// [`ClusterTransport::send`] enqueues instead (see the struct doc).pub async fn send_to(&self, peer_id: &str, msg: &RaftMessage) -> Result<()> {let addr = self.peers.get(peer_id).ok_or_else(|| anyhow::anyhow!("unknown peer: {peer_id}"))?;+ send_frame(&self.node_id, peer_id, *addr, &self.secret, msg).await+ }+}- let mut stream = TcpStream::connect(addr)- .await- .with_context(|| format!("connect to peer {peer_id} at {addr}"))?;-- // The receiver speaks first: magic, version, challenge. Bound the whole- // exchange so an accepting-but-silent peer cannot stall the Raft loop.- tokio::time::timeout(SEND_TIMEOUT, async {- let challenge = read_handshake(&mut stream).await?;- write_hello(&mut stream, &self.secret, &challenge, &self.node_id).await?;- write_frame(&mut stream, &self.secret, &challenge, &self.node_id, 0, msg).await- })+/// Open a connection to `addr` and deliver one authenticated frame.+///+/// The receiver speaks first: magic, version, challenge. The whole+/// post-connect exchange is bounded by [`SEND_TIMEOUT`] so an+/// accepting-but-silent peer cannot hold the caller indefinitely.+async fn send_frame(+ node_id: &str,+ peer_id: &str,+ addr: SocketAddr,+ secret: &ClusterSecret,+ msg: &RaftMessage,+) -> Result<()> {+ let mut stream = TcpStream::connect(addr).await- .with_context(|| format!("send to peer {peer_id} at {addr} timed out"))?- .with_context(|| format!("send to peer {peer_id} at {addr}"))+ .with_context(|| format!("connect to peer {peer_id} at {addr}"))?;++ tokio::time::timeout(SEND_TIMEOUT, async {+ let challenge = read_handshake(&mut stream).await?;+ write_hello(&mut stream, secret, &challenge, node_id).await?;+ write_frame(&mut stream, secret, &challenge, node_id, 0, msg).await+ })+ .await+ .with_context(|| format!("send to peer {peer_id} at {addr} timed out"))?+ .with_context(|| format!("send to peer {peer_id} at {addr}"))+}++/// Drain one peer's outbound queue, one message per connection attempt,+/// off the Raft loop's critical path (#1168).+///+/// On failure the message is dropped and the next attempt waits+/// [`backoff_after_failures`] — Raft retransmits, so a dropped heartbeat or+/// vote response costs nothing but a tick. Logging is per **state change**+/// (`WARN` when a peer that was up goes down, `INFO` when it answers again),+/// never per attempt: the pre-fix code logged a `WARN` every tick for every+/// dead peer, which is how a 4-minute outage produced thousands of identical+/// lines. The task exits when the queue's sender half is dropped with the+/// transport.+async fn peer_sender_task(+ node_id: String,+ peer_id: String,+ addr: SocketAddr,+ secret: ClusterSecret,+ mut rx: mpsc::Receiver<RaftMessage>,+) {+ let mut failures: u32 = 0;+ let mut down = false;+ let mut retry_at: Option<std::time::Instant> = None;++ while let Some(msg) = rx.recv().await {+ if let Some(deadline) = retry_at {+ if std::time::Instant::now() < deadline {+ continue; // in backoff — drop, the next heartbeat replaces it+ }+ }+ match send_frame(&node_id, &peer_id, addr, &secret, &msg).await {+ Ok(()) => {+ failures = 0;+ retry_at = None;+ if down {+ down = false;+ info!(node = %node_id, peer = %peer_id, %addr, "peer reachable again");+ }+ }+ Err(e) => {+ failures += 1;+ retry_at = Some(std::time::Instant::now() + backoff_after_failures(failures));+ if !down {+ down = true;+ warn!(+ node = %node_id,+ peer = %peer_id,+ %addr,+ error = %e,+ backoff_ms = backoff_after_failures(failures).as_millis() as u64,+ "peer unreachable — Raft messages to it are dropped until it answers"+ );+ } else {+ debug!(node = %node_id, peer = %peer_id, %addr, error = %e, "send still failing");+ }+ }+ }}}#[async_trait]impl ClusterTransport for TcpTransport {+ /// Enqueue `msg` on the peer's outbound queue — returns as soon as the+ /// message is accepted for delivery, never after the TCP exchange.+ ///+ /// Errors only for an unknown peer or a full queue (the message is+ /// dropped — see the struct doc for why that is the safe failure mode).+ /// Delivery failures are logged by the per-peer task, not returned here.async fn send(&self, to: &str, msg: RaftMessage) -> Result<()> {- self.send_to(to, &msg).await+ let tx = self+ .outbound+ .get(to)+ .ok_or_else(|| anyhow::anyhow!("unknown peer: {to}"))?;+ match tx.try_send(msg) {+ Ok(()) => Ok(()),+ Err(mpsc::error::TrySendError::Full(_)) => {+ // The peer has not drained OUTBOUND_QUEUE_CAPACITY messages —+ // it is down with the queue still backing up before the+ // backoff deadline. Drop and let retransmission cover it.+ debug!(node = %self.node_id, peer = %to, "outbound queue full — dropping Raft message");+ Ok(())+ }+ Err(mpsc::error::TrySendError::Closed(_)) => {+ anyhow::bail!("peer {to} sender task is gone")+ }+ }}async fn recv(&self) -> Result<(String, RaftMessage)> {@@ -427,3 +579,29 @@ async fn handle_connection(Ok(())}++#[cfg(test)]+mod tests {+ use super::*;++ /// The ladder documented on [`backoff_after_failures`]: 100 ms doubling to+ /// a 6.4 s cap. Pinned here because the #1168 churn window (an election+ /// every 1.5–11 s) brackets a 5 s synchronous send timeout — the ladder's+ /// job is to make a dead peer cost ~1 attempt per 6.4 s instead of one+ /// per tick, so a change to these numbers changes the failure story.+ #[test]+ fn backoff_ladder_doubles_then_caps() {+ assert_eq!(backoff_after_failures(1), Duration::from_millis(100));+ assert_eq!(backoff_after_failures(2), Duration::from_millis(200));+ assert_eq!(backoff_after_failures(3), Duration::from_millis(400));+ assert_eq!(backoff_after_failures(4), Duration::from_millis(800));+ assert_eq!(backoff_after_failures(5), Duration::from_millis(1_600));+ assert_eq!(backoff_after_failures(6), Duration::from_millis(3_200));+ assert_eq!(backoff_after_failures(7), Duration::from_millis(6_400));+ assert_eq!(backoff_after_failures(8), Duration::from_millis(6_400));+ assert_eq!(backoff_after_failures(1_000), Duration::from_millis(6_400));+ // failures = 0 is not a state the task reaches (the first failure+ // counts as 1), but saturate rather than panic if it ever does.+ assert_eq!(backoff_after_failures(0), Duration::from_millis(100));+ }+}diff --git a/engine/crates/xerj-cluster/tests/transport_tests.rs b/engine/crates/xerj-cluster/tests/transport_tests.rsindex b139a2db9..656e7a52a 100644--- a/engine/crates/xerj-cluster/tests/transport_tests.rs+++ b/engine/crates/xerj-cluster/tests/transport_tests.rs@@ -279,3 +279,170 @@ async fn test_single_node_cluster_via_runner() {"index should be committed and applied");}++// ── #1168: outbound sends must never block the Raft loop on one peer ────────++/// A listener that accepts connections (so `connect` succeeds) but never+/// speaks the cluster handshake — the shape of a peer that is dying slowly+/// (SIGTERM graceful shutdown with the listener still bound) or hung. The+/// pre-fix send path blocked on exactly this peer for the full 5 s+/// `SEND_TIMEOUT`, per heartbeat, while the election timeout is 150–300 ms.+///+/// Counts how many connections it accepted so tests can assert the *attempt+/// rate*, not just delivery.+async fn black_hole_peer() -> (SocketAddr, std::sync::Arc<std::sync::atomic::AtomicUsize>) {+ use std::sync::atomic::AtomicUsize;+ use std::sync::Arc;++ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let accepts = Arc::new(AtomicUsize::new(0));++ let counter = accepts.clone();+ tokio::spawn(async move {+ let mut held = Vec::new();+ while let Ok((stream, _)) = listener.accept().await {+ counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ // Hold the stream open without ever writing the handshake.+ held.push(stream);+ }+ });+ (addr, accepts)+}++fn request_vote(term: u64) -> RaftMessage {+ RaftMessage::RequestVote {+ term,+ candidate_id: "node-a".to_string(),+ last_log_index: 0,+ last_log_term: 0,+ }+}++/// `ClusterTransport::send` must return as soon as the message is *enqueued*,+/// not after the TCP exchange: a stalled (accepting-but-silent) peer costs the+/// Raft tick loop nothing. Pre-fix, this send awaited the full `SEND_TIMEOUT`+/// and then failed.+#[tokio::test]+async fn send_to_a_stalled_peer_returns_without_awaiting_the_tcp_exchange() {+ use xerj_cluster::node::ClusterTransport;++ let (stalled_addr, _accepts) = black_hole_peer().await;+ let addr_a = free_addr().await;+ let mut peers_a = HashMap::new();+ peers_a.insert("stalled".to_string(), stalled_addr);++ let transport_a = TcpTransport::new(+ "node-a".to_string(),+ addr_a,+ peers_a,+ ClusterSecret::new("shared-cluster-secret-for-tests").unwrap(),+ )+ .await+ .expect("create transport A");++ // 2 s is generous (the pre-fix path took the full 5 s timeout and erred);+ // generous is deliberate — the assertion is the ordering of magnitude,+ // not microtiming, so a co-tenant-loaded runner cannot flake it.+ tokio::time::timeout(Duration::from_secs(2), async {+ transport_a.send("stalled", request_vote(1)).await+ })+ .await+ .expect("send() must not await the stalled peer's TCP exchange")+ .expect("enqueue must succeed");+}++/// The #1168 scenario in miniature: a leader dispatching to a stalled peer and+/// a healthy peer in the same tick. The healthy peer must receive its message+/// immediately — pre-fix the sequential inline awaits made the stalled peer's+/// 5 s timeout gate every later send in the same dispatch batch, which is how+/// killing one node starved heartbeats to the survivors.+#[tokio::test]+async fn a_stalled_peer_does_not_starve_a_healthy_peer() {+ use xerj_cluster::node::ClusterTransport;++ let (stalled_addr, _accepts) = black_hole_peer().await;+ let addr_a = free_addr().await;+ let addr_b = free_addr().await;++ let mut peers_a = HashMap::new();+ peers_a.insert("stalled".to_string(), stalled_addr);+ peers_a.insert("node-b".to_string(), addr_b);+ let mut peers_b = HashMap::new();+ peers_b.insert("node-a".to_string(), addr_a);++ let secret = ClusterSecret::new("shared-cluster-secret-for-tests").unwrap();+ let transport_a = TcpTransport::new("node-a".to_string(), addr_a, peers_a, secret.clone())+ .await+ .expect("create transport A");+ let transport_b = TcpTransport::new("node-b".to_string(), addr_b, peers_b, secret)+ .await+ .expect("create transport B");++ tokio::time::sleep(Duration::from_millis(50)).await;++ // Same dispatch order as ClusterNode::dispatch: stalled peer first.+ transport_a+ .send("stalled", request_vote(1))+ .await+ .expect("enqueue to stalled peer");+ transport_a+ .send("node-b", request_vote(2))+ .await+ .expect("enqueue to healthy peer");++ // Pre-fix this took ≥ 5 s (the stalled send's timeout gated the healthy+ // send); post-fix it is one loopback round-trip. 2 s to stay flake-safe.+ let (from, msg) = tokio::time::timeout(Duration::from_secs(2), async {+ transport_b.recv().await.expect("recv error")+ })+ .await+ .expect("healthy peer starved by the stalled peer's send timeout");++ assert_eq!(from, "node-a");+ match msg {+ RaftMessage::RequestVote { term, .. } => assert_eq!(term, 2),+ other => panic!("expected RequestVote, got {other:?}"),+ }+}++/// A dead peer is retried on the backoff ladder, not once per heartbeat: 100+/// sends across 1 s (the pre-fix pattern was one attempt per 50 ms tick) must+/// produce only a handful of connection attempts. The upper bound is generous+/// (the ladder predicts ~4: 0 ms, 100 ms, 300 ms, 700 ms) so scheduler jitter+/// under a loaded co-tenant cannot flake it — the pre-fix count was ~100.+#[tokio::test]+async fn a_dead_peer_is_backed_off_not_retried_every_message() {+ use std::sync::atomic::Ordering;++ use xerj_cluster::node::ClusterTransport;++ let (stalled_addr, accepts) = black_hole_peer().await;+ let addr_a = free_addr().await;+ let mut peers_a = HashMap::new();+ peers_a.insert("stalled".to_string(), stalled_addr);++ let transport_a = TcpTransport::new(+ "node-a".to_string(),+ addr_a,+ peers_a,+ ClusterSecret::new("shared-cluster-secret-for-tests").unwrap(),+ )+ .await+ .expect("create transport A");++ for i in 0..100u64 {+ transport_a+ .send("stalled", request_vote(i))+ .await+ .expect("enqueue must keep succeeding while the peer is down");+ tokio::time::sleep(Duration::from_millis(10)).await;+ }++ let attempts = accepts.load(Ordering::Relaxed);+ assert!(+ attempts <= 8,+ "dead peer was attempted {attempts} times in ~1 s — backoff is not engaging \+ (ladder predicts ~4, pre-fix behaviour was ~100)"+ );+}