AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(cluster): exclude self from the peer set — the real #1168 root cause
xerj-orgeb50a36d738f
diff --git a/engine/crates/xerj-cluster/src/raft.rs b/engine/crates/xerj-cluster/src/raft.rsindex c44d9da95..f6fc8c321 100644--- a/engine/crates/xerj-cluster/src/raft.rs+++ b/engine/crates/xerj-cluster/src/raft.rs@@ -19,7 +19,7 @@ use std::time::{Duration, Instant};use anyhow::{bail, Result};use rand::Rng;use serde::{Deserialize, Serialize};-use tracing::{debug, info};+use tracing::{debug, info, warn};/// State a Raft node can be in.#[derive(Debug, Clone, PartialEq)]@@ -178,8 +178,22 @@ pub struct RaftNode {impl RaftNode {/// Create a new Raft node.///- /// All nodes start as followers in term 0.+ /// All nodes start as followers in term 0. A `peers` entry equal to this+ /// node's own id is **excluded** (with a warning): the documented config+ /// convention lists the full membership on every node, so a self entry is+ /// expected in the input — but leaving it in made `cluster_size` one too+ /// large (a 3-node ring computed majority = 3, i.e. unanimity — zero fault+ /// tolerance) and had every node address its own RequestVote to itself,+ /// counting its own vote twice (#1168).pub fn new(id: String, peers: Vec<String>) -> Self {+ if peers.iter().any(|p| p == &id) {+ warn!(+ node = %id,+ "peer list included this node's own id — excluded (self is not a peer: \+ cluster_size would be wrong and the node would count its own vote twice)"+ );+ }+ let peers: Vec<String> = peers.into_iter().filter(|p| p != &id).collect();let election_timeout = Self::random_election_timeout();RaftNode {id,@@ -414,6 +428,14 @@ impl RaftNode {self.current_term}+ /// The peer ids this node addresses (self already excluded — see+ /// [`RaftNode::new`]). Exposed so a caller can assert the config-shape+ /// fix (#1168): a membership list that included this node's own id must+ /// not reach the state machine as a peer.+ pub fn peers(&self) -> &[String] {+ &self.peers+ }+/// Current role.pub fn state(&self) -> &RaftState {&self.state@@ -887,6 +909,23 @@ mod tests {}}+ /// A membership list that includes this node's own id (the documented+ /// full-membership config convention) must not reach the state machine as+ /// a peer (#1168): self-in-peers made cluster_size 4 in a 3-node ring —+ /// majority 3, i.e. unanimity, no fault tolerance — and had the node+ /// grant its own RequestVote to itself, counting its own vote twice.+ #[test]+ fn test_self_is_excluded_from_peers() {+ let node = RaftNode::new(+ "n1".to_string(),+ vec!["n1".to_string(), "n2".to_string(), "n3".to_string()],+ );+ assert_eq!(node.peers(), &["n2".to_string(), "n3".to_string()]);++ // And the majority reflects the REAL cluster: 3 members → 2.+ assert_eq!(node.majority(), 2);+ }+#[test]fn test_single_node_leader() {let mut node = RaftNode::new("n1".to_string(), vec![]);diff --git a/engine/crates/xerj-cluster/src/transport.rs b/engine/crates/xerj-cluster/src/transport.rsindex d4f4b28a0..060c3640a 100644--- a/engine/crates/xerj-cluster/src/transport.rs+++ b/engine/crates/xerj-cluster/src/transport.rs@@ -332,6 +332,32 @@ impl TcpTransport {) -> Result<Self> {let (tx, rx) = mpsc::channel::<(String, RaftMessage)>(1024);+ // A peer entry naming this node's own address is excluded (#1168).+ // The documented config convention lists the full membership on every+ // node, so a self entry is expected in the input — but a node that+ // sends its own RequestVote back to itself over loopback TCP counts+ // its own vote twice and inflates the wire message rate for nothing.+ let mut self_entry_removed = false;+ let peers: HashMap<String, SocketAddr> = peers+ .into_iter()+ .filter(|(_id, addr)| {+ if *addr == listen_addr {+ self_entry_removed = true;+ false+ } else {+ true+ }+ })+ .collect();+ if self_entry_removed {+ warn!(+ node = %node_id,+ %listen_addr,+ "peer list included this node's own address — excluded (a node sending to \+ itself double-counts its own vote)"+ );+ }+// One sender task per peer (#1168). Spawned here, before the listener// binds, so a send can be enqueued the moment the transport exists.let mut outbound = HashMap::with_capacity(peers.len());diff --git a/engine/crates/xerj-cluster/tests/transport_tests.rs b/engine/crates/xerj-cluster/tests/transport_tests.rsindex 656e7a52a..3b62292e6 100644--- a/engine/crates/xerj-cluster/tests/transport_tests.rs+++ b/engine/crates/xerj-cluster/tests/transport_tests.rs@@ -446,3 +446,163 @@ async fn a_dead_peer_is_backed_off_not_retried_every_message() {(ladder predicts ~4, pre-fix behaviour was ~100)");}++/// A peers map naming the transport's own listen address (the documented+/// full-membership convention) must not produce a peer the node can send to:+/// pre-fix, a node addressing its own RequestVote over loopback TCP counted+/// its own vote twice (#1168).+#[tokio::test]+async fn a_peers_entry_naming_self_is_excluded_from_the_transport() {+ use xerj_cluster::node::ClusterTransport;++ let addr_a = free_addr().await;+ let addr_b = free_addr().await;++ // Full membership, self first — the config shape every node ships.+ let mut peers_a = HashMap::new();+ peers_a.insert("node-a".to_string(), addr_a);+ peers_a.insert("node-b".to_string(), addr_b);++ let transport_a = TcpTransport::new(+ "node-a".to_string(),+ addr_a,+ peers_a,+ ClusterSecret::new("shared-cluster-secret-for-tests").unwrap(),+ )+ .await+ .expect("create transport A");++ // Self is no longer a known peer …+ let err = transport_a+ .send("node-a", request_vote(1))+ .await+ .expect_err("sending to self must be rejected");+ assert!(+ err.to_string().contains("unknown peer"),+ "expected unknown-peer error, got: {err:#}"+ );++ // … and never became a one-shot target either.+ let direct = transport_a.send_to("node-a", &request_vote(1)).await;+ assert!(direct.is_err(), "send_to(self) must fail");++ // The real peer still works.+ let mut peers_b = HashMap::new();+ peers_b.insert("node-a".to_string(), addr_a);+ let transport_b = TcpTransport::new(+ "node-b".to_string(),+ addr_b,+ peers_b,+ ClusterSecret::new("shared-cluster-secret-for-tests").unwrap(),+ )+ .await+ .expect("create transport B");+ tokio::time::sleep(Duration::from_millis(50)).await;++ transport_a+ .send("node-b", request_vote(7))+ .await+ .expect("enqueue to the real peer");+ let (from, msg) = tokio::time::timeout(Duration::from_secs(2), async {+ transport_b.recv().await.expect("recv error")+ })+ .await+ .expect("message to the real peer must still arrive");+ assert_eq!(from, "node-a");+ assert!(matches!(msg, RaftMessage::RequestVote { term: 7, .. }));+}++/// The #1168 acceptance scenario at ring level: three nodes whose membership+/// lists include every member (self included, the documented convention),+/// a leader elected, then the leader's runner dropped mid-flight — a dead+/// node. The two survivors must converge on exactly one leader and go quiet:+/// no new terms, no re-election, for seconds afterward.+///+/// Pre-fix this churned forever: majority was 3 of 3 real nodes (unanimity),+/// and each survivor only ever "won" an election by double-counting its own+/// vote, so the survivors leapfrogged terms every ~250 ms — measured 94 terms+/// in 4 minutes on rc.83, still unstable with the transport fix alone.+#[tokio::test]+async fn ring_with_full_membership_lists_survives_leader_death() {+ use tokio::sync::watch;++ let bus = InMemoryBus::new();+ // Full membership on every node, self INCLUDED — what the server used to+ // pass straight through (#1168), and what RaftNode::new now filters.+ let membership = ["n1", "n2", "n3"];++ let mk_node = |id: &'static str| async {+ let transport = InMemoryTransport::new(id.to_string(), bus.clone()).await;+ let peers: Vec<String> = membership.iter().map(|s| s.to_string()).collect();+ ClusterNode::new(id.to_string(), peers, Box::new(transport))+ };++ let (_shutdown_tx, shutdown_rx) = watch::channel(false);+ let tick = Duration::from_millis(5);+ let mut runners = vec![+ ClusterRunner::new(mk_node("n1").await, tick, shutdown_rx.clone()),+ ClusterRunner::new(mk_node("n2").await, tick, shutdown_rx.clone()),+ ClusterRunner::new(mk_node("n3").await, tick, shutdown_rx.clone()),+ ];++ // Phase 1 — elect a leader (drive all three loops concurrently).+ {+ let mut it = std::mem::take(&mut runners).into_iter();+ let (mut x, mut y, mut z) = (it.next().unwrap(), it.next().unwrap(), it.next().unwrap());+ tokio::select! {+ _ = x.run() => {}+ _ = y.run() => {}+ _ = z.run() => {}+ _ = tokio::time::sleep(Duration::from_secs(2)) => {}+ }+ runners = vec![x, y, z];+ }+ let leader_count = runners.iter().filter(|r| r.is_leader()).count();+ assert_eq!(leader_count, 1, "exactly one leader before the failure");++ // Phase 2 — kill the leader by dropping its runner (transport dropped →+ // its InMemory receiver closes → peers' sends to it fail = dead node).+ let leader_idx = runners+ .iter()+ .position(|r| r.is_leader())+ .expect("phase 1 asserted a leader");+ drop(runners.remove(leader_idx));+ assert_eq!(runners.len(), 2, "two survivors");++ // Phase 2a — the survivors elect exactly one new leader.+ {+ let mut it = std::mem::take(&mut runners).into_iter();+ let (mut a, mut b) = (it.next().unwrap(), it.next().unwrap());+ tokio::select! {+ _ = a.run() => {}+ _ = b.run() => {}+ _ = tokio::time::sleep(Duration::from_secs(2)) => {}+ }+ runners = vec![a, b];+ }+ let leader_count = runners.iter().filter(|r| r.is_leader()).count();+ assert_eq!(+ leader_count, 1,+ "survivors must converge on exactly one leader after the leader dies"+ );+ let terms_2a: Vec<u64> = runners.iter().map(|r| r.node.raft.current_term()).collect();++ // Phase 2b — and go quiet: two more seconds, same terms, same leadership.+ {+ let mut it = std::mem::take(&mut runners).into_iter();+ let (mut a, mut b) = (it.next().unwrap(), it.next().unwrap());+ tokio::select! {+ _ = a.run() => {}+ _ = b.run() => {}+ _ = tokio::time::sleep(Duration::from_secs(2)) => {}+ }+ runners = vec![a, b];+ }+ let terms_2b: Vec<u64> = runners.iter().map(|r| r.node.raft.current_term()).collect();+ assert_eq!(+ terms_2a, terms_2b,+ "the surviving pair must be election-stable: terms kept advancing => churn (#1168)"+ );+ let leader_count = runners.iter().filter(|r| r.is_leader()).count();+ assert_eq!(leader_count, 1, "still exactly one leader at the end");+}diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex 1e654caeb..ab34be228 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -2502,6 +2502,26 @@ async fn async_main() -> Result<()> {// (`SocketAddr::parse`, just above), not the unparseable `::1:9300`.let node_id = listen_addr.to_string();+ // The documented `peers` convention lists the FULL membership on+ // every node — "the current node's id is inferred from the entry+ // whose address matches its own" (docs/clustering) — so this node's+ // own entry is expected to be in the map. Self is not a peer: left+ // in, the Raft node counted 4 members of a 3-node ring (majority 3 =+ // unanimity, no fault tolerance) and every node sent its own+ // RequestVote back to itself over loopback TCP, counting its own+ // vote twice — the measured result was a ring that churned through+ // a new election every few hundred ms forever after ANY node died+ // (#1168). Removed here, once, so the transport peer map and the+ // Raft peer list agree. The cluster crate also excludes self+ // defensively on both layers.+ if peers.remove(&node_id).is_some() {+ info!(+ node = %node_id,+ "cluster.peers included this node's own address — excluded from the peer set \+ (self is not a peer)"+ );+ }+let tick = std::time::Duration::from_millis(cfg.cluster.tick_ms);match TcpTransport::new(node_id.clone(), listen_addr, peers, cluster_secret).await {@@ -2511,6 +2531,7 @@ async fn async_main() -> Result<()> {.peers.iter().filter_map(|p| p.split_once('=').map(|(id, _)| id.to_string()))+ .filter(|id| id != &node_id).collect();let node = ClusterNode::new(node_id.clone(), peer_ids, Box::new(transport));