Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(cluster): exclude self from the peer set — the real #1168 root cause

xerj-orgeb50a36d738f
diff --git a/engine/crates/xerj-cluster/src/raft.rs b/engine/crates/xerj-cluster/src/raft.rs
index c44d9da95..f6fc8c321 100644
--- a/engine/crates/xerj-cluster/src/raft.rs
+++ b/engine/crates/xerj-cluster/src/raft.rs
@@ -19,7 +19,7 @@ use std::time::{Duration, Instant};
use anyhow::{bail, Result};
use rand::Rng;
use serde::{Deserialize, Serialize};
-use tracing::{debug, info};
+use tracing::{debug, info, warn};
/// State a Raft node can be in.
#[derive(Debug, Clone, PartialEq)]
@@ -178,8 +178,22 @@ pub struct RaftNode {
impl RaftNode {
/// Create a new Raft node.
///
- /// All nodes start as followers in term 0.
+ /// All nodes start as followers in term 0. A `peers` entry equal to this
+ /// node's own id is **excluded** (with a warning): the documented config
+ /// convention lists the full membership on every node, so a self entry is
+ /// expected in the input — but leaving it in made `cluster_size` one too
+ /// large (a 3-node ring computed majority = 3, i.e. unanimity — zero fault
+ /// tolerance) and had every node address its own RequestVote to itself,
+ /// counting its own vote twice (#1168).
pub fn new(id: String, peers: Vec<String>) -> Self {
+ if peers.iter().any(|p| p == &id) {
+ warn!(
+ node = %id,
+ "peer list included this node's own id — excluded (self is not a peer: \
+ cluster_size would be wrong and the node would count its own vote twice)"
+ );
+ }
+ let peers: Vec<String> = peers.into_iter().filter(|p| p != &id).collect();
let election_timeout = Self::random_election_timeout();
RaftNode {
id,
@@ -414,6 +428,14 @@ impl RaftNode {
self.current_term
}
+ /// The peer ids this node addresses (self already excluded — see
+ /// [`RaftNode::new`]). Exposed so a caller can assert the config-shape
+ /// fix (#1168): a membership list that included this node's own id must
+ /// not reach the state machine as a peer.
+ pub fn peers(&self) -> &[String] {
+ &self.peers
+ }
+
/// Current role.
pub fn state(&self) -> &RaftState {
&self.state
@@ -887,6 +909,23 @@ mod tests {
}
}
+ /// A membership list that includes this node's own id (the documented
+ /// full-membership config convention) must not reach the state machine as
+ /// a peer (#1168): self-in-peers made cluster_size 4 in a 3-node ring —
+ /// majority 3, i.e. unanimity, no fault tolerance — and had the node
+ /// grant its own RequestVote to itself, counting its own vote twice.
+ #[test]
+ fn test_self_is_excluded_from_peers() {
+ let node = RaftNode::new(
+ "n1".to_string(),
+ vec!["n1".to_string(), "n2".to_string(), "n3".to_string()],
+ );
+ assert_eq!(node.peers(), &["n2".to_string(), "n3".to_string()]);
+
+ // And the majority reflects the REAL cluster: 3 members → 2.
+ assert_eq!(node.majority(), 2);
+ }
+
#[test]
fn test_single_node_leader() {
let mut node = RaftNode::new("n1".to_string(), vec![]);
diff --git a/engine/crates/xerj-cluster/src/transport.rs b/engine/crates/xerj-cluster/src/transport.rs
index d4f4b28a0..060c3640a 100644
--- a/engine/crates/xerj-cluster/src/transport.rs
+++ b/engine/crates/xerj-cluster/src/transport.rs
@@ -332,6 +332,32 @@ impl TcpTransport {
) -> Result<Self> {
let (tx, rx) = mpsc::channel::<(String, RaftMessage)>(1024);
+ // A peer entry naming this node's own address is excluded (#1168).
+ // The documented config convention lists the full membership on every
+ // node, so a self entry is expected in the input — but a node that
+ // sends its own RequestVote back to itself over loopback TCP counts
+ // its own vote twice and inflates the wire message rate for nothing.
+ let mut self_entry_removed = false;
+ let peers: HashMap<String, SocketAddr> = peers
+ .into_iter()
+ .filter(|(_id, addr)| {
+ if *addr == listen_addr {
+ self_entry_removed = true;
+ false
+ } else {
+ true
+ }
+ })
+ .collect();
+ if self_entry_removed {
+ warn!(
+ node = %node_id,
+ %listen_addr,
+ "peer list included this node's own address — excluded (a node sending to \
+ itself double-counts its own vote)"
+ );
+ }
+
// One sender task per peer (#1168). Spawned here, before the listener
// binds, so a send can be enqueued the moment the transport exists.
let mut outbound = HashMap::with_capacity(peers.len());
diff --git a/engine/crates/xerj-cluster/tests/transport_tests.rs b/engine/crates/xerj-cluster/tests/transport_tests.rs
index 656e7a52a..3b62292e6 100644
--- a/engine/crates/xerj-cluster/tests/transport_tests.rs
+++ b/engine/crates/xerj-cluster/tests/transport_tests.rs
@@ -446,3 +446,163 @@ async fn a_dead_peer_is_backed_off_not_retried_every_message() {
(ladder predicts ~4, pre-fix behaviour was ~100)"
);
}
+
+/// A peers map naming the transport's own listen address (the documented
+/// full-membership convention) must not produce a peer the node can send to:
+/// pre-fix, a node addressing its own RequestVote over loopback TCP counted
+/// its own vote twice (#1168).
+#[tokio::test]
+async fn a_peers_entry_naming_self_is_excluded_from_the_transport() {
+ use xerj_cluster::node::ClusterTransport;
+
+ let addr_a = free_addr().await;
+ let addr_b = free_addr().await;
+
+ // Full membership, self first — the config shape every node ships.
+ let mut peers_a = HashMap::new();
+ peers_a.insert("node-a".to_string(), addr_a);
+ peers_a.insert("node-b".to_string(), addr_b);
+
+ let transport_a = TcpTransport::new(
+ "node-a".to_string(),
+ addr_a,
+ peers_a,
+ ClusterSecret::new("shared-cluster-secret-for-tests").unwrap(),
+ )
+ .await
+ .expect("create transport A");
+
+ // Self is no longer a known peer …
+ let err = transport_a
+ .send("node-a", request_vote(1))
+ .await
+ .expect_err("sending to self must be rejected");
+ assert!(
+ err.to_string().contains("unknown peer"),
+ "expected unknown-peer error, got: {err:#}"
+ );
+
+ // … and never became a one-shot target either.
+ let direct = transport_a.send_to("node-a", &request_vote(1)).await;
+ assert!(direct.is_err(), "send_to(self) must fail");
+
+ // The real peer still works.
+ let mut peers_b = HashMap::new();
+ peers_b.insert("node-a".to_string(), addr_a);
+ let transport_b = TcpTransport::new(
+ "node-b".to_string(),
+ addr_b,
+ peers_b,
+ ClusterSecret::new("shared-cluster-secret-for-tests").unwrap(),
+ )
+ .await
+ .expect("create transport B");
+ tokio::time::sleep(Duration::from_millis(50)).await;
+
+ transport_a
+ .send("node-b", request_vote(7))
+ .await
+ .expect("enqueue to the real peer");
+ let (from, msg) = tokio::time::timeout(Duration::from_secs(2), async {
+ transport_b.recv().await.expect("recv error")
+ })
+ .await
+ .expect("message to the real peer must still arrive");
+ assert_eq!(from, "node-a");
+ assert!(matches!(msg, RaftMessage::RequestVote { term: 7, .. }));
+}
+
+/// The #1168 acceptance scenario at ring level: three nodes whose membership
+/// lists include every member (self included, the documented convention),
+/// a leader elected, then the leader's runner dropped mid-flight — a dead
+/// node. The two survivors must converge on exactly one leader and go quiet:
+/// no new terms, no re-election, for seconds afterward.
+///
+/// Pre-fix this churned forever: majority was 3 of 3 real nodes (unanimity),
+/// and each survivor only ever "won" an election by double-counting its own
+/// vote, so the survivors leapfrogged terms every ~250 ms — measured 94 terms
+/// in 4 minutes on rc.83, still unstable with the transport fix alone.
+#[tokio::test]
+async fn ring_with_full_membership_lists_survives_leader_death() {
+ use tokio::sync::watch;
+
+ let bus = InMemoryBus::new();
+ // Full membership on every node, self INCLUDED — what the server used to
+ // pass straight through (#1168), and what RaftNode::new now filters.
+ let membership = ["n1", "n2", "n3"];
+
+ let mk_node = |id: &'static str| async {
+ let transport = InMemoryTransport::new(id.to_string(), bus.clone()).await;
+ let peers: Vec<String> = membership.iter().map(|s| s.to_string()).collect();
+ ClusterNode::new(id.to_string(), peers, Box::new(transport))
+ };
+
+ let (_shutdown_tx, shutdown_rx) = watch::channel(false);
+ let tick = Duration::from_millis(5);
+ let mut runners = vec![
+ ClusterRunner::new(mk_node("n1").await, tick, shutdown_rx.clone()),
+ ClusterRunner::new(mk_node("n2").await, tick, shutdown_rx.clone()),
+ ClusterRunner::new(mk_node("n3").await, tick, shutdown_rx.clone()),
+ ];
+
+ // Phase 1 — elect a leader (drive all three loops concurrently).
+ {
+ let mut it = std::mem::take(&mut runners).into_iter();
+ let (mut x, mut y, mut z) = (it.next().unwrap(), it.next().unwrap(), it.next().unwrap());
+ tokio::select! {
+ _ = x.run() => {}
+ _ = y.run() => {}
+ _ = z.run() => {}
+ _ = tokio::time::sleep(Duration::from_secs(2)) => {}
+ }
+ runners = vec![x, y, z];
+ }
+ let leader_count = runners.iter().filter(|r| r.is_leader()).count();
+ assert_eq!(leader_count, 1, "exactly one leader before the failure");
+
+ // Phase 2 — kill the leader by dropping its runner (transport dropped →
+ // its InMemory receiver closes → peers' sends to it fail = dead node).
+ let leader_idx = runners
+ .iter()
+ .position(|r| r.is_leader())
+ .expect("phase 1 asserted a leader");
+ drop(runners.remove(leader_idx));
+ assert_eq!(runners.len(), 2, "two survivors");
+
+ // Phase 2a — the survivors elect exactly one new leader.
+ {
+ let mut it = std::mem::take(&mut runners).into_iter();
+ let (mut a, mut b) = (it.next().unwrap(), it.next().unwrap());
+ tokio::select! {
+ _ = a.run() => {}
+ _ = b.run() => {}
+ _ = tokio::time::sleep(Duration::from_secs(2)) => {}
+ }
+ runners = vec![a, b];
+ }
+ let leader_count = runners.iter().filter(|r| r.is_leader()).count();
+ assert_eq!(
+ leader_count, 1,
+ "survivors must converge on exactly one leader after the leader dies"
+ );
+ let terms_2a: Vec<u64> = runners.iter().map(|r| r.node.raft.current_term()).collect();
+
+ // Phase 2b — and go quiet: two more seconds, same terms, same leadership.
+ {
+ let mut it = std::mem::take(&mut runners).into_iter();
+ let (mut a, mut b) = (it.next().unwrap(), it.next().unwrap());
+ tokio::select! {
+ _ = a.run() => {}
+ _ = b.run() => {}
+ _ = tokio::time::sleep(Duration::from_secs(2)) => {}
+ }
+ runners = vec![a, b];
+ }
+ let terms_2b: Vec<u64> = runners.iter().map(|r| r.node.raft.current_term()).collect();
+ assert_eq!(
+ terms_2a, terms_2b,
+ "the surviving pair must be election-stable: terms kept advancing => churn (#1168)"
+ );
+ let leader_count = runners.iter().filter(|r| r.is_leader()).count();
+ assert_eq!(leader_count, 1, "still exactly one leader at the end");
+}
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index 1e654caeb..ab34be228 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -2502,6 +2502,26 @@ async fn async_main() -> Result<()> {
// (`SocketAddr::parse`, just above), not the unparseable `::1:9300`.
let node_id = listen_addr.to_string();
+ // The documented `peers` convention lists the FULL membership on
+ // every node — "the current node's id is inferred from the entry
+ // whose address matches its own" (docs/clustering) — so this node's
+ // own entry is expected to be in the map. Self is not a peer: left
+ // in, the Raft node counted 4 members of a 3-node ring (majority 3 =
+ // unanimity, no fault tolerance) and every node sent its own
+ // RequestVote back to itself over loopback TCP, counting its own
+ // vote twice — the measured result was a ring that churned through
+ // a new election every few hundred ms forever after ANY node died
+ // (#1168). Removed here, once, so the transport peer map and the
+ // Raft peer list agree. The cluster crate also excludes self
+ // defensively on both layers.
+ if peers.remove(&node_id).is_some() {
+ info!(
+ node = %node_id,
+ "cluster.peers included this node's own address — excluded from the peer set \
+ (self is not a peer)"
+ );
+ }
+
let tick = std::time::Duration::from_millis(cfg.cluster.tick_ms);
match TcpTransport::new(node_id.clone(), listen_addr, peers, cluster_secret).await {
@@ -2511,6 +2531,7 @@ async fn async_main() -> Result<()> {
.peers
.iter()
.filter_map(|p| p.split_once('=').map(|(id, _)| id.to_string()))
+ .filter(|id| id != &node_id)
.collect();
let node = ClusterNode::new(node_id.clone(), peer_ids, Box::new(transport));