Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(server): make --auto-port opt-in and harden the port/data-dir scan

xerj-org1f7b27b0afcb
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index c931e05f5..54b351eaa 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -618,8 +618,10 @@ pub struct ServerConfig {
/// right-most address that is *not* itself a listed proxy is used — the
/// left end of the chain is written by the caller and cannot be trusted.
pub trusted_proxies: Vec<String>,
- /// When `true`, automatically pick the next available port if a listener port
- /// is already in use by another instance.
+ /// Multi-instance opt-in (default `false`). When `true`, a listener port
+ /// that is already in use makes the node bind the next free one, and a
+ /// `data_dir` held by another live process makes it use `<data_dir>-N`.
+ /// Off by default so a taken port stays a fatal startup error (#465).
#[serde(default)]
pub auto_port: bool,
}
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index 77af5e2e2..d7129a88d 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -325,10 +325,11 @@ fn help_text(feedback: bool) -> String {
flag: xerj --insecure --port 9300 -d ./other-data\n\
Without it the native REST API listens on 8080 and gRPC on\n\
8081 ([server] rest_port / grpc_port in the TOML config).\n\
- --auto-port Automatically bind the next available ports and fallback\n\
- data dir if in use by another instance (default for CLI\n\
- runs without a config file). Env: XERJ_AUTO_PORT\n\
- --no-auto-port Fail immediately if a port is already in use.\n\
+ --auto-port Opt in to multi-instance mode: if a port is taken, bind the\n\
+ next free one; if the data dir is held by another live\n\
+ process, use <data-dir>-2, -3, ... The banner prints the\n\
+ ports actually bound. Off by default. Env: XERJ_AUTO_PORT\n\
+ --no-auto-port Fail at startup if a port is taken (the default).\n\
--disable-feedback Do not print the feedback invitation above. Honoured in any\n\
position, including after --help. Env:\n\
XERJ_DISABLE_FEEDBACK=true\n\
@@ -649,20 +650,25 @@ fn load_config(args: &CliArgs) -> Result<Config> {
cfg.server.grpc_port = base + 2;
}
- // Multi-instance auto-port selection: when enabled, if default or requested
- // listener ports are already in use, the server automatically scans and binds
- // the next available free ports.
- // By default, auto_port is enabled when no explicit config file is provided
- // (the standard CLI quickstart / multi-instance agent workflow).
+ // Multi-instance auto-port selection: when enabled, if the requested
+ // listener ports are already in use, the server scans for and binds the
+ // next free ports, and moves off a data dir another live process holds.
+ //
+ // OPT-IN ONLY (flag, env, or `[server] auto_port = true`). A taken port is
+ // a fatal startup error by default on purpose (issue #465): a client
+ // pointed at the requested port would otherwise talk to — and write into —
+ // whichever other instance owns it, while this one reports success on a
+ // port nobody is addressing. Precedence: CLI flag > env > config file.
if let Some(ap) = args.auto_port {
cfg.server.auto_port = ap;
- } else if let Ok(val) = std::env::var("XERJ_AUTO_PORT") {
- cfg.server.auto_port = matches!(
- val.trim().to_ascii_lowercase().as_str(),
- "1" | "true" | "yes" | "on"
- );
- } else if args.config.is_none() {
- cfg.server.auto_port = true;
+ } else if let Ok(raw) = std::env::var("XERJ_AUTO_PORT") {
+ // Parsed strictly, like XERJ_ALLOW_INSECURE_NETWORK_BIND: a typo must
+ // not silently pick a behaviour the operator did not ask for.
+ cfg.server.auto_port = match raw.trim().to_ascii_lowercase().as_str() {
+ "1" | "true" | "yes" | "on" => true,
+ "0" | "false" | "no" | "off" => false,
+ _ => anyhow::bail!("XERJ_AUTO_PORT={raw:?} is not a boolean; use true or false"),
+ };
}
if cfg.server.auto_port {
@@ -1196,52 +1202,50 @@ fn bind_listener(addr: SocketAddr, name: &str) -> Result<std::net::TcpListener>
Ok(listener)
}
-/// If `data_dir` is currently locked by another active running process, find
-/// the first available sibling directory (`<dir>-2`, `<dir>-3`, ...) so
-/// multiple instances can run simultaneously without colliding.
+/// If `data_dir` is currently locked by another live process, find the first
+/// sibling directory (`<dir>-2`, `<dir>-3`, ...) that is not, so multiple
+/// opted-in (`--auto-port`) instances can run side by side.
+///
+/// Advisory only: the engine re-acquires `node.lock` itself at open and fails
+/// cleanly if a racing instance took the directory in between. After
+/// [`MAX_DATA_DIR_SIBLINGS`] held siblings it gives up and returns the
+/// requested dir, so that engine lock error is what the operator sees.
fn resolve_available_data_dir(requested: &str) -> String {
- let p = Path::new(requested);
- let lock_file = p.join("node.lock");
- if !lock_file.exists() {
+ // A trailing separator would turn `./data/` into `./data/-2` — a
+ // directory INSIDE the held data dir. Siblings are built from the
+ // trimmed spelling (`/` alone is left as is).
+ let base = requested.trim_end_matches(['/', std::path::MAIN_SEPARATOR]);
+ let base = if base.is_empty() { requested } else { base };
+ if !data_dir_held(Path::new(base)) {
return requested.to_string();
}
-
- if let Ok(file) = std::fs::OpenOptions::new()
- .read(true)
- .write(true)
- .open(&lock_file)
- {
- if file.try_lock().is_err() {
- let mut i = 2;
- loop {
- let candidate = format!("{requested}-{i}");
- let candidate_path = Path::new(&candidate);
- let candidate_lock = candidate_path.join("node.lock");
- if !candidate_lock.exists() {
- info!(
- "data dir '{requested}' is already locked by another running instance; using '{candidate}' for this instance"
- );
- return candidate;
- }
- if let Ok(f) = std::fs::OpenOptions::new()
- .read(true)
- .write(true)
- .open(&candidate_lock)
- {
- if f.try_lock().is_ok() {
- info!(
- "data dir '{requested}' is already locked by another running instance; using '{candidate}' for this instance"
- );
- return candidate;
- }
- }
- i += 1;
- }
+ for i in 2..=MAX_DATA_DIR_SIBLINGS + 1 {
+ let candidate = format!("{base}-{i}");
+ if !data_dir_held(Path::new(&candidate)) {
+ warn!(
+ "data dir '{requested}' is held by another running xerj process; \
+ --auto-port: using '{candidate}' for this instance"
+ );
+ return candidate;
}
}
requested.to_string()
}
+/// How many `<dir>-N` siblings [`resolve_available_data_dir`] will try.
+const MAX_DATA_DIR_SIBLINGS: usize = 64;
+
+/// Is `dir/node.lock` currently locked by a live process? A missing or
+/// unopenable lock file is "not held" — the engine's own lock acquisition is
+/// the authority and reports anything stranger.
+fn data_dir_held(dir: &Path) -> bool {
+ std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .open(dir.join("node.lock"))
+ .is_ok_and(|f| matches!(f.try_lock(), Err(std::fs::TryLockError::WouldBlock)))
+}
+
/// Bind one listener, attempting consecutive ports if `auto_port` is true and
/// the starting port is already in use (`AddrInUse`).
fn bind_listener_candidate(
@@ -1251,21 +1255,26 @@ fn bind_listener_candidate(
auto_port: bool,
avoid_ports: &[u16],
) -> Result<(std::net::TcpListener, u16)> {
- if !auto_port {
+ // Port 0 asks the kernel for an ephemeral port, which can never be "in
+ // use": there is nothing to scan, and the port to report is the one the
+ // kernel handed back, not the 0 that was requested.
+ if !auto_port || port == 0 {
let addr = SocketAddr::new(ip, port);
let listener = bind_listener(addr, name)?;
- return Ok((listener, port));
+ let bound = listener
+ .local_addr()
+ .with_context(|| format!("{name}: read the bound address"))?
+ .port();
+ return Ok((listener, bound));
}
- let mut candidate = port;
- let max_port = port.saturating_add(1000).min(65535);
-
- while candidate <= max_port {
+ // Scan in u16 space without ever incrementing past 65535 (a `+= 1` there
+ // would panic in debug and wrap to the ephemeral port 0 in release).
+ let max_port = port.saturating_add(AUTO_PORT_SCAN);
+ for candidate in port..=max_port {
if avoid_ports.contains(&candidate) {
- candidate += 1;
continue;
}
-
let addr = SocketAddr::new(ip, candidate);
match std::net::TcpListener::bind(addr) {
Ok(listener) => {
@@ -1273,24 +1282,21 @@ fn bind_listener_candidate(
.set_nonblocking(true)
.with_context(|| format!("{name}: set the listener on {addr} non-blocking"))?;
if candidate != port {
- info!(
- "{name}: port {port} in use, automatically assigned port {candidate} for multi-instance support"
- );
+ warn!("{name}: port {port} is in use; --auto-port bound {candidate} instead");
}
return Ok((listener, candidate));
}
- Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
- candidate += 1;
- }
- Err(e) => {
- return Err(anyhow::anyhow!("{name}: bind {addr}: {e}"));
- }
+ Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {}
+ Err(e) => return Err(anyhow::anyhow!("{name}: bind {addr}: {e}")),
}
}
anyhow::bail!("{name}: could not find a free port between {port} and {max_port}");
}
+/// How far past the requested port `--auto-port` scans.
+const AUTO_PORT_SCAN: u16 = 1000;
+
async fn serve_on(
router: Router,
listener: std::net::TcpListener,
@@ -3552,3 +3558,101 @@ mod server_correctness_tests {
assert_eq!(std::fs::read(&stale).unwrap(), b"new");
}
}
+
+#[cfg(test)]
+mod auto_port_tests {
+ use super::*;
+ use std::net::{IpAddr, Ipv4Addr, TcpListener};
+
+ const LO: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST);
+
+ fn held_port() -> (TcpListener, u16) {
+ let l = TcpListener::bind((LO, 0)).unwrap();
+ let p = l.local_addr().unwrap().port();
+ (l, p)
+ }
+
+ #[test]
+ fn auto_port_is_off_unless_asked_for() {
+ // No config file, no flag: a taken port must stay fatal (#465).
+ let args = parse_args_from(["--insecure".to_string()]);
+ if std::env::var_os("XERJ_AUTO_PORT").is_none() {
+ assert!(!load_config(&args).unwrap().server.auto_port);
+ }
+ let args = parse_args_from(["--auto-port".to_string()]);
+ assert!(load_config(&args).unwrap().server.auto_port);
+ }
+
+ #[test]
+ fn taken_port_is_fatal_without_auto_port() {
+ let (_held, p) = held_port();
+ assert!(bind_listener_candidate(LO, p, "t", false, &[]).is_err());
+ }
+
+ #[test]
+ fn taken_port_moves_with_auto_port() {
+ let (_held, p) = held_port();
+ let (l, got) = bind_listener_candidate(LO, p, "t", true, &[]).unwrap();
+ assert_ne!(got, p);
+ assert_eq!(l.local_addr().unwrap().port(), got);
+ }
+
+ #[test]
+ fn avoided_ports_are_skipped() {
+ let (held, p) = held_port();
+ drop(held); // p is free now, but listed as reserved for a sibling
+ let (_l, got) = bind_listener_candidate(LO, p, "t", true, &[p]).unwrap();
+ assert_ne!(got, p);
+ }
+
+ #[test]
+ fn ephemeral_port_reports_the_kernel_port() {
+ // Port 0 + auto_port used to report 0, push 0 onto the avoid list, and
+ // then scan the next listener from port 1.
+ let (_a, pa) = bind_listener_candidate(LO, 0, "a", true, &[]).unwrap();
+ let (_b, pb) = bind_listener_candidate(LO, 0, "b", true, &[pa]).unwrap();
+ assert_ne!(pa, 0);
+ assert_ne!(pb, 0);
+ assert_ne!(pa, pb);
+ }
+
+ fn hold(dir: &Path) -> std::fs::File {
+ std::fs::create_dir_all(dir).unwrap();
+ let f = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(false)
+ .open(dir.join("node.lock"))
+ .unwrap();
+ f.try_lock().unwrap();
+ f
+ }
+
+ #[test]
+ fn free_data_dir_is_kept() {
+ let t = tempfile::tempdir().unwrap();
+ let d = t.path().join("data");
+ let d = d.to_str().unwrap();
+ assert_eq!(resolve_available_data_dir(d), d);
+ // A stale lock file from a dead process is not "held".
+ std::fs::create_dir_all(d).unwrap();
+ std::fs::write(Path::new(d).join("node.lock"), "12345").unwrap();
+ assert_eq!(resolve_available_data_dir(d), d);
+ }
+
+ #[test]
+ fn held_data_dir_moves_to_a_sibling() {
+ let t = tempfile::tempdir().unwrap();
+ let d = t.path().join("data");
+ let _g1 = hold(&d);
+ let _g2 = hold(&t.path().join("data-2"));
+ let ds = d.to_str().unwrap();
+ assert_eq!(resolve_available_data_dir(ds), format!("{ds}-3"));
+ // Trailing separator: the sibling must not land INSIDE the held dir.
+ assert_eq!(
+ resolve_available_data_dir(&format!("{ds}/")),
+ format!("{ds}-3")
+ );
+ }
+}
diff --git a/engine/xerj.default.toml b/engine/xerj.default.toml
index 15b7548da..d5b265675 100644
--- a/engine/xerj.default.toml
+++ b/engine/xerj.default.toml
@@ -39,10 +39,12 @@ grpc_port = 8081
# Point existing ES clients, Kibana, or Logstash here.
es_compat_port = 9200
-# When true, automatically scan and bind the next available ports if the
-# requested ports are already in use by another running instance.
-# Default in TOML is false (fail closed on taken ports); enabled by default for
-# CLI runs when no config file is passed.
+# Multi-instance opt-in. When true, a taken port makes this node bind the next
+# free one (the banner prints the ports actually bound), and a data_dir held by
+# another live xerj process makes it use "<data_dir>-2", "-3", ... instead.
+# Off by default everywhere: a taken port is a fatal startup error so a client
+# pointed at the configured port can never silently reach another instance.
+# Also: --auto-port / --no-auto-port, or env XERJ_AUTO_PORT=true|false.
auto_port = false
# Directory where all index data, WAL files, and segments are stored.