Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(serve): bounded retry on transient serve-lock errors + errno diagnostics

AkitaOnRailsb9e557d518fd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index aedb30ba..55cdf209 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
+- `ai-memory serve` no longer hard-fails to take its single-instance lock on a
+ transient error under load. Acquiring the serve lock now retries `open` and
+ `try_lock_exclusive` a few times with a short (~25ms) backoff when they hit a
+ transient failure (EMFILE/ENFILE fd exhaustion, EINTR), mirroring
+ `acquire_drain_lock`. A genuinely contended lock (`WouldBlock`, another server
+ holds it) is never retried and still refuses startup immediately. The
+ serve-lock tests also assert with the concrete errno so any remaining
+ environmental flake is diagnosable rather than silent (#745).
- The generated OpenCode and OpenCode 2 plugins now forward a subagent session's
`parentID` as the `agent_id` marker, so `[capture] drop_subagent_captures` can
recognize and drop OpenCode subagent sessions. Previously both plugins emitted
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index c86ad0ea..1ca78327 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -185,19 +185,72 @@ fn holder_info_path(data_dir: &Path) -> std::path::PathBuf {
/// mount with unreliable locking) must not be stranded. A filesystem that
/// cannot lock at all only downgrades the guard to a warning — refusing to
/// start there would be worse than the unguarded risk.
+/// Transient failures `open`/`try_lock_exclusive` can raise on a healthy but
+/// loaded machine: fd exhaustion (EMFILE per-process, ENFILE system-wide) and
+/// interrupted syscalls (EINTR). These deserve a short retry. A `WouldBlock`
+/// (another server already holds the lock, classified by
+/// `is_drain_lock_busy_error`) is deliberately excluded — that is the correct
+/// "someone else owns it" refusal and must surface immediately, never retried.
+fn is_transient_serve_lock_error(err: &std::io::Error) -> bool {
+ if err.kind() == std::io::ErrorKind::Interrupted {
+ return true;
+ }
+ #[cfg(unix)]
+ if let Some(code) = err.raw_os_error() {
+ // EMFILE (per-process fd limit) / ENFILE (system-wide fd limit).
+ const EMFILE: i32 = 24;
+ const ENFILE: i32 = 23;
+ if code == EMFILE || code == ENFILE {
+ return true;
+ }
+ }
+ false
+}
+
fn acquire_serve_lock(data_dir: &Path, force: bool) -> Result<Option<ServeLock>> {
std::fs::create_dir_all(data_dir)
.with_context(|| format!("creating data directory {}", data_dir.display()))?;
let path = data_dir.join(SERVE_LOCK_FILE);
- let file = std::fs::OpenOptions::new()
- .create(true)
- .truncate(false)
- .read(true)
- .write(true)
- .open(&path)
- .with_context(|| format!("opening serve lock {}", path.display()))?;
use fs2::FileExt as _;
- match file.try_lock_exclusive() {
+
+ // Open the file and take the exclusive flock under a short bounded retry
+ // for transient errors only. A loaded machine (parallel test runs, an fd
+ // storm) can bounce `open` with EMFILE/ENFILE or interrupt the lock call
+ // with EINTR; a real server must not hard-fail on that. Mirrors
+ // `acquire_drain_lock`'s bounded ~25ms backoff. A `WouldBlock` (another
+ // holder) is never retried here — it falls through to the refusal path.
+ const SERVE_LOCK_ACQUIRE_ATTEMPTS: u32 = 5;
+ const SERVE_LOCK_ACQUIRE_BACKOFF: Duration = Duration::from_millis(25);
+ let mut attempt: u32 = 0;
+ let (file, lock_result) = loop {
+ attempt += 1;
+ let retriable = attempt < SERVE_LOCK_ACQUIRE_ATTEMPTS;
+ let file = match std::fs::OpenOptions::new()
+ .create(true)
+ .truncate(false)
+ .read(true)
+ .write(true)
+ .open(&path)
+ {
+ Ok(file) => file,
+ Err(err) if retriable && is_transient_serve_lock_error(&err) => {
+ std::thread::sleep(SERVE_LOCK_ACQUIRE_BACKOFF);
+ continue;
+ }
+ Err(err) => {
+ return Err(err).with_context(|| format!("opening serve lock {}", path.display()));
+ }
+ };
+ match file.try_lock_exclusive() {
+ Err(err) if retriable && is_transient_serve_lock_error(&err) => {
+ std::thread::sleep(SERVE_LOCK_ACQUIRE_BACKOFF);
+ continue;
+ }
+ result => break (file, result),
+ }
+ };
+
+ match lock_result {
Ok(()) => {
// Informational only: the flock is the guard, and this names the
// holder in a later refusal message. Best-effort, and written to
@@ -2423,11 +2476,46 @@ mod tests {
}
}
+ /// Assert `acquire_serve_lock` returned a real held lock, panicking with
+ /// the concrete cause otherwise. The bare `.unwrap()...is_some()` collapsed
+ /// a transient `Err` (EMFILE/ENFILE/EINTR under parallel fd pressure) or an
+ /// `Ok(None)` downgrade into an un-actionable flake; this turns the next
+ /// occurrence into a one-line errno diagnosis while still requiring the
+ /// lock to be genuinely held.
+ fn assert_serve_lock_held(result: Result<Option<ServeLock>>) -> ServeLock {
+ match result {
+ Ok(Some(lock)) => lock,
+ Ok(None) => panic!(
+ "acquire_serve_lock downgraded to an unguarded start (Ok(None)) although no other holder exists in this test"
+ ),
+ Err(err) => panic!("acquire_serve_lock failed: {err:?}"),
+ }
+ }
+
+ #[test]
+ fn transient_errors_are_retriable_but_a_busy_lock_is_not() {
+ use std::io::{Error, ErrorKind};
+ // EINTR is transient and cross-platform via ErrorKind::Interrupted.
+ assert!(is_transient_serve_lock_error(&Error::from(
+ ErrorKind::Interrupted
+ )));
+ #[cfg(unix)]
+ {
+ // EMFILE / ENFILE fd exhaustion is transient.
+ assert!(is_transient_serve_lock_error(&Error::from_raw_os_error(24)));
+ assert!(is_transient_serve_lock_error(&Error::from_raw_os_error(23)));
+ }
+ // A contended lock (WouldBlock) is the "someone else owns it" signal:
+ // it is busy, never transient, and must not be retried away.
+ let busy = Error::from(ErrorKind::WouldBlock);
+ assert!(!is_transient_serve_lock_error(&busy));
+ assert!(crate::commands::hook_spool::is_drain_lock_busy_error(&busy));
+ }
+
#[test]
fn second_server_on_the_same_data_dir_is_refused_and_names_the_holder() {
let dir = TempDir::new().unwrap();
- let first = acquire_serve_lock(dir.path(), false).unwrap();
- assert!(first.is_some());
+ let _first = assert_serve_lock_held(acquire_serve_lock(dir.path(), false));
let err = acquire_serve_lock(dir.path(), false)
.unwrap_err()
.to_string();
@@ -2444,7 +2532,7 @@ mod tests {
#[test]
fn force_starts_unguarded_while_the_holder_keeps_the_lock() {
let dir = TempDir::new().unwrap();
- let _first = acquire_serve_lock(dir.path(), false).unwrap();
+ let _first = assert_serve_lock_held(acquire_serve_lock(dir.path(), false));
assert!(acquire_serve_lock(dir.path(), true).unwrap().is_none());
// --force bypasses the refusal, not the holder: a plain attempt still sees it.
assert!(acquire_serve_lock(dir.path(), false).is_err());
@@ -2458,7 +2546,7 @@ mod tests {
// Dropping the holder is what process exit does to the flock: the
// leftover .serve.lock file must not outlive the lock it named.
}
- assert!(acquire_serve_lock(dir.path(), false).unwrap().is_some());
+ assert_serve_lock_held(acquire_serve_lock(dir.path(), false));
}
#[test]