Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(serve): enable TCP keepalive on accepted connections (#792)

AkitaOnRails00aa6ee8634b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 56c19cdf..8cf2a5e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
+- `ai-memory serve` no longer leaked file descriptors from half-open HTTP
+ connections until `EMFILE`, breaking the healthcheck (an unauthenticated
+ availability/DoS). A hook or MCP client whose peer died without sending FIN
+ (laptop sleep, a VPN/Tailscale flap, an abrupt kill) left its accepted
+ socket `ESTABLISHED` forever, since the OS default has TCP keepalive off —
+ each dead peer leaked one fd, exhausting the 1024-fd default in roughly 2-3
+ days of normal churn. Accepted connections now get TCP keepalive via
+ `socket2`, tunable with the new `tcp_keepalive_secs` config key (default
+ 60s; `AI_MEMORY_TCP_KEEPALIVE_SECS=0` disables keepalive). This closes the
+ half-open-socket half of the fd leak; the rmcp session-table half was
+ already fixed in 2.4.0 by the rmcp 2.x bump. (#792)
- `ai-memory bootstrap` no longer returns a 500 when the LLM emits a page
path containing a Windows-illegal character (e.g. a `:` copied verbatim
from a conventional-commit subject like `build(sandbox): orchestrate`).
diff --git a/Cargo.lock b/Cargo.lock
index b263d26d..8c08bbf3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -65,6 +65,7 @@ dependencies = [
"serde",
"serde_json",
"sha2",
+ "socket2",
"sysinfo",
"tar",
"tempfile",
diff --git a/crates/ai-memory-cli/Cargo.toml b/crates/ai-memory-cli/Cargo.toml
index 9ace14fc..0bb989a9 100644
--- a/crates/ai-memory-cli/Cargo.toml
+++ b/crates/ai-memory-cli/Cargo.toml
@@ -38,6 +38,9 @@ sha2.workspace = true
base64.workspace = true
anyhow.workspace = true
axum.workspace = true
+# Sets SO_KEEPALIVE on accepted `serve` sockets (#792); not a workspace dep,
+# `ai-memory-cli` is the only crate that needs it.
+socket2 = "0.6.3"
clap.workspace = true
clap_complete.workspace = true
crossterm.workspace = true
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index eb16e08a..d2c1811b 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -846,6 +846,59 @@ async fn run_session_consolidation_worker(
}
}
+/// Wraps [`tokio::net::TcpListener`] to enable TCP keepalive on every
+/// accepted connection.
+///
+/// Without this, a hook client whose peer dies without sending FIN (laptop
+/// sleep, a VPN/Tailscale flap, an abrupt kill) leaves its socket
+/// `ESTABLISHED` forever: the OS default is keepalive off, so the fd is
+/// never reclaimed. Over days that leaks one fd per dead peer until
+/// `accept()` starts failing with `EMFILE` and the healthcheck breaks (#792).
+/// Keepalive makes the kernel probe idle connections and close ones whose
+/// peer no longer answers.
+///
+/// This is built on axum's own [`axum::serve::ListenerExt::tap_io`] rather
+/// than a hand-rolled `impl axum::serve::Listener`. A hand-rolled newtype
+/// was tried first: it compiles as a `Listener`, but
+/// `into_make_service_with_connect_info::<SocketAddr>()` additionally needs
+/// `SocketAddr: Connected<IncomingStream<'_, L>>`, and axum only ships that
+/// impl for its own `TcpListener` and for `TapIo<L, F>` (generically, for any
+/// `L: Listener`) — never for an arbitrary third-party `L`. Implementing
+/// `Connected` ourselves is blocked by the orphan rule: neither `Connected`,
+/// `SocketAddr`, nor `IncomingStream` (a plain, non-fundamental axum type) is
+/// local to this crate. `tap_io` is the extension point axum actually
+/// provides for exactly this "touch every accepted `Io`" case, and it keeps
+/// `ConnectInfo` (real peer `SocketAddr`) working for free.
+fn keepalive_listener(
+ listener: tokio::net::TcpListener,
+ keepalive_secs: u64,
+) -> axum::serve::TapIo<
+ tokio::net::TcpListener,
+ impl FnMut(&mut tokio::net::TcpStream) + Send + 'static,
+> {
+ // `None` when `tcp_keepalive_secs = 0` (keepalive disabled) — pass
+ // accepted sockets through unmodified.
+ let keepalive = (keepalive_secs > 0).then(|| {
+ let idle = Duration::from_secs(keepalive_secs);
+ socket2::TcpKeepalive::new()
+ .with_time(idle)
+ .with_interval(idle)
+ });
+ axum::serve::ListenerExt::tap_io(listener, move |stream: &mut tokio::net::TcpStream| {
+ let Some(keepalive) = keepalive.as_ref() else {
+ return;
+ };
+ let sock_ref = socket2::SockRef::from(&*stream);
+ if let Err(error) = sock_ref.set_tcp_keepalive(keepalive) {
+ // Guard, don't panic (runtime paths never unwrap/expect): a
+ // platform or socket-state quirk here should not take down the
+ // connection, just leave it without the reaping this wrapper
+ // exists to provide.
+ tracing::warn!(%error, "failed to set TCP keepalive on accepted connection");
+ }
+ })
+}
+
/// Run the `serve` subcommand.
///
/// # Errors
@@ -1492,6 +1545,7 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {
docs/https-via-proxy.md for copy-paste templates."
);
}
+ let listener = keepalive_listener(listener, config.tcp_keepalive_secs);
let shutdown_cancel = cancel.clone();
let serve_result = {
let serve = axum::serve(
@@ -4793,4 +4847,42 @@ mod tests {
]
);
}
+
+ #[tokio::test]
+ async fn keepalive_listener_enables_socket_keepalive_when_configured() {
+ use axum::serve::Listener as _;
+
+ let raw = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let mut listener = keepalive_listener(raw, 60);
+ let addr = listener.local_addr().unwrap();
+
+ let client = tokio::spawn(async move { tokio::net::TcpStream::connect(addr).await });
+ let (accepted, _peer) = listener.accept().await;
+ let _client = client.await.unwrap().unwrap();
+
+ let sock_ref = socket2::SockRef::from(&accepted);
+ assert!(
+ sock_ref.keepalive().unwrap(),
+ "SO_KEEPALIVE must be enabled when tcp_keepalive_secs > 0"
+ );
+ }
+
+ #[tokio::test]
+ async fn keepalive_listener_disables_socket_keepalive_when_idle_is_zero() {
+ use axum::serve::Listener as _;
+
+ let raw = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let mut listener = keepalive_listener(raw, 0);
+ let addr = listener.local_addr().unwrap();
+
+ let client = tokio::spawn(async move { tokio::net::TcpStream::connect(addr).await });
+ let (accepted, _peer) = listener.accept().await;
+ let _client = client.await.unwrap().unwrap();
+
+ let sock_ref = socket2::SockRef::from(&accepted);
+ assert!(
+ !sock_ref.keepalive().unwrap(),
+ "SO_KEEPALIVE must stay off when tcp_keepalive_secs = 0"
+ );
+ }
}
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index 345e97de..1e3eae2e 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -25,6 +25,12 @@ use serde::{Deserialize, Serialize};
/// Default HTTP bind address for the local single-user server.
pub const DEFAULT_BIND: &str = "127.0.0.1:49374";
+/// Default idle time (seconds) before TCP keepalive probes start on an
+/// accepted `serve` connection. Conservative: long enough to never fire on a
+/// live, merely-quiet MCP/hook connection, short enough that a dead peer's
+/// fd is reclaimed in minutes rather than the OS default of ~2 hours (#792).
+pub const DEFAULT_TCP_KEEPALIVE_SECS: u64 = 60;
+
/// Default base URL used by thin-client CLI subcommands.
pub const DEFAULT_SERVER_URL: &str = "http://127.0.0.1:49374";
@@ -249,6 +255,14 @@ pub struct Config {
pub data_dir: PathBuf,
/// HTTP bind address used by `ai-memory serve`.
pub bind: String,
+ /// Idle-time (seconds) before the OS starts probing an accepted `serve`
+ /// connection with TCP keepalive. `0` disables keepalive entirely. A
+ /// hook client's peer can die without sending FIN (laptop sleep, a
+ /// VPN/Tailscale flap, an abrupt kill); without keepalive the socket
+ /// stays `ESTABLISHED` forever and leaks one fd per dead peer until
+ /// `accept()` fails with `EMFILE` and the healthcheck breaks (#792). Set
+ /// with `AI_MEMORY_TCP_KEEPALIVE_SECS`.
+ pub tcp_keepalive_secs: u64,
/// Base URL used by thin-client CLI commands to contact the running server.
pub server_url: String,
/// URL subpath the server is mounted under (e.g. `/wiki`). Thin-client
@@ -835,6 +849,7 @@ impl Default for Config {
Self {
data_dir: default_data_dir(),
bind: DEFAULT_BIND.into(),
+ tcp_keepalive_secs: DEFAULT_TCP_KEEPALIVE_SECS,
server_url: DEFAULT_SERVER_URL.into(),
base_path: String::new(),
home_dir: None,
@@ -2424,6 +2439,7 @@ mod tests {
let cfg = Config::default();
assert!(cfg.data_dir.ends_with("ai-memory"));
assert_eq!(cfg.bind, DEFAULT_BIND);
+ assert_eq!(cfg.tcp_keepalive_secs, DEFAULT_TCP_KEEPALIVE_SECS);
assert_eq!(cfg.server_url, DEFAULT_SERVER_URL);
assert_eq!(cfg.log_level, "info");
assert_eq!(
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 80e18fc3..7396f265 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -600,6 +600,11 @@ prefixed `AI_MEMORY_*`.
```toml
bind = "127.0.0.1:49374"
log_level = "info"
+tcp_keepalive_secs = 60 # idle time before TCP keepalive probes an accepted `serve`
+ # connection; reaps sockets left half-open by a dead peer
+ # (laptop sleep, VPN flap) that would otherwise leak fds
+ # until EMFILE (#792). 0 disables keepalive. Env:
+ # AI_MEMORY_TCP_KEEPALIVE_SECS
# Capture / launch UX (all default-on where noted). Each has an AI_MEMORY_* env
# override (AI_MEMORY_CAPTURE_ASSISTANT / AI_MEMORY_BACKFILL_ON_START /
diff --git a/docs/deploy.md b/docs/deploy.md
index 0cce680a..f6e644b8 100644
--- a/docs/deploy.md
+++ b/docs/deploy.md
@@ -406,7 +406,11 @@ ssh "$SERVER" "tail -100 $DEPLOY_DIR/data/logs/ai-memory.log.$(date +%F)"
- **`unhealthy`** status: the container is running but its embedded
`ai-memory status` healthcheck is failing. Most likely the data
dir's permissions don't match the container's user (uid 1000). Fix
- with `sudo chown -R 1000:1000 $DEPLOY_DIR/data` on the host.
+ with `sudo chown -R 1000:1000 $DEPLOY_DIR/data` on the host. A slower-onset
+ cause (days, not minutes) is fd exhaustion from dead-peer connections
+ (laptop sleep, a VPN flap) accumulating without TCP keepalive; `serve`
+ enables it by default (`tcp_keepalive_secs = 60`, `AI_MEMORY_TCP_KEEPALIVE_SECS`
+ to tune or `=0` to disable) (#792).
- **Embedding mismatch after a model change**: startup logs a warning
when stored `(provider, model, dim)` triples differ from config.
Hybrid search ignores stale rows until they are re-embedded. Start