Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agent_runtime_protocol): ping runtime websockets and drop dead ones (#5932)

Wolf Mermelstein14c9dd38f33f
diff --git a/crates/agent_harness/src/outbound/runtime_registry.rs b/crates/agent_harness/src/outbound/runtime_registry.rs
index ca70f83673..9f00c15c60 100644
--- a/crates/agent_harness/src/outbound/runtime_registry.rs
+++ b/crates/agent_harness/src/outbound/runtime_registry.rs
@@ -67,10 +67,47 @@ where
{
let connection = RuntimeConnection::connect(carrier);
- if let Some(displaced) = self.connections.insert(bot, connection) {
+ // Listed before it is watched: a socket that dies immediately would
+ // otherwise have its eviction run before the entry it is meant to
+ // remove exists, leaving the dead connection listed for good.
+ if let Some(displaced) = self.connections.insert(bot, Arc::clone(&connection)) {
displaced.evict();
tracing::info!(%bot, "a redialed runtime replaced this bot's connection");
}
+ self.watch_for_close(bot, connection);
+ }
+
+ /// Drop `connection` from the registry once its transport ends.
+ ///
+ /// Without this, a bot whose runtime went away keeps an entry that
+ /// [`RuntimeConnections::bind`] will hand out: sessions attach to a closed
+ /// socket, their handshake writes succeed into nothing, and every prompt
+ /// fails until some later dial happens to displace it. A registry of live
+ /// connections has to stop listing the dead ones.
+ fn watch_for_close(self: &Arc<Self>, bot: BotId, connection: Arc<RuntimeConnection<Sender>>) {
+ tokio::spawn(Arc::clone(self).drop_when_closed(bot, connection));
+ }
+
+ /// The body of that watch: wait for the end, then unlist it.
+ ///
+ /// Separate from the spawn so a test can await the whole thing rather than
+ /// race a task it has no handle on.
+ async fn drop_when_closed(
+ self: Arc<Self>,
+ bot: BotId,
+ connection: Arc<RuntimeConnection<Sender>>,
+ ) {
+ connection.closed().await;
+ // Only if it is still this connection: a redial that already replaced
+ // it owns the entry now, and evicting that would take the live socket
+ // down along with the dead one.
+ let removed = self
+ .connections
+ .remove_if(&bot, |_, current| Arc::ptr_eq(current, &connection))
+ .is_some();
+ if removed {
+ tracing::info!(%bot, "dropped this bot's closed runtime connection");
+ }
}
}
diff --git a/crates/agent_harness/src/outbound/runtime_registry/test.rs b/crates/agent_harness/src/outbound/runtime_registry/test.rs
index cca2f3d8b9..28da30152a 100644
--- a/crates/agent_harness/src/outbound/runtime_registry/test.rs
+++ b/crates/agent_harness/src/outbound/runtime_registry/test.rs
@@ -61,6 +61,60 @@ async fn a_redial_displaces_the_previous_connection() {
);
}
+#[tokio::test]
+async fn a_closed_connection_is_unlisted_so_nothing_binds_onto_it() {
+ let registry = RuntimeRegistry::new();
+ let runtime = ContainerMock::default();
+ registry.attach(BotId::TEST_A, runtime.clone());
+ let connection = registry
+ .connections
+ .get(&BotId::TEST_A)
+ .map(|entry| Arc::clone(&entry))
+ .expect("the dial is listed");
+
+ runtime.disconnects();
+ // The same work `attach` spawns, awaited here rather than raced.
+ Arc::clone(&registry)
+ .drop_when_closed(BotId::TEST_A, connection)
+ .await;
+
+ assert!(!registry.is_connected(BotId::TEST_A));
+ assert!(
+ registry
+ .bind(BotId::TEST_A, AgentSessionId::TEST_A)
+ .await
+ .is_none()
+ );
+}
+
+#[tokio::test]
+async fn a_redials_connection_outlives_the_one_it_displaced() {
+ let registry = RuntimeRegistry::new();
+ let first = ContainerMock::default();
+ registry.attach(BotId::TEST_A, first.clone());
+ let displaced = registry
+ .connections
+ .get(&BotId::TEST_A)
+ .map(|entry| Arc::clone(&entry))
+ .expect("the first dial is listed");
+
+ registry.attach(BotId::TEST_A, ContainerMock::default());
+ first.disconnects();
+ Arc::clone(&registry)
+ .drop_when_closed(BotId::TEST_A, displaced)
+ .await;
+
+ // The dead connection's eviction must not unlist the live one that took
+ // its place, or a redial would leave the bot unreachable.
+ assert!(registry.is_connected(BotId::TEST_A));
+ assert!(
+ registry
+ .bind(BotId::TEST_A, AgentSessionId::TEST_A)
+ .await
+ .is_some()
+ );
+}
+
#[tokio::test]
async fn one_connection_carries_many_sessions() {
let registry = RuntimeRegistry::new();
diff --git a/crates/agent_runtime_protocol/Cargo.toml b/crates/agent_runtime_protocol/Cargo.toml
index 546b170609..5b6f3594dd 100644
--- a/crates/agent_runtime_protocol/Cargo.toml
+++ b/crates/agent_runtime_protocol/Cargo.toml
@@ -26,11 +26,11 @@ specta-serde = { workspace = true }
specta-typescript = { workspace = true }
strum = { workspace = true }
thiserror = { workspace = true }
-tokio = { workspace = true, features = ["rt", "sync"], optional = true }
+tokio = { workspace = true, features = ["rt", "sync", "time"], optional = true }
tokio-tungstenite = { workspace = true, optional = true }
tracing = { workspace = true }
utoipa = { workspace = true, optional = true }
[dev-dependencies]
agent-client-protocol = { workspace = true }
-tokio = { workspace = true, features = ["io-std", "io-util", "macros", "net", "rt", "rt-multi-thread", "time"] }
+tokio = { workspace = true, features = ["io-std", "io-util", "macros", "net", "rt", "rt-multi-thread", "test-util", "time"] }
diff --git a/crates/agent_runtime_protocol/src/outbound/websocket.rs b/crates/agent_runtime_protocol/src/outbound/websocket.rs
index d38fe33c30..2776568ed0 100644
--- a/crates/agent_runtime_protocol/src/outbound/websocket.rs
+++ b/crates/agent_runtime_protocol/src/outbound/websocket.rs
@@ -16,6 +16,7 @@
//! ```
use std::sync::Arc;
+use std::time::Duration;
use axum::Router;
use axum::extract::State;
@@ -28,6 +29,10 @@ use serde::de::DeserializeOwned;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::sync::Mutex as AsyncMutex;
use tokio::sync::mpsc;
+// Tokio's clock rather than the standard library's, so the silence deadline is
+// on the same clock as the interval that checks it - and so a test can drive
+// both without waiting out six real seconds.
+use tokio::time::Instant;
use tokio_tungstenite::WebSocketStream;
use tokio_tungstenite::tungstenite::Message as TungsteniteMessage;
@@ -37,27 +42,54 @@ use crate::domain::ports::{Transport, TransportError, TransportSender};
#[cfg(test)]
mod test;
-/// A physical WebSocket frame type that can carry one JSON text payload.
+/// How often each peer sends a WebSocket ping.
+///
+/// Both ends ping on their own timer rather than one pinging and the other
+/// answering. Keepalive would only need one side - any frame resets an idle
+/// proxy's timer - but *detection* is per-endpoint: the gateway has to notice
+/// a daemon that went away, and a daemon has to notice a gateway that did, so
+/// each end needs traffic it can time out on. Symmetric also means neither
+/// end depends on the other being new enough to ping.
+const PING_INTERVAL: Duration = Duration::from_secs(2);
+
+/// How long a socket may go without any inbound frame before this side
+/// declares it dead.
+///
+/// Three missed pings. A live peer answers every ping with a pong, and a pong
+/// is an inbound frame, so silence this long means the peer is gone or the
+/// path to it is - the case a half-open socket produces, where writes keep
+/// succeeding into a connection that will never answer again.
+const SILENCE_TIMEOUT: Duration = Duration::from_secs(6);
+
+/// A physical WebSocket frame type: one JSON text payload, or a ping.
///
/// Implemented for both `axum`'s and `tokio-tungstenite`'s message types so
/// the pump loop below can drive either a server-accepted or a
/// client-dialed socket without caring which library it came from.
-trait TextFrame: Sized {
+trait WireFrame: Sized {
/// Wrap a JSON payload as a text frame.
fn text(payload: String) -> Self;
+ /// An empty ping frame.
+ fn ping() -> Self;
+
/// Extract this frame's text payload, if it has one.
///
/// Non-text frames (ping/pong/binary/close) are not part of the logical
- /// protocol and are silently skipped by the pump loop.
+ /// protocol and are silently skipped by the pump loop - but they still
+ /// count as liveness, which is the whole point of the pings.
fn into_text(self) -> Option<String>;
}
-impl TextFrame for AxumMessage {
+impl WireFrame for AxumMessage {
fn text(payload: String) -> Self {
Self::Text(payload.into())
}
+ fn ping() -> Self {
+ Self::Ping(Default::default())
+ }
+
fn into_text(self) -> Option<String> {
match self {
Self::Text(text) => Some(text.to_string()),
@@ -66,11 +98,15 @@ impl TextFrame for AxumMessage {
}
}
-impl TextFrame for TungsteniteMessage {
+impl WireFrame for TungsteniteMessage {
fn text(payload: String) -> Self {
Self::Text(payload.into())
}
+ fn ping() -> Self {
+ Self::Ping(Default::default())
+ }
+
fn into_text(self) -> Option<String> {
match self {
Self::Text(text) => Some(text.to_string()),
@@ -92,10 +128,15 @@ async fn run_pump<Rx, Socket, Frame, Error>(
incoming: mpsc::UnboundedSender<Rx>,
) where
Rx: DeserializeOwned,
- Frame: TextFrame,
+ Frame: WireFrame,
Socket: Sink<Frame, Error = Error> + Stream<Item = Result<Frame, Error>> + Unpin,
{
let (mut sink, mut stream) = socket.split();
+ let mut ping = tokio::time::interval(PING_INTERVAL);
+ // A pump that was busy longer than one tick owes one ping, not a burst of
+ // however many it slept through.
+ ping.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ let mut last_inbound = Instant::now();
loop {
tokio::select! {
next_outgoing = outgoing.recv() => {
@@ -105,6 +146,9 @@ async fn run_pump<Rx, Socket, Frame, Error>(
}
}
next_incoming = stream.next() => {
+ // Every frame counts, text or not: a pong carries no logical
+ // message and is exactly what proves the peer is still there.
+ last_inbound = Instant::now();
match next_incoming {
Some(Ok(frame)) => {
let Some(text) = frame.into_text() else { continue };
@@ -122,6 +166,21 @@ async fn run_pump<Rx, Socket, Frame, Error>(
Some(Err(_)) | None => break,
}
}
+ _ = ping.tick() => {
+ let silent_for = last_inbound.elapsed();
+ if silent_for >= SILENCE_TIMEOUT {
+ tracing::warn!(
+ silent_for_ms = silent_for.as_millis(),
+ "closing a WebSocket that stopped answering pings"
+ );
+ break;
+ }
+ // Both halves of a split socket share one connection, so this
+ // is also what flushes any pong the read half queued.
+ if sink.send(Frame::ping()).await.is_err() {
+ break;
+ }
+ }
}
}
}
diff --git a/crates/agent_runtime_protocol/src/outbound/websocket/test.rs b/crates/agent_runtime_protocol/src/outbound/websocket/test.rs
index 39ffb3872c..9d5465aead 100644
--- a/crates/agent_runtime_protocol/src/outbound/websocket/test.rs
+++ b/crates/agent_runtime_protocol/src/outbound/websocket/test.rs
@@ -241,3 +241,90 @@ async fn role_connections_run_over_a_runtime_initiated_websocket() {
server_handle.abort();
}
+
+/// A socket that accepts everything written to it and never says anything
+/// back: the half-open connection this pump's pings exist to catch, where
+/// writes keep succeeding into a peer that will never answer again.
+struct StalledSocket {
+ sent: futures::channel::mpsc::UnboundedSender<TungsteniteMessage>,
+}
+
+impl futures::Sink<TungsteniteMessage> for StalledSocket {
+ type Error = futures::channel::mpsc::SendError;
+
+ fn poll_ready(
+ self: std::pin::Pin<&mut Self>,
+ context: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Result<(), Self::Error>> {
+ std::pin::Pin::new(&mut self.get_mut().sent).poll_ready(context)
+ }
+
+ fn start_send(
+ self: std::pin::Pin<&mut Self>,
+ item: TungsteniteMessage,
+ ) -> Result<(), Self::Error> {
+ std::pin::Pin::new(&mut self.get_mut().sent).start_send(item)
+ }
+
+ fn poll_flush(
+ self: std::pin::Pin<&mut Self>,
+ context: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Result<(), Self::Error>> {
+ std::pin::Pin::new(&mut self.get_mut().sent).poll_flush(context)
+ }
+
+ fn poll_close(
+ self: std::pin::Pin<&mut Self>,
+ context: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Result<(), Self::Error>> {
+ std::pin::Pin::new(&mut self.get_mut().sent).poll_close(context)
+ }
+}
+
+impl futures::Stream for StalledSocket {
+ type Item = Result<TungsteniteMessage, futures::channel::mpsc::SendError>;
+
+ fn poll_next(
+ self: std::pin::Pin<&mut Self>,
+ _context: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Option<Self::Item>> {
+ std::task::Poll::Pending
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn an_idle_socket_is_pinged_rather_than_left_to_rot() {
+ let (sent, mut frames) = futures::channel::mpsc::unbounded();
+ let (_outgoing, outgoing_rx) = mpsc::unbounded_channel();
+ let (incoming, _incoming_rx) = mpsc::unbounded_channel::<ToServerMessage>();
+ tokio::spawn(run_pump(StalledSocket { sent }, outgoing_rx, incoming));
+
+ // Nothing has been asked of this connection, which is exactly when a
+ // dead one used to go unnoticed.
+ tokio::time::sleep(PING_INTERVAL + Duration::from_millis(100)).await;
+
+ let first = frames.next().await.expect("the pump should ping");
+ assert!(
+ matches!(first, TungsteniteMessage::Ping(_)),
+ "an idle pump's first frame should be a ping, got {first:?}"
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_socket_that_stops_answering_pings_is_closed() {
+ let (sent, _frames) = futures::channel::mpsc::unbounded();
+ let (_outgoing, outgoing_rx) = mpsc::unbounded_channel();
+ let (incoming, mut incoming_rx) = mpsc::unbounded_channel::<ToServerMessage>();
+ tokio::spawn(run_pump(StalledSocket { sent }, outgoing_rx, incoming));
+
+ // The pump dropping its sender is what every layer above reads as the
+ // connection ending, so awaiting that is awaiting the teardown itself -
+ // no polling for a state change.
+ let ended = timeout(SILENCE_TIMEOUT * 2, incoming_rx.recv())
+ .await
+ .expect("a silent socket should be given up on, not waited on forever");
+ assert!(
+ ended.is_none(),
+ "the pump should close its inbound channel when the peer goes quiet"
+ );
+}
diff --git a/crates/agent_session/src/domain/connection.rs b/crates/agent_session/src/domain/connection.rs
index 2d8e578564..a91e20354f 100644
--- a/crates/agent_session/src/domain/connection.rs
+++ b/crates/agent_session/src/domain/connection.rs
@@ -32,6 +32,7 @@ use agent_runtime_protocol::domain::schema::v0::{
};
use dashmap::DashMap;
use tokio::sync::{Mutex, mpsc, watch};
+use tokio_util::sync::CancellationToken;
use crate::domain::model::AgentSessionId;
use crate::domain::session::HandshakeStatus;
@@ -171,6 +172,13 @@ pub struct RuntimeConnection<Sender> {
bound: Bound,
routes: Mutex<Routes>,
router: OnceLock<tokio::task::AbortHandle>,
+ /// Cancelled once this connection's transport has ended.
+ ///
+ /// A token rather than a notification because whoever waits on it mostly
+ /// arrives late: a connection can die before anybody asks, and
+ /// [`CancellationToken::cancelled`] on an already-cancelled token returns
+ /// at once, where a missed notification would wait forever.
+ closed: CancellationToken,
}
impl<Sender> RuntimeConnection<Sender>
@@ -195,6 +203,7 @@ where
bound: DashMap::new(),
routes: Mutex::new(Routes::default()),
router: OnceLock::new(),
+ closed: CancellationToken::new(),
});
let router = tokio::spawn(Arc::clone(&connection).route_inbound(inbound));
let _ = connection.router.set(router.abort_handle());
@@ -207,6 +216,19 @@ where
if let Some(router) = self.router.get() {
router.abort();
}
+ // An aborted router will never reach the end of `route_inbound`, so
+ // the signal is raised here too: whoever is waiting for this
+ // connection to be over is waiting for either way of it ending.
+ self.closed.cancel();
+ }
+
+ /// Resolves once this connection's transport has ended, however it ended.
+ ///
+ /// What a registry holding connections needs in order to stop handing out
+ /// one that is closed: a socket dies with nothing to announce it, so the
+ /// holder has to be told rather than discover it on the next send.
+ pub async fn closed(&self) {
+ self.closed.cancelled().await;
}
/// The gate every session on this connection shares.
@@ -311,6 +333,7 @@ where
}
}
self.bound.clear();
+ self.closed.cancel();
}
async fn deliver(&self, session: AgentSessionId, message: ToServerMessage) {