Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(agent_runtime_protocol): ping runtime websockets and drop dead ones (#5932)
Wolf Mermelstein14c9dd38f33f
diff --git a/crates/agent_harness/src/outbound/runtime_registry.rs b/crates/agent_harness/src/outbound/runtime_registry.rsindex ca70f83673..9f00c15c60 100644--- a/crates/agent_harness/src/outbound/runtime_registry.rs+++ b/crates/agent_harness/src/outbound/runtime_registry.rs@@ -67,10 +67,47 @@ where{let connection = RuntimeConnection::connect(carrier);- if let Some(displaced) = self.connections.insert(bot, connection) {+ // Listed before it is watched: a socket that dies immediately would+ // otherwise have its eviction run before the entry it is meant to+ // remove exists, leaving the dead connection listed for good.+ if let Some(displaced) = self.connections.insert(bot, Arc::clone(&connection)) {displaced.evict();tracing::info!(%bot, "a redialed runtime replaced this bot's connection");}+ self.watch_for_close(bot, connection);+ }++ /// Drop `connection` from the registry once its transport ends.+ ///+ /// Without this, a bot whose runtime went away keeps an entry that+ /// [`RuntimeConnections::bind`] will hand out: sessions attach to a closed+ /// socket, their handshake writes succeed into nothing, and every prompt+ /// fails until some later dial happens to displace it. A registry of live+ /// connections has to stop listing the dead ones.+ fn watch_for_close(self: &Arc<Self>, bot: BotId, connection: Arc<RuntimeConnection<Sender>>) {+ tokio::spawn(Arc::clone(self).drop_when_closed(bot, connection));+ }++ /// The body of that watch: wait for the end, then unlist it.+ ///+ /// Separate from the spawn so a test can await the whole thing rather than+ /// race a task it has no handle on.+ async fn drop_when_closed(+ self: Arc<Self>,+ bot: BotId,+ connection: Arc<RuntimeConnection<Sender>>,+ ) {+ connection.closed().await;+ // Only if it is still this connection: a redial that already replaced+ // it owns the entry now, and evicting that would take the live socket+ // down along with the dead one.+ let removed = self+ .connections+ .remove_if(&bot, |_, current| Arc::ptr_eq(current, &connection))+ .is_some();+ if removed {+ tracing::info!(%bot, "dropped this bot's closed runtime connection");+ }}}diff --git a/crates/agent_harness/src/outbound/runtime_registry/test.rs b/crates/agent_harness/src/outbound/runtime_registry/test.rsindex cca2f3d8b9..28da30152a 100644--- a/crates/agent_harness/src/outbound/runtime_registry/test.rs+++ b/crates/agent_harness/src/outbound/runtime_registry/test.rs@@ -61,6 +61,60 @@ async fn a_redial_displaces_the_previous_connection() {);}+#[tokio::test]+async fn a_closed_connection_is_unlisted_so_nothing_binds_onto_it() {+ let registry = RuntimeRegistry::new();+ let runtime = ContainerMock::default();+ registry.attach(BotId::TEST_A, runtime.clone());+ let connection = registry+ .connections+ .get(&BotId::TEST_A)+ .map(|entry| Arc::clone(&entry))+ .expect("the dial is listed");++ runtime.disconnects();+ // The same work `attach` spawns, awaited here rather than raced.+ Arc::clone(®istry)+ .drop_when_closed(BotId::TEST_A, connection)+ .await;++ assert!(!registry.is_connected(BotId::TEST_A));+ assert!(+ registry+ .bind(BotId::TEST_A, AgentSessionId::TEST_A)+ .await+ .is_none()+ );+}++#[tokio::test]+async fn a_redials_connection_outlives_the_one_it_displaced() {+ let registry = RuntimeRegistry::new();+ let first = ContainerMock::default();+ registry.attach(BotId::TEST_A, first.clone());+ let displaced = registry+ .connections+ .get(&BotId::TEST_A)+ .map(|entry| Arc::clone(&entry))+ .expect("the first dial is listed");++ registry.attach(BotId::TEST_A, ContainerMock::default());+ first.disconnects();+ Arc::clone(®istry)+ .drop_when_closed(BotId::TEST_A, displaced)+ .await;++ // The dead connection's eviction must not unlist the live one that took+ // its place, or a redial would leave the bot unreachable.+ assert!(registry.is_connected(BotId::TEST_A));+ assert!(+ registry+ .bind(BotId::TEST_A, AgentSessionId::TEST_A)+ .await+ .is_some()+ );+}+#[tokio::test]async fn one_connection_carries_many_sessions() {let registry = RuntimeRegistry::new();diff --git a/crates/agent_runtime_protocol/Cargo.toml b/crates/agent_runtime_protocol/Cargo.tomlindex 546b170609..5b6f3594dd 100644--- a/crates/agent_runtime_protocol/Cargo.toml+++ b/crates/agent_runtime_protocol/Cargo.toml@@ -26,11 +26,11 @@ specta-serde = { workspace = true }specta-typescript = { workspace = true }strum = { workspace = true }thiserror = { workspace = true }-tokio = { workspace = true, features = ["rt", "sync"], optional = true }+tokio = { workspace = true, features = ["rt", "sync", "time"], optional = true }tokio-tungstenite = { workspace = true, optional = true }tracing = { workspace = true }utoipa = { workspace = true, optional = true }[dev-dependencies]agent-client-protocol = { workspace = true }-tokio = { workspace = true, features = ["io-std", "io-util", "macros", "net", "rt", "rt-multi-thread", "time"] }+tokio = { workspace = true, features = ["io-std", "io-util", "macros", "net", "rt", "rt-multi-thread", "test-util", "time"] }diff --git a/crates/agent_runtime_protocol/src/outbound/websocket.rs b/crates/agent_runtime_protocol/src/outbound/websocket.rsindex d38fe33c30..2776568ed0 100644--- a/crates/agent_runtime_protocol/src/outbound/websocket.rs+++ b/crates/agent_runtime_protocol/src/outbound/websocket.rs@@ -16,6 +16,7 @@//! ```use std::sync::Arc;+use std::time::Duration;use axum::Router;use axum::extract::State;@@ -28,6 +29,10 @@ use serde::de::DeserializeOwned;use tokio::io::{AsyncRead, AsyncWrite};use tokio::sync::Mutex as AsyncMutex;use tokio::sync::mpsc;+// Tokio's clock rather than the standard library's, so the silence deadline is+// on the same clock as the interval that checks it - and so a test can drive+// both without waiting out six real seconds.+use tokio::time::Instant;use tokio_tungstenite::WebSocketStream;use tokio_tungstenite::tungstenite::Message as TungsteniteMessage;@@ -37,27 +42,54 @@ use crate::domain::ports::{Transport, TransportError, TransportSender};#[cfg(test)]mod test;-/// A physical WebSocket frame type that can carry one JSON text payload.+/// How often each peer sends a WebSocket ping.+///+/// Both ends ping on their own timer rather than one pinging and the other+/// answering. Keepalive would only need one side - any frame resets an idle+/// proxy's timer - but *detection* is per-endpoint: the gateway has to notice+/// a daemon that went away, and a daemon has to notice a gateway that did, so+/// each end needs traffic it can time out on. Symmetric also means neither+/// end depends on the other being new enough to ping.+const PING_INTERVAL: Duration = Duration::from_secs(2);++/// How long a socket may go without any inbound frame before this side+/// declares it dead.+///+/// Three missed pings. A live peer answers every ping with a pong, and a pong+/// is an inbound frame, so silence this long means the peer is gone or the+/// path to it is - the case a half-open socket produces, where writes keep+/// succeeding into a connection that will never answer again.+const SILENCE_TIMEOUT: Duration = Duration::from_secs(6);++/// A physical WebSocket frame type: one JSON text payload, or a ping.////// Implemented for both `axum`'s and `tokio-tungstenite`'s message types so/// the pump loop below can drive either a server-accepted or a/// client-dialed socket without caring which library it came from.-trait TextFrame: Sized {+trait WireFrame: Sized {/// Wrap a JSON payload as a text frame.fn text(payload: String) -> Self;+ /// An empty ping frame.+ fn ping() -> Self;+/// Extract this frame's text payload, if it has one.////// Non-text frames (ping/pong/binary/close) are not part of the logical- /// protocol and are silently skipped by the pump loop.+ /// protocol and are silently skipped by the pump loop - but they still+ /// count as liveness, which is the whole point of the pings.fn into_text(self) -> Option<String>;}-impl TextFrame for AxumMessage {+impl WireFrame for AxumMessage {fn text(payload: String) -> Self {Self::Text(payload.into())}+ fn ping() -> Self {+ Self::Ping(Default::default())+ }+fn into_text(self) -> Option<String> {match self {Self::Text(text) => Some(text.to_string()),@@ -66,11 +98,15 @@ impl TextFrame for AxumMessage {}}-impl TextFrame for TungsteniteMessage {+impl WireFrame for TungsteniteMessage {fn text(payload: String) -> Self {Self::Text(payload.into())}+ fn ping() -> Self {+ Self::Ping(Default::default())+ }+fn into_text(self) -> Option<String> {match self {Self::Text(text) => Some(text.to_string()),@@ -92,10 +128,15 @@ async fn run_pump<Rx, Socket, Frame, Error>(incoming: mpsc::UnboundedSender<Rx>,) whereRx: DeserializeOwned,- Frame: TextFrame,+ Frame: WireFrame,Socket: Sink<Frame, Error = Error> + Stream<Item = Result<Frame, Error>> + Unpin,{let (mut sink, mut stream) = socket.split();+ let mut ping = tokio::time::interval(PING_INTERVAL);+ // A pump that was busy longer than one tick owes one ping, not a burst of+ // however many it slept through.+ ping.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);+ let mut last_inbound = Instant::now();loop {tokio::select! {next_outgoing = outgoing.recv() => {@@ -105,6 +146,9 @@ async fn run_pump<Rx, Socket, Frame, Error>(}}next_incoming = stream.next() => {+ // Every frame counts, text or not: a pong carries no logical+ // message and is exactly what proves the peer is still there.+ last_inbound = Instant::now();match next_incoming {Some(Ok(frame)) => {let Some(text) = frame.into_text() else { continue };@@ -122,6 +166,21 @@ async fn run_pump<Rx, Socket, Frame, Error>(Some(Err(_)) | None => break,}}+ _ = ping.tick() => {+ let silent_for = last_inbound.elapsed();+ if silent_for >= SILENCE_TIMEOUT {+ tracing::warn!(+ silent_for_ms = silent_for.as_millis(),+ "closing a WebSocket that stopped answering pings"+ );+ break;+ }+ // Both halves of a split socket share one connection, so this+ // is also what flushes any pong the read half queued.+ if sink.send(Frame::ping()).await.is_err() {+ break;+ }+ }}}}diff --git a/crates/agent_runtime_protocol/src/outbound/websocket/test.rs b/crates/agent_runtime_protocol/src/outbound/websocket/test.rsindex 39ffb3872c..9d5465aead 100644--- a/crates/agent_runtime_protocol/src/outbound/websocket/test.rs+++ b/crates/agent_runtime_protocol/src/outbound/websocket/test.rs@@ -241,3 +241,90 @@ async fn role_connections_run_over_a_runtime_initiated_websocket() {server_handle.abort();}++/// A socket that accepts everything written to it and never says anything+/// back: the half-open connection this pump's pings exist to catch, where+/// writes keep succeeding into a peer that will never answer again.+struct StalledSocket {+ sent: futures::channel::mpsc::UnboundedSender<TungsteniteMessage>,+}++impl futures::Sink<TungsteniteMessage> for StalledSocket {+ type Error = futures::channel::mpsc::SendError;++ fn poll_ready(+ self: std::pin::Pin<&mut Self>,+ context: &mut std::task::Context<'_>,+ ) -> std::task::Poll<Result<(), Self::Error>> {+ std::pin::Pin::new(&mut self.get_mut().sent).poll_ready(context)+ }++ fn start_send(+ self: std::pin::Pin<&mut Self>,+ item: TungsteniteMessage,+ ) -> Result<(), Self::Error> {+ std::pin::Pin::new(&mut self.get_mut().sent).start_send(item)+ }++ fn poll_flush(+ self: std::pin::Pin<&mut Self>,+ context: &mut std::task::Context<'_>,+ ) -> std::task::Poll<Result<(), Self::Error>> {+ std::pin::Pin::new(&mut self.get_mut().sent).poll_flush(context)+ }++ fn poll_close(+ self: std::pin::Pin<&mut Self>,+ context: &mut std::task::Context<'_>,+ ) -> std::task::Poll<Result<(), Self::Error>> {+ std::pin::Pin::new(&mut self.get_mut().sent).poll_close(context)+ }+}++impl futures::Stream for StalledSocket {+ type Item = Result<TungsteniteMessage, futures::channel::mpsc::SendError>;++ fn poll_next(+ self: std::pin::Pin<&mut Self>,+ _context: &mut std::task::Context<'_>,+ ) -> std::task::Poll<Option<Self::Item>> {+ std::task::Poll::Pending+ }+}++#[tokio::test(start_paused = true)]+async fn an_idle_socket_is_pinged_rather_than_left_to_rot() {+ let (sent, mut frames) = futures::channel::mpsc::unbounded();+ let (_outgoing, outgoing_rx) = mpsc::unbounded_channel();+ let (incoming, _incoming_rx) = mpsc::unbounded_channel::<ToServerMessage>();+ tokio::spawn(run_pump(StalledSocket { sent }, outgoing_rx, incoming));++ // Nothing has been asked of this connection, which is exactly when a+ // dead one used to go unnoticed.+ tokio::time::sleep(PING_INTERVAL + Duration::from_millis(100)).await;++ let first = frames.next().await.expect("the pump should ping");+ assert!(+ matches!(first, TungsteniteMessage::Ping(_)),+ "an idle pump's first frame should be a ping, got {first:?}"+ );+}++#[tokio::test(start_paused = true)]+async fn a_socket_that_stops_answering_pings_is_closed() {+ let (sent, _frames) = futures::channel::mpsc::unbounded();+ let (_outgoing, outgoing_rx) = mpsc::unbounded_channel();+ let (incoming, mut incoming_rx) = mpsc::unbounded_channel::<ToServerMessage>();+ tokio::spawn(run_pump(StalledSocket { sent }, outgoing_rx, incoming));++ // The pump dropping its sender is what every layer above reads as the+ // connection ending, so awaiting that is awaiting the teardown itself -+ // no polling for a state change.+ let ended = timeout(SILENCE_TIMEOUT * 2, incoming_rx.recv())+ .await+ .expect("a silent socket should be given up on, not waited on forever");+ assert!(+ ended.is_none(),+ "the pump should close its inbound channel when the peer goes quiet"+ );+}diff --git a/crates/agent_session/src/domain/connection.rs b/crates/agent_session/src/domain/connection.rsindex 2d8e578564..a91e20354f 100644--- a/crates/agent_session/src/domain/connection.rs+++ b/crates/agent_session/src/domain/connection.rs@@ -32,6 +32,7 @@ use agent_runtime_protocol::domain::schema::v0::{};use dashmap::DashMap;use tokio::sync::{Mutex, mpsc, watch};+use tokio_util::sync::CancellationToken;use crate::domain::model::AgentSessionId;use crate::domain::session::HandshakeStatus;@@ -171,6 +172,13 @@ pub struct RuntimeConnection<Sender> {bound: Bound,routes: Mutex<Routes>,router: OnceLock<tokio::task::AbortHandle>,+ /// Cancelled once this connection's transport has ended.+ ///+ /// A token rather than a notification because whoever waits on it mostly+ /// arrives late: a connection can die before anybody asks, and+ /// [`CancellationToken::cancelled`] on an already-cancelled token returns+ /// at once, where a missed notification would wait forever.+ closed: CancellationToken,}impl<Sender> RuntimeConnection<Sender>@@ -195,6 +203,7 @@ wherebound: DashMap::new(),routes: Mutex::new(Routes::default()),router: OnceLock::new(),+ closed: CancellationToken::new(),});let router = tokio::spawn(Arc::clone(&connection).route_inbound(inbound));let _ = connection.router.set(router.abort_handle());@@ -207,6 +216,19 @@ whereif let Some(router) = self.router.get() {router.abort();}+ // An aborted router will never reach the end of `route_inbound`, so+ // the signal is raised here too: whoever is waiting for this+ // connection to be over is waiting for either way of it ending.+ self.closed.cancel();+ }++ /// Resolves once this connection's transport has ended, however it ended.+ ///+ /// What a registry holding connections needs in order to stop handing out+ /// one that is closed: a socket dies with nothing to announce it, so the+ /// holder has to be told rather than discover it on the next send.+ pub async fn closed(&self) {+ self.closed.cancelled().await;}/// The gate every session on this connection shares.@@ -311,6 +333,7 @@ where}}self.bound.clear();+ self.closed.cancel();}async fn deliver(&self, session: AgentSessionId, message: ToServerMessage) {