Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(observability): trace agent session stalls (#5982)
Wolf Mermelstein4528ef86ad8f
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex 8a18c94cb7..2092348322 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -390,6 +390,8 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {"agent.session.command",agent.session.id = %id,agent.action.name = action.as_ref(),+ agent.command.queue_wait_ms = tracing::field::Empty,+ agent.session.runtime_phase_at_dequeue = tracing::field::Empty,otel.status_code = tracing::field::Empty,otel.status_description = tracing::field::Empty,);@@ -400,6 +402,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {action_id,completed,span,+ enqueued_at: tokio::time::Instant::now(),}).await.is_err()@@ -908,6 +911,12 @@ whereRt: AgentSessionRealtime,{/// Push the frame just appended out to whoever is watching the session.+ #[tracing::instrument(+ name = "agent.session.realtime.publish",+ err,+ skip(self, agent_session_id, entry),+ fields(agent.session.id = %agent_session_id)+ )]async fn stream(&mut self,agent_session_id: AgentSessionId,diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rsindex 73e0237e97..04b458c263 100644--- a/crates/agent_session/src/domain/service/test.rs+++ b/crates/agent_session/src/domain/service/test.rs@@ -663,6 +663,7 @@ async fn cancellation_does_not_drop_an_effect_batch_after_machine_mutation() {action_id: AgentActionId::mint(),completed,span: tracing::info_span!("test.command"),+ enqueued_at: tokio::time::Instant::now(),}).await.unwrap();@@ -737,6 +738,7 @@ async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {action_id: AgentActionId::mint(),completed,span: tracing::info_span!("test.command"),+ enqueued_at: tokio::time::Instant::now(),}).await.unwrap();diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rsindex d26df5f32f..d199785f16 100644--- a/crates/agent_session/src/domain/session/actors.rs+++ b/crates/agent_session/src/domain/session/actors.rs@@ -45,6 +45,7 @@ pub(crate) struct SessionCommand {pub(crate) action_id: AgentActionId,pub(crate) completed: oneshot::Sender<Result<()>>,pub(crate) span: tracing::Span,+ pub(crate) enqueued_at: Instant,}pub(crate) struct SessionCompletion {@@ -156,11 +157,21 @@ wherelet input = tokio::select! {() = handshake_timeout => Input::Closed(CloseReason::HandshakeTimedOut),command = self.commands.recv() => match command {- Some(SessionCommand { user_id, action, action_id, completed, span }) => Input::Command {- from: user_id,- action,- action_id,- token: SessionCompletion { completed, span },+ Some(SessionCommand { user_id, action, action_id, completed, span, enqueued_at }) => {+ span.record(+ "agent.command.queue_wait_ms",+ enqueued_at.elapsed().as_millis() as u64,+ );+ span.record(+ "agent.session.runtime_phase_at_dequeue",+ self.machine.status().as_ref(),+ );+ Input::Command {+ from: user_id,+ action,+ action_id,+ token: SessionCompletion { completed, span },+ }},// The service dropped every handle; nobody can reach us.None => Input::Closed(CloseReason::Abandoned),diff --git a/crates/agent_session/src/domain/session/types.rs b/crates/agent_session/src/domain/session/types.rsindex 179376688b..ef3a3302ef 100644--- a/crates/agent_session/src/domain/session/types.rs+++ b/crates/agent_session/src/domain/session/types.rs@@ -79,7 +79,8 @@ pub enum HandshakeStatus {}/// Observable phase of a session connection.-#[derive(Debug, Clone, PartialEq, Eq)]+#[derive(Debug, Clone, PartialEq, Eq, strum::AsRefStr)]+#[strum(serialize_all = "snake_case")]pub enum RuntimeStatus {/// Waiting for the runtime to report its agent ready.Booting,diff --git a/services/connection_gateway/src/api/connection.rs b/services/connection_gateway/src/api/connection.rsindex 797a4b2b29..d113de0dc4 100644--- a/services/connection_gateway/src/api/connection.rs+++ b/services/connection_gateway/src/api/connection.rs@@ -1,5 +1,6 @@use crate::{config::Config,+ constants::SLOW_WEBSOCKET_OPERATION_THRESHOLD,context::{ApiContext, AppState, AuthorizationService},model::{connection::ConnectionContext,@@ -76,7 +77,8 @@ async fn handle_websocket_connection(// Create guard that records last online time when websocket connection closeslet last_online_guard = ctx.last_online_worker.new_guard(macro_user_id.clone());- let sender_task = tokio::spawn(async move { forwarder(sink, receiver).await });+ let sender_connection_id = connection_id.clone();+ let sender_task = tokio::spawn(forwarder(sink, receiver, sender_connection_id));if let Err(err) = ctx.connection_manager@@ -136,17 +138,24 @@ async fn handle_websocket_connection(/// Forwards messages from a [Receiver] to a [SplitSink]/// This is useful as [SplitSink] does not implement [Clone]-pub async fn forwarder(+async fn forwarder(mut sink: SplitSink<WebSocket, AxumWebsocketMessage>,mut receiver: Receiver<OutgoingMessage>,+ connection_id: String,) -> Result<()> {while let Some(mut message) = receiver.recv().await {+ let queue_depth = receiver.len();+ let queue_max_capacity = receiver.max_capacity();let span = match &message {OutgoingMessage::Message(message) => {let span = tracing::info_span!("connection_gateway.websocket_send",otel.kind = "producer",message_type = %message.message_type,+ connection.id = %connection_id,+ connection.queue.observed_depth = queue_depth,+ connection.queue.max_capacity = queue_max_capacity,+ websocket.write_ms = tracing::field::Empty,otel.status_code = tracing::field::Empty,otel.status_description = tracing::field::Empty,);@@ -162,7 +171,28 @@ pub async fn forwarder(}if let Ok(msg) = message.try_into() {- let result = sink.send(msg).instrument(span.clone()).await;+ let write_span = span.clone();+ let result = async {+ let started = tokio::time::Instant::now();+ let mut send = std::pin::pin!(sink.send(msg));+ let result = tokio::select! {+ result = &mut send => result,+ () = tokio::time::sleep(SLOW_WEBSOCKET_OPERATION_THRESHOLD) => {+ tracing::warn!(+ connection.id = %connection_id,+ connection.queue.observed_depth = receiver.len(),+ connection.queue.max_capacity = queue_max_capacity,+ "websocket write is blocked"+ );+ send.await+ }+ };+ (result, started.elapsed())+ }+ .instrument(write_span)+ .await;+ let (result, write) = result;+ span.record("websocket.write_ms", write.as_millis() as u64);if let Err(err) = result {record_span_error(&span, &err);tracing::warn!(diff --git a/services/connection_gateway/src/constants.rs b/services/connection_gateway/src/constants.rsindex 59788dcd84..dc4d1d9f97 100644--- a/services/connection_gateway/src/constants.rs+++ b/services/connection_gateway/src/constants.rs@@ -1,4 +1,5 @@use axum::http::HeaderValue;+use std::time::Duration;pub const ORIGINS: [HeaderValue; 23] = [HeaderValue::from_static("http://localhost:3000"),@@ -28,3 +29,6 @@ pub const ORIGINS: [HeaderValue; 23] = [/// The default timeout threshold is 1 minutepub const DEFAULT_TIMEOUT_THRESHOLD: u64 = 60_000;++/// Emit diagnostics while a websocket queue or write remains blocked this long.+pub(crate) const SLOW_WEBSOCKET_OPERATION_THRESHOLD: Duration = Duration::from_secs(1);diff --git a/services/connection_gateway/src/service/connection.rs b/services/connection_gateway/src/service/connection.rsindex be17db3195..6cf49a7e01 100644--- a/services/connection_gateway/src/service/connection.rs+++ b/services/connection_gateway/src/service/connection.rs@@ -1,3 +1,4 @@+use crate::constants::SLOW_WEBSOCKET_OPERATION_THRESHOLD;use crate::model::{connection::{Connection, StoredConnectionEntity},message::{Message, OutgoingMessage},@@ -13,6 +14,9 @@ use std::{};use tokio::{sync::mpsc::Sender, task::AbortHandle};+#[cfg(test)]+mod test;+#[async_trait]pub trait ConnectionRepo: Send + Sync {/// Inserts a new [StoredConnectionEntity] into storage@@ -199,13 +203,49 @@ impl ConnectionManager {/// Sends a message to a a connection/// If the connection is not found, or dropped, then we remove the connection id/// from the map, and should kill (TODO:) any remaining tasks associated with the connection+ #[tracing::instrument(+ name = "connection_gateway.queue_send",+ err,+ skip(self, id, message),+ fields(+ connection.id = id,+ connection.queue.observed_depth = tracing::field::Empty,+ connection.queue.max_capacity = tracing::field::Empty,+ connection.queue.wait_ms = tracing::field::Empty,+ )+ )]pub async fn send_message(&self, id: &str, message: Message) -> Result<()> {let sender = match self.connections.get(id) {Some(connection) => connection.sender.clone(),None => anyhow::bail!("connection not found"),};- if let Err(err) = sender.send(OutgoingMessage::Message(message)).await {+ let remaining_capacity = sender.capacity();+ let max_capacity = sender.max_capacity();+ let depth = max_capacity.saturating_sub(remaining_capacity);+ let span = tracing::Span::current();+ span.record("connection.queue.observed_depth", depth as u64);+ span.record("connection.queue.max_capacity", max_capacity as u64);++ let started = tokio::time::Instant::now();+ let mut send = std::pin::pin!(sender.send(OutgoingMessage::Message(message)));+ let result = tokio::select! {+ result = &mut send => result,+ () = tokio::time::sleep(SLOW_WEBSOCKET_OPERATION_THRESHOLD) => {+ let depth = max_capacity.saturating_sub(sender.capacity());+ tracing::warn!(+ connection.id = id,+ connection.queue.observed_depth = depth,+ connection.queue.max_capacity = max_capacity,+ "websocket outbound queue send is blocked"+ );+ send.await+ }+ };+ let wait = started.elapsed();+ span.record("connection.queue.wait_ms", wait.as_millis() as u64);++ if let Err(err) = result {self.remove_connection(id).await?;anyhow::bail!("failed to send message: {}", err);}diff --git a/services/connection_gateway/src/service/connection/test.rs b/services/connection_gateway/src/service/connection/test.rsnew file mode 100644index 0000000000..184167e8d9--- /dev/null+++ b/services/connection_gateway/src/service/connection/test.rs@@ -0,0 +1,105 @@+use super::*;++struct UnusedRepo;++#[async_trait]+impl ConnectionRepo for UnusedRepo {+ async fn insert_connection_entry(+ &self,+ _connection: UserEntityConnection<'_>,+ ) -> anyhow::Result<StoredConnectionEntity> {+ unimplemented!()+ }++ async fn get_entries_by_entity(+ &self,+ _entity: &Entity<'_>,+ ) -> anyhow::Result<Vec<StoredConnectionEntity>> {+ unimplemented!()+ }++ async fn get_entries_by_connection_id(+ &self,+ _connection_id: &str,+ ) -> anyhow::Result<Vec<StoredConnectionEntity>> {+ unimplemented!()+ }++ async fn get_connection(&self, _connection_id: &str) -> anyhow::Result<StoredConnectionEntity> {+ unimplemented!()+ }++ async fn get_entry_for_connection_entity(+ &self,+ _entity: EntityConnection<'_>,+ ) -> anyhow::Result<Option<StoredConnectionEntity>> {+ unimplemented!()+ }++ async fn remove_all_entries_for_by_connection_id(+ &self,+ _connection_id: &str,+ ) -> anyhow::Result<()> {+ Ok(())+ }++ async fn remove_entity(&self, _entity: &EntityConnection<'_>) -> anyhow::Result<()> {+ unimplemented!()+ }++ async fn update_last_entity_ping(+ &self,+ _entity: &EntityConnection<'_>,+ _timestamp: u64,+ ) -> anyhow::Result<StoredConnectionEntity> {+ unimplemented!()+ }++ async fn update_user_connection_last_ping(+ &self,+ _connection_id: &str,+ _user: &str,+ _timestamp: u64,+ ) -> anyhow::Result<()> {+ unimplemented!()+ }+}++#[tokio::test(start_paused = true)]+async fn a_saturated_connection_reports_slow_without_cancelling_the_send() {+ let manager = ConnectionManager::new(UnusedRepo);+ let (sender, mut receiver) = tokio::sync::mpsc::channel(1);+ sender+ .send(OutgoingMessage::Message(Message::new(+ "test".to_owned(),+ "first".to_owned(),+ )))+ .await+ .unwrap();+ let forwarder = tokio::spawn(std::future::pending::<()>());+ manager.connections.insert(+ "connection".to_owned(),+ Connection {+ sender,+ abort_handle: forwarder.abort_handle(),+ },+ );++ let send = tokio::spawn({+ let manager = manager.clone();+ async move {+ manager+ .send_message(+ "connection",+ Message::new("test".to_owned(), "second".to_owned()),+ )+ .await+ }+ });+ tokio::task::yield_now().await;+ tokio::time::advance(SLOW_WEBSOCKET_OPERATION_THRESHOLD).await;+ assert!(!send.is_finished(), "telemetry must not cancel the send");++ receiver.recv().await.unwrap();+ send.await.unwrap().unwrap();+}