Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(observability): trace agent session stalls (#5982)

Wolf Mermelstein4528ef86ad8f
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rs
index 8a18c94cb7..2092348322 100644
--- a/crates/agent_session/src/domain/service.rs
+++ b/crates/agent_session/src/domain/service.rs
@@ -390,6 +390,8 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
"agent.session.command",
agent.session.id = %id,
agent.action.name = action.as_ref(),
+ agent.command.queue_wait_ms = tracing::field::Empty,
+ agent.session.runtime_phase_at_dequeue = tracing::field::Empty,
otel.status_code = tracing::field::Empty,
otel.status_description = tracing::field::Empty,
);
@@ -400,6 +402,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
action_id,
completed,
span,
+ enqueued_at: tokio::time::Instant::now(),
})
.await
.is_err()
@@ -908,6 +911,12 @@ where
Rt: AgentSessionRealtime,
{
/// Push the frame just appended out to whoever is watching the session.
+ #[tracing::instrument(
+ name = "agent.session.realtime.publish",
+ err,
+ skip(self, agent_session_id, entry),
+ fields(agent.session.id = %agent_session_id)
+ )]
async fn stream(
&mut self,
agent_session_id: AgentSessionId,
diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rs
index 73e0237e97..04b458c263 100644
--- a/crates/agent_session/src/domain/service/test.rs
+++ b/crates/agent_session/src/domain/service/test.rs
@@ -663,6 +663,7 @@ async fn cancellation_does_not_drop_an_effect_batch_after_machine_mutation() {
action_id: AgentActionId::mint(),
completed,
span: tracing::info_span!("test.command"),
+ enqueued_at: tokio::time::Instant::now(),
})
.await
.unwrap();
@@ -737,6 +738,7 @@ async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {
action_id: AgentActionId::mint(),
completed,
span: tracing::info_span!("test.command"),
+ enqueued_at: tokio::time::Instant::now(),
})
.await
.unwrap();
diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rs
index d26df5f32f..d199785f16 100644
--- a/crates/agent_session/src/domain/session/actors.rs
+++ b/crates/agent_session/src/domain/session/actors.rs
@@ -45,6 +45,7 @@ pub(crate) struct SessionCommand {
pub(crate) action_id: AgentActionId,
pub(crate) completed: oneshot::Sender<Result<()>>,
pub(crate) span: tracing::Span,
+ pub(crate) enqueued_at: Instant,
}
pub(crate) struct SessionCompletion {
@@ -156,11 +157,21 @@ where
let input = tokio::select! {
() = handshake_timeout => Input::Closed(CloseReason::HandshakeTimedOut),
command = self.commands.recv() => match command {
- Some(SessionCommand { user_id, action, action_id, completed, span }) => Input::Command {
- from: user_id,
- action,
- action_id,
- token: SessionCompletion { completed, span },
+ Some(SessionCommand { user_id, action, action_id, completed, span, enqueued_at }) => {
+ span.record(
+ "agent.command.queue_wait_ms",
+ enqueued_at.elapsed().as_millis() as u64,
+ );
+ span.record(
+ "agent.session.runtime_phase_at_dequeue",
+ self.machine.status().as_ref(),
+ );
+ Input::Command {
+ from: user_id,
+ action,
+ action_id,
+ token: SessionCompletion { completed, span },
+ }
},
// The service dropped every handle; nobody can reach us.
None => Input::Closed(CloseReason::Abandoned),
diff --git a/crates/agent_session/src/domain/session/types.rs b/crates/agent_session/src/domain/session/types.rs
index 179376688b..ef3a3302ef 100644
--- a/crates/agent_session/src/domain/session/types.rs
+++ b/crates/agent_session/src/domain/session/types.rs
@@ -79,7 +79,8 @@ pub enum HandshakeStatus {
}
/// Observable phase of a session connection.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq, strum::AsRefStr)]
+#[strum(serialize_all = "snake_case")]
pub enum RuntimeStatus {
/// Waiting for the runtime to report its agent ready.
Booting,
diff --git a/services/connection_gateway/src/api/connection.rs b/services/connection_gateway/src/api/connection.rs
index 797a4b2b29..d113de0dc4 100644
--- a/services/connection_gateway/src/api/connection.rs
+++ b/services/connection_gateway/src/api/connection.rs
@@ -1,5 +1,6 @@
use crate::{
config::Config,
+ constants::SLOW_WEBSOCKET_OPERATION_THRESHOLD,
context::{ApiContext, AppState, AuthorizationService},
model::{
connection::ConnectionContext,
@@ -76,7 +77,8 @@ async fn handle_websocket_connection(
// Create guard that records last online time when websocket connection closes
let last_online_guard = ctx.last_online_worker.new_guard(macro_user_id.clone());
- let sender_task = tokio::spawn(async move { forwarder(sink, receiver).await });
+ let sender_connection_id = connection_id.clone();
+ let sender_task = tokio::spawn(forwarder(sink, receiver, sender_connection_id));
if let Err(err) = ctx
.connection_manager
@@ -136,17 +138,24 @@ async fn handle_websocket_connection(
/// Forwards messages from a [Receiver] to a [SplitSink]
/// This is useful as [SplitSink] does not implement [Clone]
-pub async fn forwarder(
+async fn forwarder(
mut sink: SplitSink<WebSocket, AxumWebsocketMessage>,
mut receiver: Receiver<OutgoingMessage>,
+ connection_id: String,
) -> Result<()> {
while let Some(mut message) = receiver.recv().await {
+ let queue_depth = receiver.len();
+ let queue_max_capacity = receiver.max_capacity();
let span = match &message {
OutgoingMessage::Message(message) => {
let span = tracing::info_span!(
"connection_gateway.websocket_send",
otel.kind = "producer",
message_type = %message.message_type,
+ connection.id = %connection_id,
+ connection.queue.observed_depth = queue_depth,
+ connection.queue.max_capacity = queue_max_capacity,
+ websocket.write_ms = tracing::field::Empty,
otel.status_code = tracing::field::Empty,
otel.status_description = tracing::field::Empty,
);
@@ -162,7 +171,28 @@ pub async fn forwarder(
}
if let Ok(msg) = message.try_into() {
- let result = sink.send(msg).instrument(span.clone()).await;
+ let write_span = span.clone();
+ let result = async {
+ let started = tokio::time::Instant::now();
+ let mut send = std::pin::pin!(sink.send(msg));
+ let result = tokio::select! {
+ result = &mut send => result,
+ () = tokio::time::sleep(SLOW_WEBSOCKET_OPERATION_THRESHOLD) => {
+ tracing::warn!(
+ connection.id = %connection_id,
+ connection.queue.observed_depth = receiver.len(),
+ connection.queue.max_capacity = queue_max_capacity,
+ "websocket write is blocked"
+ );
+ send.await
+ }
+ };
+ (result, started.elapsed())
+ }
+ .instrument(write_span)
+ .await;
+ let (result, write) = result;
+ span.record("websocket.write_ms", write.as_millis() as u64);
if let Err(err) = result {
record_span_error(&span, &err);
tracing::warn!(
diff --git a/services/connection_gateway/src/constants.rs b/services/connection_gateway/src/constants.rs
index 59788dcd84..dc4d1d9f97 100644
--- a/services/connection_gateway/src/constants.rs
+++ b/services/connection_gateway/src/constants.rs
@@ -1,4 +1,5 @@
use axum::http::HeaderValue;
+use std::time::Duration;
pub const ORIGINS: [HeaderValue; 23] = [
HeaderValue::from_static("http://localhost:3000"),
@@ -28,3 +29,6 @@ pub const ORIGINS: [HeaderValue; 23] = [
/// The default timeout threshold is 1 minute
pub const DEFAULT_TIMEOUT_THRESHOLD: u64 = 60_000;
+
+/// Emit diagnostics while a websocket queue or write remains blocked this long.
+pub(crate) const SLOW_WEBSOCKET_OPERATION_THRESHOLD: Duration = Duration::from_secs(1);
diff --git a/services/connection_gateway/src/service/connection.rs b/services/connection_gateway/src/service/connection.rs
index be17db3195..6cf49a7e01 100644
--- a/services/connection_gateway/src/service/connection.rs
+++ b/services/connection_gateway/src/service/connection.rs
@@ -1,3 +1,4 @@
+use crate::constants::SLOW_WEBSOCKET_OPERATION_THRESHOLD;
use crate::model::{
connection::{Connection, StoredConnectionEntity},
message::{Message, OutgoingMessage},
@@ -13,6 +14,9 @@ use std::{
};
use tokio::{sync::mpsc::Sender, task::AbortHandle};
+#[cfg(test)]
+mod test;
+
#[async_trait]
pub trait ConnectionRepo: Send + Sync {
/// Inserts a new [StoredConnectionEntity] into storage
@@ -199,13 +203,49 @@ impl ConnectionManager {
/// Sends a message to a a connection
/// If the connection is not found, or dropped, then we remove the connection id
/// from the map, and should kill (TODO:) any remaining tasks associated with the connection
+ #[tracing::instrument(
+ name = "connection_gateway.queue_send",
+ err,
+ skip(self, id, message),
+ fields(
+ connection.id = id,
+ connection.queue.observed_depth = tracing::field::Empty,
+ connection.queue.max_capacity = tracing::field::Empty,
+ connection.queue.wait_ms = tracing::field::Empty,
+ )
+ )]
pub async fn send_message(&self, id: &str, message: Message) -> Result<()> {
let sender = match self.connections.get(id) {
Some(connection) => connection.sender.clone(),
None => anyhow::bail!("connection not found"),
};
- if let Err(err) = sender.send(OutgoingMessage::Message(message)).await {
+ let remaining_capacity = sender.capacity();
+ let max_capacity = sender.max_capacity();
+ let depth = max_capacity.saturating_sub(remaining_capacity);
+ let span = tracing::Span::current();
+ span.record("connection.queue.observed_depth", depth as u64);
+ span.record("connection.queue.max_capacity", max_capacity as u64);
+
+ let started = tokio::time::Instant::now();
+ let mut send = std::pin::pin!(sender.send(OutgoingMessage::Message(message)));
+ let result = tokio::select! {
+ result = &mut send => result,
+ () = tokio::time::sleep(SLOW_WEBSOCKET_OPERATION_THRESHOLD) => {
+ let depth = max_capacity.saturating_sub(sender.capacity());
+ tracing::warn!(
+ connection.id = id,
+ connection.queue.observed_depth = depth,
+ connection.queue.max_capacity = max_capacity,
+ "websocket outbound queue send is blocked"
+ );
+ send.await
+ }
+ };
+ let wait = started.elapsed();
+ span.record("connection.queue.wait_ms", wait.as_millis() as u64);
+
+ if let Err(err) = result {
self.remove_connection(id).await?;
anyhow::bail!("failed to send message: {}", err);
}
diff --git a/services/connection_gateway/src/service/connection/test.rs b/services/connection_gateway/src/service/connection/test.rs
new file mode 100644
index 0000000000..184167e8d9
--- /dev/null
+++ b/services/connection_gateway/src/service/connection/test.rs
@@ -0,0 +1,105 @@
+use super::*;
+
+struct UnusedRepo;
+
+#[async_trait]
+impl ConnectionRepo for UnusedRepo {
+ async fn insert_connection_entry(
+ &self,
+ _connection: UserEntityConnection<'_>,
+ ) -> anyhow::Result<StoredConnectionEntity> {
+ unimplemented!()
+ }
+
+ async fn get_entries_by_entity(
+ &self,
+ _entity: &Entity<'_>,
+ ) -> anyhow::Result<Vec<StoredConnectionEntity>> {
+ unimplemented!()
+ }
+
+ async fn get_entries_by_connection_id(
+ &self,
+ _connection_id: &str,
+ ) -> anyhow::Result<Vec<StoredConnectionEntity>> {
+ unimplemented!()
+ }
+
+ async fn get_connection(&self, _connection_id: &str) -> anyhow::Result<StoredConnectionEntity> {
+ unimplemented!()
+ }
+
+ async fn get_entry_for_connection_entity(
+ &self,
+ _entity: EntityConnection<'_>,
+ ) -> anyhow::Result<Option<StoredConnectionEntity>> {
+ unimplemented!()
+ }
+
+ async fn remove_all_entries_for_by_connection_id(
+ &self,
+ _connection_id: &str,
+ ) -> anyhow::Result<()> {
+ Ok(())
+ }
+
+ async fn remove_entity(&self, _entity: &EntityConnection<'_>) -> anyhow::Result<()> {
+ unimplemented!()
+ }
+
+ async fn update_last_entity_ping(
+ &self,
+ _entity: &EntityConnection<'_>,
+ _timestamp: u64,
+ ) -> anyhow::Result<StoredConnectionEntity> {
+ unimplemented!()
+ }
+
+ async fn update_user_connection_last_ping(
+ &self,
+ _connection_id: &str,
+ _user: &str,
+ _timestamp: u64,
+ ) -> anyhow::Result<()> {
+ unimplemented!()
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_saturated_connection_reports_slow_without_cancelling_the_send() {
+ let manager = ConnectionManager::new(UnusedRepo);
+ let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
+ sender
+ .send(OutgoingMessage::Message(Message::new(
+ "test".to_owned(),
+ "first".to_owned(),
+ )))
+ .await
+ .unwrap();
+ let forwarder = tokio::spawn(std::future::pending::<()>());
+ manager.connections.insert(
+ "connection".to_owned(),
+ Connection {
+ sender,
+ abort_handle: forwarder.abort_handle(),
+ },
+ );
+
+ let send = tokio::spawn({
+ let manager = manager.clone();
+ async move {
+ manager
+ .send_message(
+ "connection",
+ Message::new("test".to_owned(), "second".to_owned()),
+ )
+ .await
+ }
+ });
+ tokio::task::yield_now().await;
+ tokio::time::advance(SLOW_WEBSOCKET_OPERATION_THRESHOLD).await;
+ assert!(!send.is_finished(), "telemetry must not cancel the send");
+
+ receiver.recv().await.unwrap();
+ send.await.unwrap().unwrap();
+}