Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(otel): trace agent triggers end to end (#5819)
Wolf Mermelstein87788136afeb
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 89d34bd2b7..adebd361e5 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -94,6 +94,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -180,6 +181,7 @@"crates/macro_event_topics","crates/macro_queues","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -255,6 +257,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -314,6 +317,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -377,6 +381,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -947,6 +952,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1032,6 +1038,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/maybe_send",@@ -1136,6 +1143,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1326,6 +1334,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1519,6 +1528,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1671,6 +1681,7 @@"crates/macro_authorization","crates/macro_env","crates/macro_env_var",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/model",@@ -1710,6 +1721,7 @@"crates/macro_entrypoint","crates/macro_env","crates/macro_env_var",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/model",@@ -1732,6 +1744,7 @@"crates/connection_gateway_client","crates/connection_gateway_models","crates/cowlike",+ "crates/macro_tower_layers","crates/model-entity","crates/workspace-hack"],@@ -2970,6 +2983,7 @@"crates/macro_middleware","crates/macro_queues","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -3518,6 +3532,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -4087,6 +4102,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -4223,6 +4239,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -4300,6 +4317,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -5066,6 +5084,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -5147,6 +5166,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -5211,6 +5231,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -5432,6 +5453,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mcp_client",@@ -6197,6 +6219,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -6278,6 +6301,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -6401,6 +6425,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -6512,6 +6537,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -6596,6 +6622,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -6671,6 +6698,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/maybe_send",@@ -6744,6 +6772,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -7109,6 +7138,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -7185,6 +7215,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -7275,6 +7306,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_tower_layers","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",diff --git a/Cargo.lock b/Cargo.lockindex 5aba238791..440b291378 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -217,6 +217,7 @@ dependencies = ["macro_authorization","macro_env_var","macro_event_broker",+ "macro_tower_layers","macro_user_id","macro_uuid","mockall",@@ -297,6 +298,7 @@ dependencies = ["specta","specta-serde","specta-typescript",+ "strum 0.27.2","thiserror 2.0.18","tokio","tokio-tungstenite 0.28.0",@@ -339,6 +341,7 @@ dependencies = ["thiserror 2.0.18","tokio","tokio-tungstenite 0.28.0",+ "tokio-util","tower 0.5.3","tracing","utoipa",@@ -3648,11 +3651,13 @@ dependencies = ["macro_entrypoint","macro_env","macro_env_var",+ "macro_tower_layers","macro_user_id","mockall","model","model-entity","models_bulk_upload",+ "opentelemetry","redis","secretsmanager_client","serde",@@ -3666,6 +3671,7 @@ dependencies = ["tower 0.5.3","tower-http","tracing",+ "tracing-opentelemetry","tungstenite 0.28.0","url","utoipa",@@ -3680,6 +3686,7 @@ version = "0.1.0"dependencies = ["anyhow","connection_gateway_models",+ "macro_tower_layers","model-entity","reqwest 0.13.4","serde",@@ -8555,10 +8562,12 @@ dependencies = ["either","macro_env","macro_env_var",+ "opentelemetry","rdkafka","thiserror 2.0.18","tokio","tracing",+ "tracing-opentelemetry","uuid","workspace-hack",]diff --git a/crates/agent_harness/Cargo.toml b/crates/agent_harness/Cargo.tomlindex 3c682582b1..c8ae2eaefc 100644--- a/crates/agent_harness/Cargo.toml+++ b/crates/agent_harness/Cargo.toml@@ -23,6 +23,7 @@ kafka_util = { path = "../kafka_util" }macro_env_var = { path = "../macro_env_var" }macro_event_broker = { path = "../macro_event_broker", features = ["outbound"] }macro_authorization = { path = "../macro_authorization", features = ["axum"] }+macro_tower_layers = { path = "../macro_tower_layers" }macro_user_id = { path = "../macro_user_id" }macro_uuid = { path = "../macro_uuid" }reqwest = { workspace = true }diff --git a/crates/agent_harness/src/domain/ports/test.rs b/crates/agent_harness/src/domain/ports/test.rsindex fd1d31effc..49148dd47b 100644--- a/crates/agent_harness/src/domain/ports/test.rs+++ b/crates/agent_harness/src/domain/ports/test.rs@@ -99,6 +99,7 @@ async fn container_session_runs_and_logs_end_to_end() {agent.wait_for_requests(2).await;agent.opens_session(NewSessionResponse::new("acp-container-test"));agent.wait_for_requests(3).await;+ agent.completes_prompt().await;send.await.unwrap().unwrap();assert_eq!(@@ -116,15 +117,14 @@ async fn container_session_runs_and_logs_end_to_end() {assert!(matches!(logs[5].entry.content, Message::ToRuntime(_)));assert_eq!(logs[5].entry.user_id, Some(owner()));- sessions- .send_action(- id,- Some(owner()),- AgentAction::prompt("and run clippy"),- AgentActionId::mint(),- )- .await- .unwrap();+ let send = sessions.send_action(+ id,+ Some(owner()),+ AgentAction::prompt("and run clippy"),+ AgentActionId::mint(),+ );+ let (result, ()) = tokio::join!(send, agent.completes_prompt());+ result.unwrap();assert_eq!(containers.spawned(), 1);assert_eq!(containers.resumed(), 0);assert_eq!(diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rsindex 9d32d84a55..cf4be0e772 100644--- a/crates/agent_harness/src/domain/service.rs+++ b/crates/agent_harness/src/domain/service.rs@@ -16,6 +16,8 @@ use dashmap::DashMap;use dashmap::mapref::entry::Entry;use macro_user_id::user_id::MacroUserIdStr;use tokio::sync::{mpsc, oneshot};+use tracing::Instrument as _;+use tracing::instrument::WithSubscriber as _;use crate::domain::error::{HarnessError, Result};use crate::domain::model::{@@ -30,6 +32,9 @@ type SessionWorkers = DashMap<AgentSessionId, mpsc::UnboundedSender<QueuedCommanstruct QueuedCommand {command: HarnessCommand,completed: oneshot::Sender<Result<()>>,+ /// The caller's span, carried across the queue so the work the worker does+ /// on its own task still hangs off whatever triggered it.+ span: tracing::Span,}struct AgentHarnessInner<Sessions, Containers, Announcer, Runtimes> {@@ -77,16 +82,23 @@ where/// Queue one command behind any work already running for its session.////// Queue admission happens synchronously so callers can spawn the returned- /// completion future without reordering commands.+ /// completion future without reordering commands. It is also where the+ /// caller's span is captured: the returned future only awaits a oneshot, so+ /// the span has to travel with the command to reach the work itself.pub fn execute(&self,session_id: AgentSessionId,mut command: HarnessCommand,) -> impl Future<Output = Result<()>> + Send + 'static {+ let caller = tracing::Span::current();let result = loop {let commands = self.commands(session_id);let (completed, result) = oneshot::channel();- let queued = QueuedCommand { command, completed };+ let queued = QueuedCommand {+ command,+ completed,+ span: caller.clone(),+ };match commands.send(queued) {Ok(()) => break result,@@ -122,7 +134,10 @@ wheresession_id: AgentSessionId,receiver: mpsc::UnboundedReceiver<QueuedCommand>,) {- tokio::spawn(run_session_worker(session_id, self.inner.clone(), receiver));+ // The worker outlives the call that created it, so it has to carry the+ // subscriber forward itself or every command it runs traces nowhere.+ let inner = self.inner.clone();+ tokio::spawn(run_session_worker(session_id, inner, receiver).with_current_subscriber());}/// Post the announcement for a prompt an external runtime delivers.@@ -458,9 +473,14 @@ where%session_id,bot_id = %command.bot_id,message_id = %command.origin.message_id,+ channel_id = %command.origin.channel_id,+ thread_id = %command.origin.thread_id,+ agent.trigger.kind = "mention",+ agent.session.id = tracing::field::Empty,))]async fn open(&self, session_id: AgentSessionId, command: OpenSession) -> Result<()> {let OpenSession { bot_id, origin } = command;+ tracing::Span::current().record("agent.session.id", tracing::field::display(session_id));let repo_url = self.defaults.repo_url.clone();let sandbox_size = self.sessions.user_sandbox_size(&origin.sender).await?;@@ -537,7 +557,7 @@ where/// Three steps, in this order: persist whatever the action changes about/// the session, work out whether anyone needs telling, then deliver it./// Announcing last means nothing is announced that was never sent.- #[tracing::instrument(err, skip(self, command), fields(%session_id))]+ #[tracing::instrument(err, skip(self, command), fields(agent.session.id = %session_id))]async fn deliver(&self, session_id: AgentSessionId, command: DeliverAction) -> Result<()> {let DeliverAction {id,@@ -641,8 +661,12 @@ async fn run_session_worker<Sessions, Containers, Announcer, Runtimes>(Runtimes: RuntimeConnections,{while let Some(queued) = receiver.recv().await {- let QueuedCommand { command, completed } = queued;- let result = inner.execute(session_id, command).await;+ let QueuedCommand {+ command,+ completed,+ span,+ } = queued;+ let result = inner.execute(session_id, command).instrument(span).await;let _ = completed.send(result);}}diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rsindex 0076bf994c..22d4ecb9ab 100644--- a/crates/agent_harness/src/domain/service/test.rs+++ b/crates/agent_harness/src/domain/service/test.rs@@ -119,7 +119,7 @@ fn harness() -> (}/// Play the agent's half of the ACP handshake.-async fn complete_handshake(container: &ContainerMock) {+async fn complete_session_handshake(container: &ContainerMock) {let agent = container.agent();let already = agent.received_requests().len();container.sends_ready();@@ -129,6 +129,12 @@ async fn complete_handshake(container: &ContainerMock) {agent.opens_session(NewSessionResponse::new("acp-test"));}+async fn complete_handshake(container: &ContainerMock) {+ complete_session_handshake(container).await;+ let agent = container.agent();+ agent.completes_prompt().await;+}+async fn complete_resume(container: &ContainerMock) {let agent = container.agent();container.sends_ready();@@ -146,6 +152,7 @@ async fn complete_resume(container: &ContainerMock) {ClientRequest::ResumeSessionRequest(request) if request.session_id.to_string() == "acp-test"));agent.resumes_session(ResumeSessionResponse::new());+ agent.completes_prompt().await;}fn prompts(agent: &FakeAgent) -> Vec<Vec<ContentBlock>> {@@ -333,13 +340,13 @@ async fn forward_to_a_live_session_reuses_the_transport() {let (opened, container) = tokio::join!(open, drive);opened.expect("open should succeed");- service- .execute(- id,- HarnessCommand::Deliver(forward_message("and add a regression test")),- )- .await- .expect("forward to a live session should succeed");+ let forward = service.execute(+ id,+ HarnessCommand::Deliver(forward_message("and add a regression test")),+ );+ let agent = container.agent();+ let (result, ()) = tokio::join!(forward, agent.completes_prompt());+ result.expect("forward to a live session should succeed");assert_eq!(containers.spawned(), 1, "no second container");assert_eq!(containers.resumed(), 0, "no resume for a live session");@@ -460,6 +467,7 @@ async fn concurrent_forwards_share_one_session_recovery() {.expect("the resumed container is findable");complete_resume(&resumed).await;resumed.agent().wait_for_requests(4).await;+ resumed.agent().completes_prompt().await;resumed};@@ -673,16 +681,16 @@ async fn a_prompt_through_control_reaches_the_agent_without_announcing() {let container = live_session(&service, &containers, id).await;let announced_before = announcer.announced().len();- service- .control_event(- id,- ControlEvent {- action: AgentAction::prompt("and now the docs"),- actor: Some(sender()),- },- )- .await- .expect("prompting through control should succeed");+ let prompted = service.control_event(+ id,+ ControlEvent {+ action: AgentAction::prompt("and now the docs"),+ actor: Some(sender()),+ },+ );+ let agent = container.agent();+ let (result, ()) = tokio::join!(prompted, agent.completes_prompt());+ result.expect("prompting through control should succeed");assert_eq!(prompts(&container.agent()).len(),@@ -702,16 +710,16 @@ async fn compact_through_control_reaches_opencode_as_a_slash_command() {let id = AgentSessionId::new();let container = live_session(&service, &containers, id).await;- service- .control_event(- id,- ControlEvent {- action: AgentAction::Compact,- actor: Some(sender()),- },- )- .await- .expect("compaction should reach the running agent");+ let compacted = service.control_event(+ id,+ ControlEvent {+ action: AgentAction::Compact,+ actor: Some(sender()),+ },+ );+ let agent = container.agent();+ let (result, ()) = tokio::join!(compacted, agent.completes_prompt());+ result.expect("compaction should reach the running agent");assert_eq!(prompts(&container.agent()),@@ -784,6 +792,7 @@ async fn complete_bound_handshake(runtime: &ContainerMock) {agent.completes_initialize(InitializeResponse::new(PROTOCOL_VERSION));agent.wait_for_requests(2).await;agent.opens_session(NewSessionResponse::new("acp-test"));+ agent.completes_prompt().await;}/// As [`complete_bound_handshake`], for a session the runtime is restoring.@@ -806,6 +815,7 @@ async fn complete_bound_resume(runtime: &ContainerMock) {ClientRequest::ResumeSessionRequest(request) if request.session_id.to_string() == "acp-test"));agent.resumes_session(ResumeSessionResponse::new());+ agent.completes_prompt().await;}/// Wait until a session has noticed that its transport ended.@@ -918,9 +928,10 @@ async fn a_bound_session_stays_on_its_connection_until_it_drops() {// Already bound: a second prompt goes straight down the same connection,// with no second handshake to drive.- prompt(&service, session.id, "and now the docs")- .await- .expect("a bound session needs no rebinding");+ let prompted = prompt(&service, session.id, "and now the docs");+ let agent = first.agent();+ let (result, ()) = tokio::join!(prompted, agent.completes_prompt());+ result.expect("a bound session needs no rebinding");assert_eq!(prompts(&first.agent()),["fix the failing test", "and now the docs"]@@ -1200,7 +1211,7 @@ async fn open_managed_session_spawns_at_the_users_default_size() {let container = containers.container(session_of(&containers)).expect("the spawned container is findable");- complete_handshake(&container).await;+ complete_session_handshake(&container).await;};let (opened, _) = tokio::join!(open, drive);let session = opened.expect("open should succeed");diff --git a/crates/agent_harness/src/outbound/daytona/client.rs b/crates/agent_harness/src/outbound/daytona/client.rsindex 26e3fafa22..04b138a387 100644--- a/crates/agent_harness/src/outbound/daytona/client.rs+++ b/crates/agent_harness/src/outbound/daytona/client.rs@@ -124,7 +124,7 @@ impl DaytonaClient {/// Resource fields must be omitted: Daytona returns 400 ("Cannot specify/// Sandbox resources when using a snapshot") if `cpu` / `memory` / `disk`/// are sent with `snapshot`. Size is applied after start via [`Self::resize`].- #[tracing::instrument(err, skip(self, env))]+ #[tracing::instrument(name = "daytona.sandbox.create", err, skip(self, env))]pub async fn create(&self, snapshot: &Snapshot, env: Env, labels: Labels) -> Result<String> {let request = configuration_parameters(snapshot, env, labels);let sandbox: SandboxDto = self@@ -184,7 +184,7 @@ impl DaytonaClient {}/// Find one sandbox carrying the supplied label.- #[tracing::instrument(err, skip(self))]+ #[tracing::instrument(name = "daytona.sandbox.find", err, skip(self))]pub async fn find_by_label(&self, label: &str, value: &str) -> Result<Option<String>> {let labels = Labels::from(HashMap::from([(label.to_owned(), value.to_owned())]));let filter = serde_json::to_string(&labels).map_err(DaytonaError::EncodeLabelFilter)?;@@ -211,7 +211,7 @@ impl DaytonaClient {}/// Poll a sandbox until it has started or the deadline passes.- #[tracing::instrument(err, skip(self))]+ #[tracing::instrument(name = "daytona.sandbox.wait_started", err, skip(self))]pub async fn wait_for_started(&self, sandbox_id: &str, timeout: Duration) -> Result<()> {let deadline = Instant::now() + timeout;loop {@@ -373,7 +373,7 @@ impl DaytonaClient {}/// Execute one command in a sandbox.- #[tracing::instrument(err, skip(self))]+ #[tracing::instrument(name = "daytona.command.execute", err, skip(self, command))]pub async fn exec(&self, sandbox_id: &str, command: &str, timeout: Duration) -> Result<String> {let toolbox: ToolboxProxyUrlDto = self.json(@@ -451,7 +451,7 @@ impl DaytonaClient {}/// Poll the sidecar readiness endpoint until it succeeds.- #[tracing::instrument(err, skip(self))]+ #[tracing::instrument(name = "daytona.sidecar.wait_ready", err, skip(self))]pub async fn wait_for_ping(&self,ping_url: &str,@@ -461,6 +461,9 @@ impl DaytonaClient {let deadline = Instant::now() + timeout;loop {let mut request = self.http.get(ping_url);+ let mut trace_headers = reqwest::header::HeaderMap::new();+ macro_tower_layers::inject_trace_headers(&mut trace_headers);+ request = request.headers(trace_headers);if let Some(token) = preview_token {request = request.header("x-daytona-preview-token", token);}@@ -485,7 +488,10 @@ impl DaytonaClient {request: reqwest::RequestBuilder,operation: &'static str,) -> Result<T> {+ let mut trace_headers = reqwest::header::HeaderMap::new();+ macro_tower_layers::inject_trace_headers(&mut trace_headers);let response = request+ .headers(trace_headers).bearer_auth(self.api_key.expose()).send().awaitdiff --git a/crates/agent_harness/src/outbound/daytona/manager.rs b/crates/agent_harness/src/outbound/daytona/manager.rsindex 43c53355ce..9daf3f0c6b 100644--- a/crates/agent_harness/src/outbound/daytona/manager.rs+++ b/crates/agent_harness/src/outbound/daytona/manager.rs@@ -8,6 +8,7 @@ use agent_session::domain::model::AgentSessionId;use futures::{StreamExt as _, stream};use tokio_tungstenite::tungstenite::client::IntoClientRequest;use tokio_util::{sync::CancellationToken, task::TaskTracker};+use tracing::Instrument as _;use super::client::DaytonaClient;use super::errors::DaytonaError;@@ -110,6 +111,12 @@ impl DaytonaContainerManager {}}+ #[tracing::instrument(+ name = "agent.container.boot",+ err,+ skip(self, id),+ fields(agent.container.provider = "daytona", agent.container.id = id.as_str())+ )]async fn bring_up(&self, id: &DaytonaSandboxId) -> Result<DaytonaContainer> {self.client.wait_for_started(id.as_str(), provision::ENSURE_TIMEOUT)@@ -122,6 +129,7 @@ impl DaytonaContainerManager {&provision::ensure_ready_command(),provision::ENSURE_TIMEOUT,)+ .instrument(tracing::info_span!("agent.container.ensure_ready")).await.map_err(unavailable)?;tracing::info!(sandbox_id = %id.as_str(), %output, "readiness recipe finished");@@ -139,7 +147,10 @@ impl DaytonaContainerManager {).await.map_err(unavailable)?;- let socket = dial_sidecar(&preview).await.map_err(unavailable)?;+ let socket = dial_sidecar(&preview)+ .instrument(tracing::info_span!("agent.container.websocket_connect"))+ .await+ .map_err(unavailable)?;if !self.managed.containers.activate(id, Instant::now()) {return Err(HarnessError::Container("sandbox is no longer managed".to_owned(),@@ -336,7 +347,12 @@ impl DaytonaContainerManager {impl ContainerManager for DaytonaContainerManager {type Transport = DaytonaContainer;- #[tracing::instrument(err, skip(self))]+ #[tracing::instrument(+ name = "agent.container.spawn",+ err,+ skip(self),+ fields(agent.container.provider = "daytona")+ )]async fn spawn(&self, command: SpawnContainer) -> Result<DaytonaContainer> {let SpawnContainer {session_id,diff --git a/crates/agent_harness/src/outbound/namespace/client.rs b/crates/agent_harness/src/outbound/namespace/client.rsindex 62b1477370..72c9ee39e2 100644--- a/crates/agent_harness/src/outbound/namespace/client.rs+++ b/crates/agent_harness/src/outbound/namespace/client.rs@@ -316,9 +316,12 @@ impl NamespaceClient {B: Serialize + ?Sized,{let url = format!("{}/{service}/{method}", host.trim_end_matches('/'));+ let mut trace_headers = reqwest::header::HeaderMap::new();+ macro_tower_layers::inject_trace_headers(&mut trace_headers);let response = self.http.post(url)+ .headers(trace_headers).bearer_auth(self.token.expose()).json(body).send()diff --git a/crates/agent_harness/src/outbound/namespace/manager.rs b/crates/agent_harness/src/outbound/namespace/manager.rsindex f6f72ad7e3..249fbfc113 100644--- a/crates/agent_harness/src/outbound/namespace/manager.rs+++ b/crates/agent_harness/src/outbound/namespace/manager.rs@@ -3,6 +3,7 @@ use std::sync::Arc;use agent_runtime_protocol::domain::ports::Transport;use agent_runtime_protocol::domain::schema::v0::{ToRuntimeMessage, ToServerMessage};use agent_session::domain::model::{AgentSessionId, SandboxSize};+use tracing::Instrument as _;use super::client::NamespaceClient;use super::errors::NamespaceError;@@ -42,6 +43,12 @@ impl NamespaceContainerManager {}}+ #[tracing::instrument(+ name = "agent.container.boot",+ err,+ skip(self),+ fields(agent.container.provider = "namespace", agent.container.id = %instance.id)+ )]async fn bring_up(&self, instance: Instance) -> Result<NamespaceContainer> {self.client.wait_until_ready(&instance.id, provision::ENSURE_TIMEOUT)@@ -53,6 +60,7 @@ impl NamespaceContainerManager {&instance,&["bash", "-lc", &provision::ensure_ready_command()],)+ .instrument(tracing::info_span!("agent.container.ensure_ready")).await.map_err(unavailable)?;if output.exit_code != 0 {@@ -69,7 +77,10 @@ impl NamespaceContainerManager {.create_ingress(&instance.id, provision::SIDECAR_PORT).await.map_err(unavailable)?;- let socket = dial_sidecar(&sidecar_url).await.map_err(unavailable)?;+ let socket = dial_sidecar(&sidecar_url)+ .instrument(tracing::info_span!("agent.container.websocket_connect"))+ .await+ .map_err(unavailable)?;Ok(NamespaceContainer {instance: Arc::new(instance),@@ -82,7 +93,12 @@ impl NamespaceContainerManager {impl ContainerManager for NamespaceContainerManager {type Transport = NamespaceContainer;- #[tracing::instrument(err, skip(self))]+ #[tracing::instrument(+ name = "agent.container.spawn",+ err,+ skip(self),+ fields(agent.container.provider = "namespace")+ )]async fn spawn(&self, command: SpawnContainer) -> Result<Self::Transport> {let container = ContainerSpec {image_ref: self.image_ref.clone(),diff --git a/crates/agent_harness/src/outbound/sidecar.rs b/crates/agent_harness/src/outbound/sidecar.rsindex c450c0c4a3..bdaea15f93 100644--- a/crates/agent_harness/src/outbound/sidecar.rs+++ b/crates/agent_harness/src/outbound/sidecar.rs@@ -17,6 +17,8 @@ use tokio::io::{AsyncRead, AsyncWrite};use tokio::sync::{mpsc, oneshot};use tokio_tungstenite::WebSocketStream;use tokio_tungstenite::tungstenite::Message;+use tracing::Instrument as _;+use tracing::instrument::WithSubscriber as _;#[cfg(test)]mod test;@@ -26,17 +28,18 @@ mod test;/// Not `Clone` itself - `recv` has to be exclusive - so containers share one/// through an `Arc`, which is what makes them cloneable.pub struct SidecarTransport {- outbound: mpsc::UnboundedSender<Outbound>,+ outbound: mpsc::UnboundedSender<OutboundFrame>,inbound: mpsc::UnboundedReceiver<ToServerMessage>,}/// The sidecar's sending half.pub struct SidecarSender {- outbound: mpsc::UnboundedSender<Outbound>,+ outbound: mpsc::UnboundedSender<OutboundFrame>,}-struct Outbound {+struct OutboundFrame {frame: RawJsonRpcMessage,+ parent: tracing::Span,completed: oneshot::Sender<Result<(), TransportError>>,}@@ -72,7 +75,7 @@ impl SidecarTransport {};let _ = inbound_tx.send(ready);- tokio::spawn(pump(socket, outbound_rx, inbound_tx, on_frame));+ tokio::spawn(pump(socket, outbound_rx, inbound_tx, on_frame).with_current_subscriber());Self {outbound: outbound_tx,@@ -105,9 +108,14 @@ impl TransportSender<ToRuntimeMessage> for SidecarSender {return Ok(());};+ let parent = tracing::Span::current();let (completed, result) = oneshot::channel();self.outbound- .send(Outbound { frame, completed })+ .send(OutboundFrame {+ frame,+ parent,+ completed,+ }).map_err(|_| TransportError::Client("the sidecar connection is closed".to_owned()))?;result.await@@ -118,7 +126,7 @@ impl TransportSender<ToRuntimeMessage> for SidecarSender {/// Relay frames until either side closes.async fn pump<Socket, Observer>(socket: WebSocketStream<Socket>,- mut outbound: mpsc::UnboundedReceiver<Outbound>,+ mut outbound: mpsc::UnboundedReceiver<OutboundFrame>,inbound: mpsc::UnboundedSender<ToServerMessage>,on_frame: Observer,) where@@ -130,7 +138,10 @@ async fn pump<Socket, Observer>(loop {tokio::select! {outgoing = outbound.recv() => {- let Some(Outbound { frame, completed }) = outgoing else { break };+ let Some(OutboundFrame { frame, parent, completed }) = outgoing else { break };+ let method = frame_method(&frame);+ // Serialization failure is fatal for this session because the+ // caller's action cannot reach the agent.let json = match serde_json::to_string(&frame) {Ok(json) => json,Err(error) => {@@ -141,10 +152,32 @@ async fn pump<Socket, Observer>(continue;}};- if let Err(error) = socket_tx.send(Message::Text(json.into())).await {- let _ = completed.send(Err(TransportError::Client(error.to_string())));+ let send = socket_tx.send(Message::Text(json.into()));+ let span = tracing::info_span!(+ parent: &parent,+ "agent.acp.websocket_send",+ network.protocol.name = "websocket",+ rpc.system.name = tracing::field::Empty,+ rpc.method = tracing::field::Empty,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ if let Some(method) = method {+ span.record("rpc.system.name", "jsonrpc");+ span.record("rpc.method", method);+ }+ let result = send+ .instrument(span.clone())+ .await+ .map_err(|error| format!("sending an ACP websocket frame failed: {error}"));+ if let Err(error) = result {+ span.record("otel.status_code", "ERROR");+ span.record("otel.status_description", tracing::field::display(&error));+ drop(span);+ let _ = completed.send(Err(TransportError::Client(error)));break;}+ drop(span);let _ = completed.send(Ok(()));}incoming = socket_rx.next() => {@@ -181,3 +214,11 @@ async fn pump<Socket, Observer>(tracing::debug!("sidecar pump finished");}++fn frame_method(frame: &RawJsonRpcMessage) -> Option<&str> {+ match frame {+ RawJsonRpcMessage::Request(request) => Some(request.method.as_ref()),+ RawJsonRpcMessage::Notification(notification) => Some(notification.method.as_ref()),+ RawJsonRpcMessage::Response(_) => None,+ }+}diff --git a/crates/agent_harness/src/outbound/sidecar/test.rs b/crates/agent_harness/src/outbound/sidecar/test.rsindex 519ea9190d..0e8c8d7b24 100644--- a/crates/agent_harness/src/outbound/sidecar/test.rs+++ b/crates/agent_harness/src/outbound/sidecar/test.rs@@ -2,9 +2,9 @@ use agent_client_protocol::schema::v1::{PromptRequest, RequestId, SessionId};use agent_client_protocol::{JsonRpcMessage, RawJsonRpcMessage};use std::sync::Arc;use std::sync::atomic::{AtomicUsize, Ordering};-use tokio::net::{TcpListener, TcpStream};use super::*;+use tokio::net::{TcpListener, TcpStream};/// One ACP frame built from the protocol's typed request.fn frame() -> RawJsonRpcMessage {diff --git a/crates/agent_harness/src/testing/helpers/agent.rs b/crates/agent_harness/src/testing/helpers/agent.rsindex ce14175e93..3fed915fd3 100644--- a/crates/agent_harness/src/testing/helpers/agent.rs+++ b/crates/agent_harness/src/testing/helpers/agent.rs@@ -1,10 +1,11 @@//! A fake agent process, speaking only raw ACP.+use std::collections::VecDeque;use std::sync::{Arc, Mutex};use agent_client_protocol::schema::v1::{- ClientNotification, ClientRequest, InitializeResponse, NewSessionResponse, RequestId,- ResumeSessionResponse,+ ClientNotification, ClientRequest, InitializeResponse, NewSessionResponse, PromptResponse,+ RequestId, ResumeSessionResponse, StopReason,};use agent_client_protocol::{Error as AcpError, JsonRpcMessage, JsonRpcNotification, JsonRpcResponse, RawJsonRpcMessage,@@ -32,6 +33,7 @@ struct Progress {/// Requests received and not yet answered.initializing: Option<RequestId>,opening: Option<RequestId>,+ prompts: VecDeque<RequestId>,}/// Parse a frame into whichever ACP message enum the caller asked for.@@ -119,6 +121,21 @@ impl FakeAgent {self.sends_reply(id, response);}+ /// Wait for and complete the next `session/prompt` request.+ pub async fn completes_prompt(&self) {+ let mut received = self.received.subscribe();+ let id = loop {+ if let Some(id) = self.lock_progress().prompts.pop_front() {+ break id;+ }+ received+ .changed()+ .await+ .expect("fake agent frame history should remain open");+ };+ self.sends_reply(id, PromptResponse::new(StopReason::EndTurn));+ }+/// Refuse the `session/new` this agent received.////// # Panics@@ -248,6 +265,15 @@ impl FakeAgent {);progress.opening = Some(request.id.clone());}+ ClientRequest::PromptRequest(_) => {+ assert_eq!(+ progress.stage,+ Stage::SessionOpen,+ "harness sent {} before its ACP session existed",+ request.method+ );+ progress.prompts.push_back(request.id.clone());+ }other => assert_eq!(progress.stage,Stage::SessionOpen,diff --git a/crates/agent_runtime_protocol/Cargo.toml b/crates/agent_runtime_protocol/Cargo.tomlindex ca8554a6f0..546b170609 100644--- a/crates/agent_runtime_protocol/Cargo.toml+++ b/crates/agent_runtime_protocol/Cargo.toml@@ -24,6 +24,7 @@ serde_json = { workspace = true }specta = { workspace = true }specta-serde = { workspace = true }specta-typescript = { workspace = true }+strum = { workspace = true }thiserror = { workspace = true }tokio = { workspace = true, features = ["rt", "sync"], optional = true }tokio-tungstenite = { workspace = true, optional = true }diff --git a/crates/agent_runtime_protocol/src/domain/action.rs b/crates/agent_runtime_protocol/src/domain/action.rsindex 7a218cca42..e6e6241d41 100644--- a/crates/agent_runtime_protocol/src/domain/action.rs+++ b/crates/agent_runtime_protocol/src/domain/action.rs@@ -127,9 +127,10 @@ impl AgentSetModelAction {}/// One thing a caller wants an agent to do.-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, strum::AsRefStr)]#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]#[serde(tag = "type", rename_all = "camelCase")]+#[strum(serialize_all = "snake_case")]pub enum AgentAction {/// Send the agent a prompt.Prompt(AgentPromptAction),diff --git a/crates/agent_session/Cargo.toml b/crates/agent_session/Cargo.tomlindex cc8b485cf1..71a8c17972 100644--- a/crates/agent_session/Cargo.toml+++ b/crates/agent_session/Cargo.toml@@ -43,6 +43,7 @@ serde = { workspace = true }serde_json = { workspace = true }strum = { workspace = true }tokio = { workspace = true, features = ["rt", "sync"] }+tokio-util = { workspace = true, features = ["rt"] }tracing = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/agent_session/src/domain/error.rs b/crates/agent_session/src/domain/error.rsindex 7778d0c352..b233655576 100644--- a/crates/agent_session/src/domain/error.rs+++ b/crates/agent_session/src/domain/error.rs@@ -22,6 +22,10 @@ pub enum AgentSessionError {"agent session {0} cannot be restored because the agent supports neither session/resume nor session/load")]ResumeUnsupported(AgentSessionId),+ #[error("agent session {0} action delivery timed out")]+ DeliveryTimedOut(AgentSessionId),+ #[error("agent session {0} log persistence timed out")]+ LogTimedOut(AgentSessionId),#[error(transparent)]Transport(#[from] TransportError),#[error(transparent)]diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex 5f1d845875..61c2abb465 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -37,7 +37,9 @@ use dashmap::mapref::entry::Entry;use entity_access::domain::models::{EntityAccessReceipt, EntityType, OwnerAccessLevel};use macro_user_id::user_id::MacroUserIdStr;use macro_uuid::Uuid;-use tokio::sync::{mpsc, oneshot};+use tokio::sync::{Mutex, mpsc, oneshot, watch};+use tokio_util::{sync::CancellationToken, task::TaskTracker};+use tracing::instrument::WithSubscriber as _;use bots::domain::models::BotId;@@ -53,11 +55,54 @@ use super::ports::{AgentSessionRealtime, AgentSessionRepo, NoOpAgentSessionNameGenerator,};use super::session::actors::{SessionActor, SessionCommand, Stepped};+use super::session::{CloseReason, Input};/// Buffered not-yet-accepted commands per session actor.const COMMAND_BUFFER: usize = 1028;+/// Persistence may delay lifecycle teardown, but never indefinitely.+const SESSION_PERSIST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);++struct ActiveSession {+ commands: Option<mpsc::Sender<SessionCommand>>,+ stopped: watch::Receiver<bool>,+ marker: Arc<()>,+ deleting: bool,+ stopping: bool,+}++type ActiveSessions = DashMap<AgentSessionId, ActiveSession>;-type ActiveSessions = DashMap<AgentSessionId, mpsc::Sender<SessionCommand>>;+struct AttachReservation {+ active: Arc<ActiveSessions>,+ id: AgentSessionId,+ marker: Arc<()>,+ stopped: Option<watch::Sender<bool>>,+ committed: bool,+}++impl AttachReservation {+ fn commit(mut self) -> (Arc<()>, watch::Sender<bool>) {+ self.committed = true;+ (+ self.marker.clone(),+ self.stopped.take().expect("reservation owns stop signal"),+ )+ }+}++impl Drop for AttachReservation {+ fn drop(&mut self) {+ if self.committed {+ return;+ }+ if let Some(stopped) = self.stopped.take() {+ let _ = stopped.send(true);+ }+ self.active.remove_if(&self.id, |_, active| {+ Arc::ptr_eq(&active.marker, &self.marker) && !active.stopping+ });+ }+}/// Durable and live use cases for agent sessions.#[cfg_attr(feature = "test-utils", mockall::automock)]@@ -167,12 +212,16 @@ pub trait AgentSessionService: Send + Sync + 'static {/// `Folds` answers "what messages does this session's log derive" -/// `agent_fold` folding the log on read - and `Rt` streams each frame to/// whoever is watching the session's channel right now.+#[derive(Clone)]pub struct AgentSessionServiceImpl<R, Folds, Rt, Namer = NoOpAgentSessionNameGenerator> {repo: R,folds: Folds,realtime: Rt,name_generator: Namer,active: Arc<ActiveSessions>,+ tasks: TaskTracker,+ cancellation: CancellationToken,+ lifecycle: Arc<Mutex<()>>,}impl<R, Folds, Rt> AgentSessionServiceImpl<R, Folds, Rt> {@@ -189,6 +238,9 @@ impl<R, Folds, Rt> AgentSessionServiceImpl<R, Folds, Rt> {realtime,name_generator: NoOpAgentSessionNameGenerator,active: Arc::new(DashMap::new()),+ tasks: TaskTracker::new(),+ cancellation: CancellationToken::new(),+ lifecycle: Arc::new(Mutex::new(())),}}}@@ -206,29 +258,80 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {realtime: self.realtime,name_generator,active: self.active,+ tasks: self.tasks,+ cancellation: self.cancellation,+ lifecycle: self.lifecycle,}}- fn register_transport<Connector>(+ /// Stop active actors and wait for their tasks to release their transports.+ pub async fn shutdown(&self) {+ let lifecycle = self.lifecycle.lock().await;+ self.cancellation.cancel();+ for mut session in self.active.iter_mut() {+ session.commands.take();+ session.stopping = true;+ }+ self.tasks.close();+ drop(lifecycle);+ self.tasks.wait().await;+ self.active.clear();+ }++ async fn reserve_attach(&self, id: AgentSessionId) -> Result<AttachReservation> {+ let _lifecycle = self.lifecycle.lock().await;+ if self.cancellation.is_cancelled() {+ return Err(AgentSessionError::Disconnected(id));+ }+ let (stopped_tx, stopped) = watch::channel(false);+ let marker = Arc::new(());+ match self.active.entry(id) {+ Entry::Occupied(_) => Err(AgentSessionError::AlreadyConnected(id)),+ Entry::Vacant(entry) => {+ entry.insert(ActiveSession {+ commands: None,+ stopped,+ marker: marker.clone(),+ deleting: false,+ stopping: false,+ });+ Ok(AttachReservation {+ active: self.active.clone(),+ id,+ marker,+ stopped: Some(stopped_tx),+ committed: false,+ })+ }+ }+ }++ async fn activate_reserved<Connector>(&self,- id: AgentSessionId,- acp_session_id: Option<SessionId>,- workspace: String,+ session: AgentSession,attachment: RuntimeAttachment<Connector>,+ reservation: AttachReservation,) -> Result<()>whereR: AgentSessionRepo + AgentSessionLogRepo + Clone,Rt: AgentSessionRealtime + Clone + Send + Sync + 'static,Connector: AgentConnector,{+ let _lifecycle = self.lifecycle.lock().await;+ let id = session.id;+ if self.cancellation.is_cancelled() {+ return Err(AgentSessionError::Disconnected(id));+ }let (commands, command_rx) = mpsc::channel(COMMAND_BUFFER);-- match self.active.entry(id) {- Entry::Occupied(_) => return Err(AgentSessionError::AlreadyConnected(id)),- Entry::Vacant(entry) => {- entry.insert(commands.clone());- }+ let Some(mut active) = self.active.get_mut(&id) else {+ return Err(AgentSessionError::Disconnected(id));+ };+ if !Arc::ptr_eq(&active.marker, &reservation.marker) || active.stopping {+ return Err(AgentSessionError::Disconnected(id));}+ active.commands = Some(commands.clone());+ drop(active);+ let (marker, stopped_tx) = reservation.commit();// The actor owns this session's log writes, so it gets the live writer// rather than the bare repository - see module docs. Its fold starts@@ -238,14 +341,23 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {let logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());let actor = SessionActor::new(id,- acp_session_id,- workspace,+ session.acp_session_id,+ session.workspace,attachment.connector,logs,command_rx,attachment.handshake,);- tokio::spawn(run_session(actor, Arc::downgrade(&self.active), commands));+ self.tasks.spawn(+ run_session(+ actor,+ Arc::downgrade(&self.active),+ marker,+ stopped_tx,+ self.cancellation.clone(),+ )+ .with_current_subscriber(),+ );Ok(())}@@ -259,31 +371,70 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {let commands = self.active.get(&id)- .map(|entry| entry.value().clone())+ .and_then(|entry| entry.commands.clone()).ok_or(AgentSessionError::Disconnected(id))?;let (completed, result) = oneshot::channel();+ let span = tracing::info_span!(+ "agent.session.command",+ agent.session.id = %id,+ agent.action.name = action.as_ref(),+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );if commands.send(SessionCommand {user_id,action,action_id,completed,+ span,}).await.is_err(){- self.active- .remove_if(&id, |_, active| active.same_channel(&commands));return Err(AgentSessionError::Disconnected(id));}match result.await {Ok(result) => result,- Err(_) => {- self.active- .remove_if(&id, |_, active| active.same_channel(&commands));- Err(AgentSessionError::Disconnected(id))+ Err(_) => Err(AgentSessionError::Disconnected(id)),+ }+ }++ fn begin_stop(&self, id: AgentSessionId, deleting: bool) -> (watch::Receiver<bool>, Arc<()>) {+ match self.active.entry(id) {+ Entry::Occupied(mut entry) => {+ let active = entry.get_mut();+ let attaching = active.commands.is_none() && !active.stopping;+ active.commands.take();+ active.deleting |= deleting;+ active.stopping = true;+ let stopped = if attaching {+ let (_stopped_tx, stopped) = watch::channel(true);+ stopped+ } else {+ active.stopped.clone()+ };+ (stopped, active.marker.clone())}+ Entry::Vacant(entry) => {+ let (_stopped_tx, stopped) = watch::channel(true);+ let marker = Arc::new(());+ entry.insert(ActiveSession {+ commands: None,+ stopped: stopped.clone(),+ marker: marker.clone(),+ deleting,+ stopping: true,+ });+ (stopped, marker)+ }+ }+ }++ async fn wait_stopped(mut stopped: watch::Receiver<bool>) {+ if !*stopped.borrow() {+ let _ = stopped.wait_for(|value| *value).await;}}}@@ -341,27 +492,40 @@ where}async fn delete_session(&self, id: AgentSessionId) -> Result<()> {- self.repo.delete(id).await+ let (stopped, marker) = self.begin_stop(id, true);+ Self::wait_stopped(stopped).await;+ let result = self.repo.delete(id).await;+ self.active+ .remove_if(&id, |_, active| Arc::ptr_eq(&active.marker, &marker));+ result}async fn close_session(&self, id: AgentSessionId) -> Result<()> {// Dropping the sender is the whole operation: the actor's next step// reads `None` from its command channel, treats it as `Abandoned`, and// tears the transport down on its way out.- self.active.remove(&id);+ let (stopped, marker) = self.begin_stop(id, false);+ Self::wait_stopped(stopped).await;+ self.active.remove_if(&id, |_, active| {+ Arc::ptr_eq(&active.marker, &marker) && !active.deleting+ });Ok(())}async fn mark_disconnected(&self, id: AgentSessionId) -> Result<()> {let mut logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());- logs.append(AgentSessionLog {- agent_session_id: id,- user_id: None,- content: Message::ToServer(ToServerMessage::Event {- event: SystemEvent::Disconnected,+ tokio::time::timeout(+ SESSION_PERSIST_TIMEOUT,+ logs.append(AgentSessionLog {+ agent_session_id: id,+ user_id: None,+ content: Message::ToServer(ToServerMessage::Event {+ event: SystemEvent::Disconnected,+ }),}),- })+ ).await+ .unwrap_or(Err(AgentSessionError::LogTimedOut(id)))}async fn attach_session<Connector>(@@ -372,13 +536,10 @@ wherewhereConnector: AgentConnector,{+ let reservation = self.reserve_attach(id).await?;let session = self.repo.get(id).await?;- self.register_transport(- session.id,- session.acp_session_id,- session.workspace,- attachment,- )+ self.activate_reserved(session, attachment, reservation)+ .await}async fn send_action(@@ -758,12 +919,24 @@ whereasync fn run_session<Connector, Logs>(mut actor: SessionActor<Connector, Logs>,active: std::sync::Weak<ActiveSessions>,- commands: mpsc::Sender<SessionCommand>,+ marker: Arc<()>,+ stopped: watch::Sender<bool>,+ cancellation: CancellationToken,) whereConnector: AgentConnector,Logs: AgentSessionLogWriter + AgentSessionRepo,{- while actor.step().await == Stepped::Continue {}+ loop {+ let input = tokio::select! {+ biased;+ () = cancellation.cancelled() => Input::Closed(CloseReason::Abandoned),+ input = actor.next_input() => input,+ };+ let stepped = actor.dispatch(input).await;+ if stepped == Stepped::Stopped {+ break;+ }+ }// Refuse late commands before releasing the registry entry, so a caller// cannot enqueue into an actor that will never step again.@@ -772,7 +945,10 @@ async fn run_session<Connector, Logs>(// Tear down the old transport before allowing another actor to attach.drop(actor);+ let _ = stopped.send(true);if let Some(active) = active.upgrade() {- active.remove_if(&id, |_, current| current.same_channel(&commands));+ active.remove_if(&id, |_, current| {+ current.commands.is_some() && Arc::ptr_eq(¤t.marker, &marker)+ });}}diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rsindex 5ec04927a2..e98d85cb48 100644--- a/crates/agent_session/src/domain/service/test.rs+++ b/crates/agent_session/src/domain/service/test.rs@@ -1,14 +1,22 @@use super::*;-use crate::domain::model::{DEFAULT_AGENT_SESSION_NAME, Message};+use crate::PROTOCOL_VERSION;+use crate::domain::model::{DEFAULT_AGENT_SESSION_NAME, Message, SessionBot};use crate::domain::ports::NoOpRealtime;+use crate::domain::session::HandshakeStatus;use crate::testing::{InMemoryAgentSessionRepo, RecordingRealtime, test_agent_session};use agent_fold::domain::fold::fold;use agent_fold::domain::service::FoldedMessageService;use agent_fold::testing::{TURN, parse_log_as, test_session};-use agent_runtime_protocol::domain::schema::v0::ToServerMessage;+use agent_runtime_protocol::domain::ports::{+ Transport, TransportError, TransportReceiver, TransportSender,+};+use agent_runtime_protocol::domain::schema::v0::ToRuntimeMessage;+use agent_runtime_protocol::domain::schema::v0::{AcpMessage, ToServerMessage};use entity_access::domain::models::{EntityAccessReceipt, EntityType, OwnerAccessLevel};use macro_uuid::Uuid;use std::sync::{Arc, Mutex};+use tokio::sync::Notify;+use tracing::instrument::WithSubscriber as _;struct Fixture {service: AgentSessionServiceImpl<@@ -92,6 +100,64 @@ impl AgentSessionRealtime for RenameRealtime {}}+struct PendingTransport;++#[derive(Clone)]+struct PendingSender;++struct PendingReceiver;++struct RecordingTransport {+ outbound: mpsc::Sender<ToRuntimeMessage>,+ inbound: mpsc::Receiver<ToServerMessage>,+}++#[derive(Clone)]+struct RecordingSender(mpsc::Sender<ToRuntimeMessage>);++impl Transport<ToRuntimeMessage, ToServerMessage> for RecordingTransport {+ type Sender = RecordingSender;+ type Receiver = mpsc::Receiver<ToServerMessage>;++ fn split(self) -> (Self::Sender, Self::Receiver) {+ (RecordingSender(self.outbound), self.inbound)+ }+}++impl TransportSender<ToRuntimeMessage> for RecordingSender {+ async fn send(&self, message: ToRuntimeMessage) -> std::result::Result<(), TransportError> {+ self.0+ .send(message)+ .await+ .map_err(|_| TransportError::Client("test receiver closed".to_owned()))+ }+}++#[derive(Clone)]+struct BlockingPromptLogs {+ repo: InMemoryAgentSessionRepo,+ entered: Arc<Notify>,+ release: Arc<Notify>,+ hang_disconnect: bool,+}++impl AgentSessionLogWriter for BlockingPromptLogs {+ async fn append(&mut self, log: AgentSessionLog) -> Result<()> {+ let is_prompt = matches!(+ &log.content,+ Message::ToRuntime(ToRuntimeMessage::Acp(AcpMessage(+ agent_client_protocol::RawJsonRpcMessage::Request(request)+ ))) if request.method.as_ref() == "session/prompt"+ );+ if is_prompt {+ self.entered.notify_one();+ self.release.notified().await;+ }+ AgentSessionLogRepo::create(&self.repo, log).await?;+ Ok(())+ }+}+fn owner_access(session: AgentSessionId) -> EntityAccessReceipt<OwnerAccessLevel> {EntityAccessReceipt::dangerously_assert_internal_user(&session.as_uuid().to_string(),@@ -257,6 +323,420 @@ async fn background_naming_does_not_overwrite_a_manual_name() {);}+impl AgentSessionRepo for BlockingPromptLogs {+ async fn create(&self, params: CreateAgentSessionParams) -> Result<AgentSession> {+ AgentSessionRepo::create(&self.repo, params).await+ }++ async fn get(&self, id: AgentSessionId) -> Result<AgentSession> {+ self.repo.get(id).await+ }++ async fn session_bot(&self, id: BotId) -> Result<SessionBot> {+ self.repo.session_bot(id).await+ }++ async fn find_for_channel(+ &self,+ thread_id: Option<Uuid>,+ bot_id: Option<BotId>,+ ) -> Result<ChannelSession> {+ self.repo.find_for_channel(thread_id, bot_id).await+ }++ async fn set_acp_session_id(+ &self,+ id: AgentSessionId,+ acp_session_id: SessionId,+ ) -> Result<()> {+ self.repo.set_acp_session_id(id, acp_session_id).await+ }++ async fn set_model(&self, id: AgentSessionId, model: &str) -> Result<()> {+ self.repo.set_model(id, model).await+ }++ async fn set_name(&self, id: AgentSessionId, name: &str) -> Result<()> {+ self.repo.set_name(id, name).await+ }++ async fn set_name_if_default(&self, id: AgentSessionId, name: &str) -> Result<bool> {+ self.repo.set_name_if_default(id, name).await+ }++ async fn set_sandbox_size(&self, id: AgentSessionId, size: SandboxSize) -> Result<()> {+ self.repo.set_sandbox_size(id, size).await+ }++ async fn user_sandbox_size(&self, user_id: &MacroUserIdStr<'static>) -> Result<SandboxSize> {+ self.repo.user_sandbox_size(user_id).await+ }++ async fn set_user_sandbox_size(+ &self,+ user_id: &MacroUserIdStr<'static>,+ size: SandboxSize,+ ) -> Result<()> {+ self.repo.set_user_sandbox_size(user_id, size).await+ }++ async fn delete(&self, id: AgentSessionId) -> Result<()> {+ self.repo.delete(id).await+ }+}++impl AgentSessionLogRepo for BlockingPromptLogs {+ async fn create(&self, log: AgentSessionLog) -> Result<StoredAgentSessionLog> {+ if self.hang_disconnect+ && matches!(+ &log.content,+ Message::ToServer(ToServerMessage::Event {+ event: SystemEvent::Disconnected+ })+ )+ {+ return std::future::pending().await;+ }+ AgentSessionLogRepo::create(&self.repo, log).await+ }++ async fn list_by_session(+ &self,+ agent_session_id: AgentSessionId,+ ) -> Result<Vec<StoredAgentSessionLog>> {+ AgentSessionLogRepo::list_by_session(&self.repo, agent_session_id).await+ }+}++impl Transport<ToRuntimeMessage, ToServerMessage> for PendingTransport {+ type Sender = PendingSender;+ type Receiver = PendingReceiver;++ fn split(self) -> (Self::Sender, Self::Receiver) {+ (PendingSender, PendingReceiver)+ }+}++impl TransportSender<ToRuntimeMessage> for PendingSender {+ async fn send(&self, _message: ToRuntimeMessage) -> std::result::Result<(), TransportError> {+ Ok(())+ }+}++impl TransportReceiver<ToServerMessage> for PendingReceiver {+ async fn recv(&mut self) -> std::result::Result<Option<ToServerMessage>, TransportError> {+ std::future::pending().await+ }+}++async fn open_test_session(+ inbound: &mpsc::Sender<ToServerMessage>,+ outbound: &mut mpsc::Receiver<ToRuntimeMessage>,+ session: AgentSessionId,+) {+ inbound+ .send(ToServerMessage::Event {+ event: SystemEvent::AcpReady,+ })+ .await+ .unwrap();+ let _initialize = outbound.recv().await.expect("initialize request");+ inbound+ .send(ToServerMessage::Acp(AcpMessage(+ agent_client_protocol::RawJsonRpcMessage::response(+ agent_client_protocol::schema::v1::RequestId::Str(format!(+ "agent_session:{session}:0"+ )),+ Ok(serde_json::to_value(+ agent_client_protocol::schema::v1::InitializeResponse::new(PROTOCOL_VERSION),+ )+ .unwrap()),+ ),+ )))+ .await+ .unwrap();+ let _open = outbound.recv().await.expect("session/new request");+ inbound+ .send(ToServerMessage::Acp(AcpMessage(+ agent_client_protocol::RawJsonRpcMessage::response(+ agent_client_protocol::schema::v1::RequestId::Str(format!(+ "agent_session:{session}:1"+ )),+ Ok(serde_json::to_value(+ agent_client_protocol::schema::v1::NewSessionResponse::new("acp-1"),+ )+ .unwrap()),+ ),+ )))+ .await+ .unwrap();+}++#[tokio::test]+async fn a_stopping_lifecycle_entry_blocks_a_second_attach() {+ let fx = fixture();+ let (_stopped, marker) = fx.service.begin_stop(fx.session, false);++ let result = fx+ .service+ .attach_session(fx.session, RuntimeAttachment::solo(PendingTransport))+ .await;++ assert!(matches!(result, Err(AgentSessionError::AlreadyConnected(id)) if id == fx.session));+ fx.service.active.remove_if(&fx.session, |_, active| {+ Arc::ptr_eq(&active.marker, &marker)+ });+}++#[tokio::test]+async fn close_claiming_an_attach_reservation_prevents_actor_start() {+ let fx = fixture();+ let reservation = fx+ .service+ .reserve_attach(fx.session)+ .await+ .expect("attach reserves before reading");+ let session = fx.repo.get(fx.session).await.expect("session exists");+ let (stopped, marker) = fx.service.begin_stop(fx.session, false);++ let result = fx+ .service+ .activate_reserved(+ session,+ RuntimeAttachment::solo(PendingTransport),+ reservation,+ )+ .await;+ AgentSessionServiceImpl::<+ InMemoryAgentSessionRepo,+ FoldedMessageService<InMemoryAgentSessionRepo>,+ NoOpRealtime,+ >::wait_stopped(stopped)+ .await;++ assert!(matches!(result, Err(AgentSessionError::Disconnected(id)) if id == fx.session));+ assert!(+ fx.service+ .active+ .get(&fx.session)+ .unwrap()+ .commands+ .is_none()+ );+ fx.service.active.remove_if(&fx.session, |_, active| {+ Arc::ptr_eq(&active.marker, &marker)+ });+}++#[tokio::test]+async fn shutdown_prevents_a_reserved_attach_from_spawning() {+ let fx = fixture();+ let reservation = fx+ .service+ .reserve_attach(fx.session)+ .await+ .expect("attach reserves before reading");+ let session = fx.repo.get(fx.session).await.expect("session exists");++ fx.service.shutdown().await;+ let result = fx+ .service+ .activate_reserved(+ session,+ RuntimeAttachment::solo(PendingTransport),+ reservation,+ )+ .await;++ assert!(matches!(result, Err(AgentSessionError::Disconnected(id)) if id == fx.session));+ assert!(fx.service.tasks.is_closed());+}++#[tokio::test]+async fn close_does_not_remove_a_concurrent_delete_guard() {+ let fx = fixture();+ let (_stopped, marker) = fx.service.begin_stop(fx.session, true);++ fx.service+ .close_session(fx.session)+ .await+ .expect("close observes the stopped actor");++ assert!(fx.service.active.contains_key(&fx.session));+ fx.service.active.remove_if(&fx.session, |_, active| {+ Arc::ptr_eq(&active.marker, &marker)+ });+}++#[tokio::test]+async fn cancellation_does_not_drop_an_effect_batch_after_machine_mutation() {+ let repo = InMemoryAgentSessionRepo::new();+ let session = test_session();+ repo.insert_session(test_agent_session(session));+ let entered = Arc::new(Notify::new());+ let release = Arc::new(Notify::new());+ let logs = BlockingPromptLogs {+ repo: repo.clone(),+ entered: entered.clone(),+ release: release.clone(),+ hang_disconnect: false,+ };+ let (outbound_tx, mut outbound_rx) = mpsc::channel(8);+ let (inbound_tx, inbound_rx) = mpsc::channel(8);+ let (commands, command_rx) = mpsc::channel(8);+ let (handshake, _) = watch::channel(HandshakeStatus::Pending);+ let actor = SessionActor::new(+ session,+ None,+ "/workspace".to_owned(),+ RecordingTransport {+ outbound: outbound_tx,+ inbound: inbound_rx,+ },+ logs,+ command_rx,+ handshake,+ );+ let active = Arc::new(ActiveSessions::new());+ let cancellation = CancellationToken::new();+ let marker = Arc::new(());+ let (stopped_tx, _) = watch::channel(false);+ let task = tokio::spawn(run_session(+ actor,+ Arc::downgrade(&active),+ marker,+ stopped_tx,+ cancellation.clone(),+ ));++ open_test_session(&inbound_tx, &mut outbound_rx, session).await;++ let (completed, result) = oneshot::channel();+ commands+ .send(SessionCommand {+ user_id: None,+ action: AgentAction::prompt("keep dispatching"),+ action_id: AgentActionId::mint(),+ completed,+ span: tracing::info_span!("test.command"),+ })+ .await+ .unwrap();+ entered.notified().await;+ cancellation.cancel();+ release.notify_one();++ let prompt = outbound_rx+ .recv()+ .await+ .expect("prompt is still dispatched");+ assert!(matches!(+ prompt,+ ToRuntimeMessage::Acp(AcpMessage(+ agent_client_protocol::RawJsonRpcMessage::Request(request)+ )) if request.method.as_ref() == "session/prompt"+ ));+ result.await.unwrap().expect("delivery completes");+ task.await.unwrap();+}++#[tokio::test(start_paused = true)]+async fn live_inbound_logs_do_not_reuse_the_expired_handshake_deadline() {+ let repo = InMemoryAgentSessionRepo::new();+ let session = test_session();+ repo.insert_session(test_agent_session(session));+ let entered = Arc::new(Notify::new());+ let release = Arc::new(Notify::new());+ let logs = BlockingPromptLogs {+ repo: repo.clone(),+ entered: entered.clone(),+ release: release.clone(),+ hang_disconnect: false,+ };+ let (outbound_tx, mut outbound_rx) = mpsc::channel(8);+ let (inbound_tx, inbound_rx) = mpsc::channel(8);+ let (commands, command_rx) = mpsc::channel(8);+ let (handshake, _) = watch::channel(HandshakeStatus::Pending);+ let actor = SessionActor::new(+ session,+ None,+ "/workspace".to_owned(),+ RecordingTransport {+ outbound: outbound_tx,+ inbound: inbound_rx,+ },+ logs,+ command_rx,+ handshake,+ );+ let active = Arc::new(ActiveSessions::new());+ let cancellation = CancellationToken::new();+ let (stopped_tx, _) = watch::channel(false);+ let task = tokio::spawn(+ run_session(+ actor,+ Arc::downgrade(&active),+ Arc::new(()),+ stopped_tx,+ cancellation.clone(),+ )+ .with_current_subscriber(),+ );+ open_test_session(&inbound_tx, &mut outbound_rx, session).await;++ release.notify_one();+ let (completed, result) = oneshot::channel();+ commands+ .send(SessionCommand {+ user_id: None,+ action: AgentAction::prompt("keep working"),+ action_id: AgentActionId::mint(),+ completed,+ span: tracing::info_span!("test.command"),+ })+ .await+ .unwrap();+ entered.notified().await;+ let _prompt = outbound_rx.recv().await.expect("prompt request");+ result.await.unwrap().expect("prompt delivered");++ tokio::time::advance(std::time::Duration::from_secs(61)).await;+ inbound_tx+ .send(ToServerMessage::Acp(AcpMessage(+ agent_client_protocol::RawJsonRpcMessage::notification(+ "session/update".to_owned(),+ serde_json::json!({ "sessionId": "acp-1", "update": {} }),+ )+ .unwrap(),+ )))+ .await+ .unwrap();++ let mut persisted = false;+ for _ in 0..20 {+ tokio::task::yield_now().await;+ persisted = AgentSessionLogRepo::list_by_session(&repo, session)+ .await+ .unwrap()+ .iter()+ .any(|stored| {+ matches!(+ &stored.entry.content,+ Message::ToServer(ToServerMessage::Acp(AcpMessage(+ agent_client_protocol::RawJsonRpcMessage::Notification(notification)+ ))) if notification.method.as_ref() == "session/update"+ )+ });+ if persisted {+ break;+ }+ }+ assert!(persisted, "live update should use the regular log timeout");++ cancellation.cancel();+ task.await.unwrap();+}+/// Any protocol frame will do: the service only stores it, turn detection is/// the fold's answer.fn any_event(session: AgentSessionId) -> AgentSessionLog {@@ -346,6 +826,29 @@ async fn marking_disconnected_persists_and_publishes_the_event() {assert_eq!(realtime.published().len(), 1);}+#[tokio::test(start_paused = true)]+async fn marking_disconnected_is_bounded_when_persistence_hangs() {+ let repo = InMemoryAgentSessionRepo::new();+ let session = test_session();+ repo.insert_session(test_agent_session(session));+ let hanging = BlockingPromptLogs {+ repo: repo.clone(),+ entered: Arc::new(Notify::new()),+ release: Arc::new(Notify::new()),+ hang_disconnect: true,+ };+ let service =+ AgentSessionServiceImpl::new(hanging, FoldedMessageService::new(repo), NoOpRealtime);+ let disconnect = tokio::spawn(async move { service.mark_disconnected(session).await });+ tokio::task::yield_now().await;+ tokio::time::advance(SESSION_PERSIST_TIMEOUT).await;++ assert!(matches!(+ disconnect.await.unwrap(),+ Err(AgentSessionError::LogTimedOut(id)) if id == session+ ));+}+/// The point of the rework: a connection folds its session once, when it/// starts, and every frame after that is folded into the state it kept.///diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rsindex 41e49d5c46..d26df5f32f 100644--- a/crates/agent_session/src/domain/session/actors.rs+++ b/crates/agent_session/src/domain/session/actors.rs@@ -9,25 +9,34 @@//! not sent.//!//! Stepping is the caller's choice: production spawns a "step until stopped"-//! loop (see [`super::super::service`]), while a test can construct an actor-//! and crank it by hand - after every [`SessionActor::step`] the world has-//! fully caught up with that input.+//! loop (see [`super::super::service`]), while tests exercise its input and+//! dispatch halves directly.-use std::collections::VecDeque;+use std::{collections::VecDeque, time::Duration};+use agent_client_protocol::RawJsonRpcMessage;use agent_client_protocol::schema::v1::SessionId;use agent_runtime_protocol::domain::action::{AgentAction, AgentActionId};-use agent_runtime_protocol::domain::schema::v0::{SystemEvent, ToRuntimeMessage, ToServerMessage};+use agent_runtime_protocol::domain::schema::v0::{+ AcpMessage, SystemEvent, ToRuntimeMessage, ToServerMessage,+};use macro_user_id::user_id::MacroUserIdStr;use tokio::sync::{mpsc, oneshot, watch};+use tokio::time::Instant;+use tracing::Instrument as _;-use crate::domain::error::Result;+use crate::domain::error::{AgentSessionError, Result};use crate::domain::model::{AgentSessionId, AgentSessionLog, Message};use agent_runtime_protocol::domain::ports::{TransportReceiver, TransportSender};use crate::domain::ports::{AgentConnector, AgentSessionLogWriter, AgentSessionRepo};-use super::{CloseReason, Effect, HandshakeStatus, Input, SessionMachine};+use super::{CloseReason, Effect, HandshakeStatus, Input, RuntimeStatus, SessionMachine};++/// How long the ACP handshake has to finish before the session is declared dead.+const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(60);+/// How long one caller's action has to reach the runtime.+const COMMAND_DELIVERY_TIMEOUT: Duration = Duration::from_secs(60);/// A caller's request to deliver one action, and the wire back to them.pub(crate) struct SessionCommand {@@ -35,6 +44,12 @@ pub(crate) struct SessionCommand {pub(crate) action: AgentAction,pub(crate) action_id: AgentActionId,pub(crate) completed: oneshot::Sender<Result<()>>,+ pub(crate) span: tracing::Span,+}++pub(crate) struct SessionCompletion {+ completed: oneshot::Sender<Result<()>>,+ span: tracing::Span,}/// Whether a [`SessionActor`] has more steps to take.@@ -49,7 +64,7 @@ pub(crate) enum Stepped {/// One session connection's imperative shell: pulls one input, runs the/// machine, and executes the effects - nothing else.pub(crate) struct SessionActor<Connector: AgentConnector, Logs> {- machine: SessionMachine<oneshot::Sender<Result<()>>>,+ machine: SessionMachine<SessionCompletion>,/// The carrier's halves. Sending is shared with whoever else is on the/// same connection; receiving is this actor's alone, which is why it can/// be read with `&mut` and needs no lock.@@ -61,6 +76,10 @@ pub(crate) struct SessionActor<Connector: AgentConnector, Logs> {/// the handshake, watched for when another session runs it.handshake: watch::Sender<HandshakeStatus>,handshake_seen: watch::Receiver<HandshakeStatus>,+ /// The span covering the handshake, dropped once the runtime is live or+ /// dead. Every effect run before then hangs off it.+ handshake_span: Option<tracing::Span>,+ handshake_deadline: Instant,}impl<Connector, Logs> SessionActor<Connector, Logs>@@ -84,6 +103,12 @@ wherehandshake_seen.mark_changed();let (outbound, inbound) = connector.split();+ let handshake_span = tracing::info_span!(+ "agent.acp.handshake",+ agent.session.id = %id,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );Self {outbound,@@ -96,6 +121,8 @@ where},logs,commands,+ handshake_span: Some(handshake_span),+ handshake_deadline: Instant::now() + HANDSHAKE_TIMEOUT,}}@@ -110,15 +137,30 @@ whereself.commands.close();}- /// Wait for one input and execute every effect it produces.- pub(crate) async fn step(&mut self) -> Stepped {- let input = tokio::select! {+ /// Wait for the next input without mutating protocol state. Keeping this+ /// separate from dispatch lets shutdown cancel the wait, never an effect+ /// batch after the machine has already advanced.+ pub(crate) async fn next_input(&mut self) -> Input<SessionCompletion> {+ loop {+ let handshake_deadline = self.handshake_deadline;+ let handshake_timeout = async {+ if matches!(+ self.machine.status(),+ RuntimeStatus::Booting | RuntimeStatus::Handshaking+ ) {+ tokio::time::sleep_until(handshake_deadline).await;+ } else {+ std::future::pending::<()>().await;+ }+ };+ let input = tokio::select! {+ () = handshake_timeout => Input::Closed(CloseReason::HandshakeTimedOut),command = self.commands.recv() => match command {- Some(SessionCommand { user_id, action, action_id, completed }) => Input::Command {+ Some(SessionCommand { user_id, action, action_id, completed, span }) => Input::Command {from: user_id,action,action_id,- token: completed,+ token: SessionCompletion { completed, span },},// The service dropped every handle; nobody can reach us.None => Input::Closed(CloseReason::Abandoned),@@ -129,7 +171,7 @@ whereOk(()) = self.handshake_seen.changed() => match *self.handshake_seen.borrow_and_update() {HandshakeStatus::Ready(restore) => Input::Ready { restore },// Nothing to act on yet, but the wait must resume.- HandshakeStatus::Pending | HandshakeStatus::InFlight => return Stepped::Continue,+ HandshakeStatus::Pending | HandshakeStatus::InFlight => continue,},inbound = self.inbound.recv() => match inbound {Ok(Some(message)) => Input::Inbound(message),@@ -139,48 +181,137 @@ whereInput::Closed(CloseReason::TransportFailed)}},- };-- self.dispatch(input).await+ };+ return input;+ }}/// Run the machine and execute its effects in order. An effect that fails/// to execute voids the rest of its batch and feeds [`Input::Closed`]/// back in - so a `Complete` never fires for an action that was not sent,/// and the machine (not this loop) decides what failure means.- async fn dispatch(&mut self, input: Input<oneshot::Sender<Result<()>>>) -> Stepped {- let mut effects = VecDeque::from(self.machine.handle(input));+ pub(crate) async fn dispatch(&mut self, input: Input<SessionCompletion>) -> Stepped {+ let handshake_deadline = self.handshake_deadline;+ let mut handshake_span = self.handshake_span.clone();+ let produced = match &handshake_span {+ Some(span) => span.in_scope(|| self.machine.handle(input)),+ None => self.machine.handle(input),+ };+ let mut effects = VecDeque::from(produced);+ self.finish_handshake_if_complete();+ if self.handshake_span.is_none() {+ handshake_span = None;+ }let mut stepped = Stepped::Continue;while let Some(effect) = effects.pop_front() {match effect {Effect::Send { from, message } => {- if let Err(error) = self.deliver(from, message).await {+ let command_span = effects.front().and_then(|effect| match effect {+ Effect::Complete { token, .. } => Some(token.span.clone()),+ _ => None,+ });+ let delivery = self.deliver(from, message);+ let (result, close_reason) =+ match (command_span.as_ref(), handshake_span.as_ref()) {+ (Some(span), _) => match tokio::time::timeout(+ COMMAND_DELIVERY_TIMEOUT,+ delivery.instrument(span.clone()),+ )+ .await+ {+ Ok(result) => (result, CloseReason::SendFailed),+ Err(_) => (+ Err(AgentSessionError::DeliveryTimedOut(self.machine.id())),+ CloseReason::SendFailed,+ ),+ },+ (None, Some(span)) => match tokio::time::timeout_at(+ handshake_deadline,+ delivery.instrument(span.clone()),+ )+ .await+ {+ Ok(result) => (result, CloseReason::SendFailed),+ Err(_) => (+ Err(AgentSessionError::Handshake(format!(+ "timed out after {} seconds",+ HANDSHAKE_TIMEOUT.as_secs()+ ))),+ CloseReason::HandshakeTimedOut,+ ),+ },+ (None, None) => {+ match tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, delivery).await+ {+ Ok(result) => (result, CloseReason::SendFailed),+ Err(_) => (+ Err(AgentSessionError::DeliveryTimedOut(self.machine.id())),+ CloseReason::SendFailed,+ ),+ }+ }+ };+ if let Err(error) = result {+ if let Some(span) = command_span {+ span.record("otel.status_code", "ERROR");+ span.record("otel.status_description", tracing::field::display(&error));+ }tracing::error!(error = ?error,id = %self.machine.id(),"agent session failed to deliver an action");- if let Some(Effect::Complete { token, .. }) = effects.pop_front() {- let _ = token.send(Err(error));- }- effects.clear();- effects.extend(self.machine.handle(Input::Closed(CloseReason::SendFailed)));+ self.fail_remaining_completions(&mut effects, error);+ effects.extend(self.machine.handle(Input::Closed(close_reason)));+ self.finish_handshake_if_complete();}}Effect::Log { message } => {- if let Err(error) = self.log(None, Message::ToServer(message)).await {+ let log = self.log(None, Message::ToServer(message));+ let (result, close_reason) = match handshake_span.as_ref() {+ Some(span) => match tokio::time::timeout_at(+ handshake_deadline,+ log.instrument(span.clone()),+ )+ .await+ {+ Ok(result) => (result, CloseReason::LogFailed),+ Err(_) => (+ Err(AgentSessionError::Handshake(format!(+ "timed out after {} seconds",+ HANDSHAKE_TIMEOUT.as_secs()+ ))),+ CloseReason::HandshakeTimedOut,+ ),+ },+ None => match tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, log).await {+ Ok(result) => (result, CloseReason::LogFailed),+ Err(_) => (+ Err(AgentSessionError::LogTimedOut(self.machine.id())),+ CloseReason::LogFailed,+ ),+ },+ };+ if let Err(error) = result {tracing::error!(error = ?error,id = %self.machine.id(),"agent session failed to persist an inbound message");- effects.clear();- effects.extend(self.machine.handle(Input::Closed(CloseReason::LogFailed)));+ self.fail_remaining_completions(&mut effects, error);+ effects.extend(self.machine.handle(Input::Closed(close_reason)));+ self.finish_handshake_if_complete();}}Effect::PersistAcpSession { session_id } => {- if let Err(error) = self.persist_acp_session(session_id).await {+ let result = tokio::time::timeout(+ COMMAND_DELIVERY_TIMEOUT,+ self.persist_acp_session(session_id),+ )+ .await+ .unwrap_or_else(|_| Err(AgentSessionError::LogTimedOut(self.machine.id())));+ if let Err(error) = result {tracing::error!(error = ?error,id = %self.machine.id(),@@ -197,24 +328,38 @@ wherelet _ = self.handshake.send(HandshakeStatus::Ready(restore));}Effect::Complete { token, result } => {- let _ = token.send(result);+ if let Err(error) = &result {+ token.span.record("otel.status_code", "ERROR");+ token+ .span+ .record("otel.status_description", tracing::field::display(error));+ }+ let _ = token.completed.send(result);}Effect::Stop { reason } => {- self.log(+ let terminal_log = self.log(None,Message::ToServer(ToServerMessage::Event {event: SystemEvent::Disconnected,}),- )- .await- .inspect_err(|error| {- tracing::error!(- error = ?error,- id = %self.machine.id(),- "agent session failed to persist its disconnect"- );- })- .ok();+ );+ let result = tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, terminal_log).await;+ match result {+ Ok(Ok(())) => {}+ Ok(Err(error)) => {+ tracing::error!(+ error = ?error,+ id = %self.machine.id(),+ "agent session failed to persist its disconnect"+ );+ }+ Err(_) => {+ tracing::error!(+ id = %self.machine.id(),+ "agent session timed out persisting its disconnect"+ );+ }+ }tracing::info!(id = %self.machine.id(), %reason, "agent session stopped");stepped = Stepped::Stopped;}@@ -226,11 +371,25 @@ where/// Log then send: the log entry is written first so the session's history/// never lacks a message its agent received.+ #[tracing::instrument(+ name = "agent.acp.deliver",+ err,+ skip(self, from, message),+ fields(+ agent.session.id = %self.machine.id(),+ rpc.system.name = tracing::field::Empty,+ rpc.method = tracing::field::Empty,+ )+ )]async fn deliver(&mut self,from: Option<MacroUserIdStr<'static>>,message: ToRuntimeMessage,) -> Result<()> {+ if let Some(method) = acp_method(&message) {+ tracing::Span::current().record("rpc.system.name", "jsonrpc");+ tracing::Span::current().record("rpc.method", method);+ }self.log(from, Message::ToRuntime(message.clone())).await?;self.outbound.send(message).await?;Ok(())@@ -255,4 +414,63 @@ where.set_acp_session_id(self.machine.id(), acp_session_id).await}++ fn finish_handshake_if_complete(&mut self) {+ match self.machine.status() {+ RuntimeStatus::Live { .. } => {+ self.handshake_span.take();+ }+ RuntimeStatus::Dead => {+ if let Some(span) = self.handshake_span.take() {+ span.record("otel.status_code", "ERROR");+ span.record("otel.status_description", "ACP handshake failed");+ }+ }+ RuntimeStatus::Booting | RuntimeStatus::Handshaking => {}+ }+ }++ /// The failing effect's own caller gets `cause` itself; anything still+ /// queued behind it never reached the runtime, so it gets `Disconnected`.+ fn fail_remaining_completions(+ &self,+ effects: &mut VecDeque<Effect<SessionCompletion>>,+ cause: AgentSessionError,+ ) {+ let mut stop = None;+ let mut cause = Some(cause);+ while let Some(effect) = effects.pop_front() {+ match effect {+ Effect::Complete { token, .. } => {+ let error = cause+ .take()+ .unwrap_or_else(|| AgentSessionError::Disconnected(self.machine.id()));+ token.span.record("otel.status_code", "ERROR");+ token+ .span+ .record("otel.status_description", tracing::field::display(&error));+ let _ = token.completed.send(Err(error));+ }+ effect @ Effect::Stop { .. } => stop = Some(effect),+ Effect::Send { .. }+ | Effect::Log { .. }+ | Effect::PersistAcpSession { .. }+ | Effect::Initialized { .. } => {}+ }+ }+ if let Some(stop) = stop {+ effects.push_back(stop);+ }+ }+}++fn acp_method(message: &ToRuntimeMessage) -> Option<&str> {+ let ToRuntimeMessage::Acp(AcpMessage(frame)) = message else {+ return None;+ };+ match frame {+ RawJsonRpcMessage::Request(request) => Some(request.method.as_ref()),+ RawJsonRpcMessage::Notification(notification) => Some(notification.method.as_ref()),+ RawJsonRpcMessage::Response(_) => None,+ }}diff --git a/crates/agent_session/src/domain/session/tests.rs b/crates/agent_session/src/domain/session/tests.rsindex 6995026be5..0a28dc0546 100644--- a/crates/agent_session/src/domain/session/tests.rs+++ b/crates/agent_session/src/domain/session/tests.rs@@ -110,6 +110,13 @@ fn sent_methods(effects: &[Effect<u32>]) -> Vec<String> {message: ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Request(request))),..} => Some(request.method.to_string()),+ Effect::Send {+ message:+ ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Notification(+ notification,+ ))),+ ..+ } => Some(notification.method.to_string()),_ => None,}).collect()@@ -192,7 +199,7 @@ fn a_second_acp_ready_only_logs() {}#[test]-fn session_new_success_flushes_the_queue_positionally() {+fn session_new_success_flushes_the_queue() {let mut machine = machine();machine.handle(command("first", 1));machine.handle(command("second", 2));@@ -200,8 +207,6 @@ fn session_new_success_flushes_the_queue_positionally() {let effects = machine.handle(session_opened("acp-42"));- // Each action's completion directly follows its send: delivery is- // positional, not counted.assert!(matches!(effects[..],[diff --git a/crates/agent_session/src/domain/session/types.rs b/crates/agent_session/src/domain/session/types.rsindex 3492cc4595..179376688b 100644--- a/crates/agent_session/src/domain/session/types.rs+++ b/crates/agent_session/src/domain/session/types.rs@@ -121,6 +121,8 @@ pub enum CloseReason {TransportFailed,/// The shell could not put an action on the transport.SendFailed,+ /// The runtime did not finish its ACP handshake before the deadline.+ HandshakeTimedOut,/// The shell could not persist a log entry, and an unlogged session may/// not keep running: the log stream is the session's history.LogFailed,@@ -133,6 +135,7 @@ impl std::fmt::Display for CloseReason {Self::TransportClosed => "the transport closed",Self::TransportFailed => "the transport failed",Self::SendFailed => "an action could not be sent",+ Self::HandshakeTimedOut => "the ACP handshake timed out",Self::LogFailed => "a log entry could not be persisted",})}diff --git a/crates/agent_trigger/Cargo.toml b/crates/agent_trigger/Cargo.tomlindex bf307d89e2..e050c19947 100644--- a/crates/agent_trigger/Cargo.toml+++ b/crates/agent_trigger/Cargo.toml@@ -23,6 +23,7 @@ strum = { workspace = true }serde_json = { workspace = true }thiserror = { workspace = true }tracing = { workspace = true }+tokio = { workspace = true, features = ["rt"] }workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]diff --git a/crates/agent_trigger/src/domain/mod.rs b/crates/agent_trigger/src/domain/mod.rsindex 4d7ef959d1..987a7eaf1e 100644--- a/crates/agent_trigger/src/domain/mod.rs+++ b/crates/agent_trigger/src/domain/mod.rs@@ -1,3 +1,4 @@pub mod broker_events;+pub mod processing;pub mod service;pub mod yield_event;diff --git a/crates/agent_trigger/src/domain/processing.rs b/crates/agent_trigger/src/domain/processing.rsnew file mode 100644index 0000000000..119ce6a581--- /dev/null+++ b/crates/agent_trigger/src/domain/processing.rs@@ -0,0 +1,67 @@+//! Shared channel-event processing for agent trigger consumers.++use agent_session::domain::error::AgentSessionError;+use agent_session::domain::ports::AgentSessionRepo;+use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};+use macro_event_broker::{EventBrokerError, MacroEvent as _, MacroEventBroker};++use super::broker_events::AgentTriggerEventName;+use super::service::{AgentBotLookup, AgentTriggerService};++/// Failure while evaluating or publishing one channel event.+#[derive(Debug, thiserror::Error)]+pub enum ProcessChannelEventError {+ /// Trigger evaluation could not read its session or bot context.+ #[error(transparent)]+ Evaluate(#[from] AgentSessionError),+ /// A yielded event could not be queued for publication.+ #[error(transparent)]+ Publish(#[from] EventBrokerError),+ /// The publication task stopped before reporting its result.+ #[error("agent event publication task failed")]+ PublishTask(#[source] tokio::task::JoinError),+}++/// Evaluate and publish all agent triggers yielded by one channel event.+///+/// Transport adapters retain ownership of decode and offset commit so their+/// `kafka.process` span can cover the complete record lifecycle.+pub async fn process_channel_event<Repo, Bots, Broker>(+ trigger: &AgentTriggerService<Repo, Bots>,+ publisher: &Broker,+ event: &ChannelMacroEvent,+) -> Result<(), ProcessChannelEventError>+where+ Repo: AgentSessionRepo,+ Bots: AgentBotLookup,+ Broker: MacroEventBroker,+{+ let ChannelTopicEvent::MessagePosted(posted) = &event.event().event else {+ return Ok(());+ };+ tracing::Span::current().record("macro.event.type", "channel.message_posted");++ let yielded_events = trigger.evaluate(posted).await?;+ tracing::info!(+ message_id = %posted.message_id,+ yielded_count = yielded_events.len(),+ "agent trigger evaluated channel message"+ );+ if yielded_events.is_empty() {+ tracing::debug!(message_id = %posted.message_id, "agent trigger yielded no event");+ }+ for yielded in yielded_events {+ let event_type: &'static str = AgentTriggerEventName::from(&yielded.event().event).into();+ tracing::info!(+ macro.event.id = %yielded.event().event_id,+ macro.event.type = event_type,+ "agent trigger yielded event"+ );+ publisher+ .send_event(&yielded)?+ .await+ .map_err(ProcessChannelEventError::PublishTask)??;+ }++ Ok(())+}diff --git a/crates/agent_trigger/src/domain/service.rs b/crates/agent_trigger/src/domain/service.rsindex 63a0d45c55..bb67a01cb0 100644--- a/crates/agent_trigger/src/domain/service.rs+++ b/crates/agent_trigger/src/domain/service.rs@@ -45,6 +45,8 @@ wherechannel_id = %posted.channel_id,message_id = %posted.message_id,thread_id = ?posted.thread_id,+ channel.message.scope = tracing::field::Empty,+ agent.mention.bot_count = tracing::field::Empty,))]pub async fn evaluate(&self,@@ -52,6 +54,15 @@ where) -> Result<Vec<AgentSessionMacroEvent>> {let mut mentioned = bot_mention_ids(&posted.mentions);mentioned.sort_by_key(ToString::to_string);+ tracing::Span::current().record(+ "channel.message.scope",+ if posted.thread_id.is_some() {+ "thread"+ } else {+ "channel_top_level"+ },+ );+ tracing::Span::current().record("agent.mention.bot_count", mentioned.len());let mut seen_sessions = HashSet::new();let mut events = Vec::new();@@ -74,6 +85,16 @@ whereOk(events)}+ #[tracing::instrument(+ err,+ skip(self, posted, seen_sessions),+ fields(+ channel_id = %posted.channel_id,+ message_id = %posted.message_id,+ bot_id = ?mentioned_bot,+ agent.trigger.outcome = tracing::field::Empty,+ )+ )]async fn evaluate_bot(&self,posted: &ChannelMessagePostedMetadata,@@ -87,11 +108,9 @@ whereif let Some(session_id) = session_id(&existing)&& !seen_sessions.insert(session_id){- log_no_event(- posted,- mentioned_bot,- NoEventReason::DuplicateSession { session_id },- );+ let reason = NoEventReason::DuplicateSession { session_id };+ tracing::Span::current().record("agent.trigger.outcome", reason.as_ref());+ log_no_event(posted, mentioned_bot, reason);return Ok(None);}let bot = match &existing {@@ -109,8 +128,16 @@ wherementioned_bot,};match yield_event(&message, has_agent) {- AgentSessionEventDecision::Event(event) => Ok(Some(event)),+ AgentSessionEventDecision::Event(event) => {+ let outcome = match existing {+ ChannelSession::None => "top_level_mentioned",+ ChannelSession::CreatedFromThread(_) => "mention_thread",+ };+ tracing::Span::current().record("agent.trigger.outcome", outcome);+ Ok(Some(event))+ }AgentSessionEventDecision::NoEvent(reason) => {+ tracing::Span::current().record("agent.trigger.outcome", reason.as_ref());log_no_event(posted, mentioned_bot, reason);Ok(None)}diff --git a/crates/agent_trigger/src/domain/yield_event.rs b/crates/agent_trigger/src/domain/yield_event.rsindex b2bd9cc2a4..649cd79b9e 100644--- a/crates/agent_trigger/src/domain/yield_event.rs+++ b/crates/agent_trigger/src/domain/yield_event.rs@@ -11,7 +11,8 @@ use crate::domain::broker_events::{};/// Why evaluating a message did not produce an agent-session event.-#[derive(Debug, Clone, Copy, PartialEq, Eq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::AsRefStr)]+#[strum(serialize_all = "snake_case")]pub enum NoEventReason {/// No existing session and no mentioned bot identified the target agent.MissingBotContext,diff --git a/crates/channel_bots/src/domain/trigger_detector/tests.rs b/crates/channel_bots/src/domain/trigger_detector/tests.rsindex b331cce92f..0d7c503498 100644--- a/crates/channel_bots/src/domain/trigger_detector/tests.rs+++ b/crates/channel_bots/src/domain/trigger_detector/tests.rs@@ -162,6 +162,7 @@ fn candidate(deleted_at: None,},mentioned_bot_ids,+ span: tracing::Span::none(),}}diff --git a/crates/channel_bots/src/inbound/bot_trigger_router.rs b/crates/channel_bots/src/inbound/bot_trigger_router.rsindex 4b6f9947ed..00b10076be 100644--- a/crates/channel_bots/src/inbound/bot_trigger_router.rs+++ b/crates/channel_bots/src/inbound/bot_trigger_router.rs@@ -5,6 +5,7 @@ use std::sync::Arc;use channels::domain::ports::ChannelService;use channels::domain::side_effects::ChannelBotTrigger;use tokio::sync::mpsc::UnboundedReceiver;+use tracing::Instrument as _;use crate::domain::{models::BotEvent,@@ -61,8 +62,9 @@ wheretokio::spawn(async move {while let Some(candidate) = candidates.recv().await {let router = self.clone();+ let span = candidate.span.clone();tokio::spawn(async move {- router.run(candidate).await;+ router.run(candidate).instrument(span).await;});}});diff --git a/crates/channels/src/domain/service.rs b/crates/channels/src/domain/service.rsindex 26cf0328d4..b6e8d4bb12 100644--- a/crates/channels/src/domain/service.rs+++ b/crates/channels/src/domain/service.rs@@ -19,6 +19,7 @@ use crate::domain::{ChannelMessagesErr, ChannelMessagesQueryResult, ChannelMutationErr,ChannelReferenceSharePermissions, ChannelRepo, ChannelService,},+ side_effects::bot_mention_ids,};use bot_id::BotIdStr;use bot_id::cowlike::CowLike;@@ -591,7 +592,16 @@ where}}- #[tracing::instrument(err, skip(self, req))]+ #[tracing::instrument(+ err,+ skip(self, req),+ fields(+ channel.id = %channel_id,+ channel.message.scope = tracing::field::Empty,+ channel.message.mention_count = tracing::field::Empty,+ agent.mention.bot_count = tracing::field::Empty,+ )+ )]async fn post_message(&self,actor: Sender,@@ -605,6 +615,19 @@ whereif actor.as_bot().is_some() && req.mentions.is_empty() {req.mentions = self.extract_content_mentions(&req.content).await;}+ tracing::Span::current().record(+ "channel.message.scope",+ if req.thread_id.is_some() {+ "thread"+ } else {+ "channel_top_level"+ },+ );+ tracing::Span::current().record("channel.message.mention_count", req.mentions.len());+ tracing::Span::current().record(+ "agent.mention.bot_count",+ bot_mention_ids(&req.mentions).len(),+ );let message = self.repodiff --git a/crates/channels/src/domain/side_effects.rs b/crates/channels/src/domain/side_effects.rsindex 3eadb94776..a7781e89ec 100644--- a/crates/channels/src/domain/side_effects.rs+++ b/crates/channels/src/domain/side_effects.rs@@ -25,6 +25,7 @@ use macro_event_broker::{MacroEventBroker, NoopMacroEventBroker};use macro_user_id::{cowlike::CowLike, user_id::MacroUserIdStr};use std::collections::HashSet;use tokio::sync::mpsc::UnboundedSender;+use tracing::Instrument as _;use uuid::Uuid;/// Entity type used by message mentions that target a bot.@@ -44,6 +45,8 @@ pub struct ChannelBotTrigger {/// Bots explicitly mentioned in the message that are active in the/// channel. Empty when the message mentions no bot.pub mentioned_bot_ids: Vec<BotId>,+ /// Trace active when the candidate was dispatched.+ pub span: tracing::Span,}/// Sender for bot-trigger candidates derived from channel messages.@@ -423,6 +426,11 @@ impl<C, R, N, K, B> ChannelSideEffectService<C, R, N, K, B> {channel_id,message: message.clone(),mentioned_bot_ids: active_bot_mention_ids(mentions, participants),+ span: tracing::info_span!(+ "channel.bot_trigger",+ channel.id = %channel_id,+ channel.message.id = %message.id,+ ),}).is_err(){@@ -454,9 +462,12 @@ where{fn dispatch(&self, event: ChannelEvent) {let handler = self.handler.clone();- tokio::spawn(async move {- handler.handle(event).await;- });+ tokio::spawn(+ async move {+ handler.handle(event).await;+ }+ .instrument(tracing::info_span!("channel.side_effects")),+ );}}diff --git a/crates/connection_gateway_client/Cargo.toml b/crates/connection_gateway_client/Cargo.tomlindex 2124e819b2..8176dd2c09 100644--- a/crates/connection_gateway_client/Cargo.toml+++ b/crates/connection_gateway_client/Cargo.toml@@ -8,6 +8,7 @@ version = "0.1.0"anyhow = { workspace = true }connection_gateway_models = { path = "../connection_gateway_models" }model-entity = { path = "../model-entity" }+macro_tower_layers = { path = "../macro_tower_layers" }reqwest = { workspace = true }serde = { workspace = true }serde_json = { workspace = true }diff --git a/crates/connection_gateway_client/src/client.rs b/crates/connection_gateway_client/src/client.rsindex 8e0536ed74..39168fa475 100644--- a/crates/connection_gateway_client/src/client.rs+++ b/crates/connection_gateway_client/src/client.rs@@ -23,6 +23,12 @@ struct Response {}impl ConnectionGatewayClient {+ fn with_trace_headers(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder {+ let mut headers = reqwest::header::HeaderMap::new();+ macro_tower_layers::inject_trace_headers(&mut headers);+ request.headers(headers)+ }+/// Create a new connection gateway client.pub fn new(internal_auth_key: String, connection_gateway_url: String) -> Self {let mut headers = reqwest::header::HeaderMap::new();@@ -51,12 +57,12 @@ impl ConnectionGatewayClient {message,};+ let request = self.client.post(format!(+ "{}/message/send/{}/{}",+ self.connection_gateway_url, entity.entity_type, entity.entity_id+ ));let res = self- .client- .post(format!(- "{}/message/send/{}/{}",- self.connection_gateway_url, entity.entity_type, entity.entity_id- ))+ .with_trace_headers(request).json(&body).send().await?@@ -80,15 +86,11 @@ impl ConnectionGatewayClient {entities,message_type,};- let res = self- .client- .post(format!(- "{}/message/batch_send",- self.connection_gateway_url- ))- .json(&body)- .send()- .await?;+ let request = self.client.post(format!(+ "{}/message/batch_send",+ self.connection_gateway_url+ ));+ let res = self.with_trace_headers(request).json(&body).send().await?;let json = res.json().await?;@@ -103,15 +105,11 @@ impl ConnectionGatewayClient {messages: Vec<UniqueMessage>,) -> anyhow::Result<Vec<MessageReceipt>> {let body = BatchSendUniqueMessagesBody { messages };- let res = self- .client- .post(format!(- "{}/message/batch_send_unique",- self.connection_gateway_url- ))- .json(&body)- .send()- .await?;+ let request = self.client.post(format!(+ "{}/message/batch_send_unique",+ self.connection_gateway_url+ ));+ let res = self.with_trace_headers(request).json(&body).send().await?;let json = res.json().await?;@@ -122,14 +120,11 @@ impl ConnectionGatewayClient {/// Get users who are interacting with a given entity.pub async fn track_entity_users(&self, entity: Entity<'_>) -> anyhow::Result<Vec<String>> {- let res = self- .client- .get(format!(- "{}/track/{}/{}",- self.connection_gateway_url, entity.entity_type, entity.entity_id- ))- .send()- .await?;+ let request = self.client.get(format!(+ "{}/track/{}/{}",+ self.connection_gateway_url, entity.entity_type, entity.entity_id+ ));+ let res = self.with_trace_headers(request).send().await?;let json = res.json().await?;diff --git a/crates/kafka_util/Cargo.toml b/crates/kafka_util/Cargo.tomlindex 882a4f17b9..c828f6b6ab 100644--- a/crates/kafka_util/Cargo.toml+++ b/crates/kafka_util/Cargo.toml@@ -10,9 +10,11 @@ aws-types = { workspace = true }either = { workspace = true }macro_env = { path = "../macro_env" }macro_env_var = { path = "../macro_env_var" }+opentelemetry = { workspace = true, features = ["trace"] }rdkafka = { workspace = true }thiserror = { workspace = true }tokio = { workspace = true }tracing = { workspace = true }+tracing-opentelemetry = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/kafka_util/src/lib.rs b/crates/kafka_util/src/lib.rsindex 0b6ed58282..e7acd2d9d7 100644--- a/crates/kafka_util/src/lib.rs+++ b/crates/kafka_util/src/lib.rs@@ -8,16 +8,19 @@#[cfg(test)]mod test;+use std::collections::HashMap;use std::marker::PhantomData;use std::time::Duration;use either::Either;use macro_env::Environment;+use opentelemetry::propagation::{Extractor, Injector};use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};use rdkafka::error::{KafkaError, KafkaResult};-use rdkafka::message::{BorrowedMessage, Message as _};+use rdkafka::message::{BorrowedMessage, Header, Headers, Message, OwnedHeaders};use rdkafka::producer::{FutureProducer, FutureRecord};use rdkafka::{ClientConfig, Offset, TopicPartitionList};+use tracing_opentelemetry::OpenTelemetrySpanExt as _;use uuid::Uuid;pub use msk_iam::{MskIamClientContext, configure_sasl_iam};@@ -28,6 +31,97 @@ const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";const MESSAGE_TIMEOUT_MS: &str = "5000";const SEND_TIMEOUT: Duration = Duration::from_secs(5);+#[derive(Default)]+struct TraceHeaderInjector(HashMap<String, String>);++impl Injector for TraceHeaderInjector {+ fn set(&mut self, key: &str, value: String) {+ self.0.insert(key.to_owned(), value);+ }+}++struct TraceHeaderExtractor<'a, H: Headers>(&'a H);++impl<H: Headers> Extractor for TraceHeaderExtractor<'_, H> {+ fn get(&self, key: &str) -> Option<&str> {+ self.0.iter().find_map(|header| {+ (header.key == key)+ .then(|| {+ header+ .value+ .and_then(|value| std::str::from_utf8(value).ok())+ })+ .flatten()+ })+ }++ fn keys(&self) -> Vec<&str> {+ self.0.iter().map(|header| header.key).collect()+ }+}++fn current_trace_headers() -> OwnedHeaders {+ let context = tracing::Span::current().context();+ let mut injector = TraceHeaderInjector::default();+ opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.inject_context(&context, &mut injector);+ });++ injector+ .0+ .into_iter()+ .fold(OwnedHeaders::new(), |headers, (key, value)| {+ headers.insert(Header {+ key: &key,+ value: Some(&value),+ })+ })+}++fn remote_trace_context<M: Message>(message: &M) -> opentelemetry::Context {+ opentelemetry::global::get_text_map_propagator(|propagator| {+ message+ .headers()+ .map_or_else(opentelemetry::Context::new, |headers| {+ propagator.extract(&TraceHeaderExtractor(headers))+ })+ })+}++/// Creates a Kafka processing span parented to the message's propagated W3C context.+///+/// The caller should instrument the complete decode, handling, and commit future+/// with this span so the consumer operation represents all record processing.+pub fn consumer_span<M: Message>(message: &M, consumer_group: &'static str) -> tracing::Span {+ let span = tracing::info_span!(+ "kafka.process",+ otel.kind = "consumer",+ messaging.system = "kafka",+ messaging.operation.name = "process",+ messaging.operation.type = "process",+ messaging.consumer.group.name = consumer_group,+ messaging.destination.name = message.topic(),+ messaging.destination.partition.id = %message.partition(),+ messaging.kafka.offset = message.offset(),+ macro.event.id = tracing::field::Empty,+ macro.event.type = tracing::field::Empty,+ agent.session.id = tracing::field::Empty,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ let parent = remote_trace_context(message);+ // Setting an empty context is intentional: headerless records must not+ // inherit an unrelated process- or task-lifetime ambient span.+ let _ = span.set_parent(parent);+ span+}++/// Marks a manually-created messaging span as failed.+pub fn record_span_error(span: &tracing::Span, error: &(impl std::fmt::Display + ?Sized)) {+ span.record("otel.status_code", "ERROR");+ span.record("otel.status_description", tracing::field::display(error));+}+/// Failure to construct an environment-specific Kafka consumer.#[derive(Debug, thiserror::Error)]pub enum KafkaConsumerError {@@ -246,9 +340,23 @@ impl KafkaEventProducer {}/// Sends a keyed payload to `topic` and waits for delivery confirmation.- #[tracing::instrument(err, skip(self, payload), fields(topic, key))]+ #[tracing::instrument(+ name = "kafka.publish",+ err,+ skip(self, payload),+ fields(+ otel.kind = "producer",+ messaging.system = "kafka",+ messaging.operation.name = "publish",+ messaging.operation.type = "send",+ messaging.destination.name = topic,+ )+ )]pub async fn send(&self, topic: &str, key: &str, payload: &[u8]) -> KafkaResult<()> {- let record = FutureRecord::to(topic).key(key).payload(payload);+ let record = FutureRecord::to(topic)+ .key(key)+ .payload(payload)+ .headers(current_trace_headers());either::for_both!(&self.producer.0, producer => producer.send(record, SEND_TIMEOUT).await).map(|_| ()).map_err(|(error, _)| error)diff --git a/crates/macro_event_broker/src/domain/models.rs b/crates/macro_event_broker/src/domain/models.rsindex 6b1d51b0f8..a5e6141bd8 100644--- a/crates/macro_event_broker/src/domain/models.rs+++ b/crates/macro_event_broker/src/domain/models.rs@@ -181,7 +181,6 @@ impl<T, M> MessageWrapper<T, M> {impl<T: MessageParts, M: MacroEventCollection> MessageWrapper<T, M> {/// Decodes the message into one of the declared event types.- #[tracing::instrument(err, skip(self))]pub fn decode_payload(&self) -> Result<M, EventBrokerError> {M::decode(&self.inner)}diff --git a/crates/macro_event_broker/src/domain/service.rs b/crates/macro_event_broker/src/domain/service.rsindex a9b5be68dd..6dceaef4d9 100644--- a/crates/macro_event_broker/src/domain/service.rs+++ b/crates/macro_event_broker/src/domain/service.rs@@ -43,7 +43,6 @@ where/// Receives the next message from the underlying consumer.////// Call [`MessageWrapper::decode_payload`] to decode its declared event.- #[tracing::instrument(skip(self), err)]pub async fn recv<'a>(&'a self,) -> Result<MessageWrapper<C::MessageType<'a>, M>, rootcause::Report> {@@ -89,7 +88,6 @@ impl<P: EventPublisher, S: Spawner> MacroEventBrokerService<P, S> {}impl<P: EventPublisher, S: Spawner> MacroEventBroker for MacroEventBrokerService<P, S> {- #[tracing::instrument(err, skip(self, event), fields(topic = event.topic(), key = %event.key()))]fn send_event<E: MacroEvent + ?Sized>(&self,event: &E,diff --git a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rsindex 90a9a3be95..380669ccc4 100644--- a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs+++ b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs@@ -68,7 +68,6 @@ impl<T: GroupName, M> KafkaConsumerAdapter<T, M> {}/// Commits a message using the caller-selected commit mode.- #[tracing::instrument(err, skip(self))]pub fn commit_message(&self,message: &BorrowedMessage<'_>,diff --git a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rsindex 432bfbb0df..47a725ec20 100644--- a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs+++ b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs@@ -27,7 +27,6 @@ impl KafkaEventPublisher {}impl EventPublisher for KafkaEventPublisher {- #[tracing::instrument(err, skip(self, payload), fields(topic = T::TOPIC_STR, key = %key))]async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {self.producer.send(T::TOPIC_STR, key, payload)diff --git a/crates/macro_tower_layers/src/test.rs b/crates/macro_tower_layers/src/test.rsindex 93bd9483c7..655b5a9ea7 100644--- a/crates/macro_tower_layers/src/test.rs+++ b/crates/macro_tower_layers/src/test.rs@@ -318,7 +318,6 @@ fn inject_trace_headers_emits_current_span_traceparent() {let mut headers = http::HeaderMap::new();inject_trace_headers(&mut headers);-let traceparent = headers.get("traceparent").expect("traceparent header should be injected inside a span")diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rsindex 9a5bb64575..09104d0d54 100644--- a/services/agent_harness_service/src/main.rs+++ b/services/agent_harness_service/src/main.rs@@ -11,10 +11,10 @@ mod bots_directory;mod config;mod trigger;-use std::sync::Arc;+use std::{future::Future, pin::Pin, sync::Arc};use agent_fold::domain::service::FoldedMessageService;-use agent_harness::domain::model::SessionDefaults;+use agent_harness::domain::model::{HarnessCommand, SessionDefaults};use agent_harness::domain::service::AgentHarnessService;use agent_harness::inbound::kafka::{RoutedTrigger, route_agent_trigger};use agent_harness::inbound::runtime_gateway::RuntimeGatewayState;@@ -47,7 +47,7 @@ use channels::outbound::pg_channels_repo::PgChannelsRepo;use channels::outbound::pg_side_effect_context::PgChannelSideEffectContext;use config::{Config, Environment};use connection_gateway_client::ConnectionGatewayClient;-use kafka_util::{GroupName, KafkaEventConsumer};+use kafka_util::{GroupName, KafkaEventConsumer, consumer_span, record_span_error};use lexical_client::LexicalClient;use macro_auth::middleware::decode_jwt::JwtValidationArgs;use macro_authorization::{@@ -63,6 +63,7 @@ use macro_service_urls::{ConnectionGatewayUrl, LexicalServiceUrl};use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};use sqlx::postgres::PgPoolOptions;+use tracing::Instrument as _;/// Consumer group owning this harness's agent-session offsets.///@@ -87,9 +88,25 @@ fn commit_message(consumer: &HarnessConsumer, message: &BorrowedMessage<'_>) ->.map_err(|error| anyhow::anyhow!("failed to commit agent session offset: {error:?}"))}+type HarnessWork =+ Pin<Box<dyn Future<Output = agent_harness::domain::error::Result<()>> + Send + 'static>>;++struct PendingHarnessWork {+ session_id: agent_session::domain::model::AgentSessionId,+ span: tracing::Span,+ work: HarnessWork,+ description: &'static str,+}+#[tokio::main]async fn main() -> anyhow::Result<()> {- MacroEntrypoint::default().init();+ let entrypoint = MacroEntrypoint::default().init();+ let result = run().await;+ entrypoint.shutdown();+ result+}++async fn run() -> anyhow::Result<()> {agent_harness::install_tls_provider();let config = Config::from_env()?;let bot_id = BotId::new_from_uuid(config.harness_bot_id);@@ -334,69 +351,115 @@ async fn main() -> anyhow::Result<()> {continue;}};- let kafka_message = message.inner();- let event = match message.decode_payload() {- Ok(DeclaredMacroEvent::AgentSessionMacroEvent(event)) => event,- Err(error) => {- tracing::error!(- error = ?error,- partition = kafka_message.partition(),- offset = kafka_message.offset(),- "dropping undecodable agent session event"- );- match commit_message(&consumer, kafka_message) {- Ok(()) => continue,- Err(error) => {- run_error = Some(error);- break;- }+ let span = consumer_span(message.inner(), AgentHarnessConsumerGroup::GROUP_NAME);+ let processing = async {+ let kafka_message = message.inner();+ let event = match message.decode_payload() {+ Ok(DeclaredMacroEvent::AgentSessionMacroEvent(event)) => event,+ Err(error) => {+ record_span_error(&tracing::Span::current(), &error);+ tracing::error!(+ error = ?error,+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),+ "dropping undecodable agent session event"+ );+ commit_message(&consumer, kafka_message)?;+ return Ok::<_, anyhow::Error>(None);}- }- };+ };+ tracing::Span::current()+ .record("macro.event.id", tracing::field::display(event.event().event_id));- let routed = match route_agent_trigger(event.event().event.clone(), bot_id) {- Ok(routed) => routed,- Err(skipped) => {- tracing::debug!(?skipped, "skipped an agent session event");- match commit_message(&consumer, kafka_message) {- Ok(()) => continue,- Err(error) => {- run_error = Some(error);- break;- }+ let routed = match route_agent_trigger(event.event().event.clone(), bot_id) {+ Ok(routed) => routed,+ Err(skipped) => {+ tracing::debug!(?skipped, "skipped an agent session event");+ commit_message(&consumer, kafka_message)?;+ return Ok(None);}- }- };+ };- // Intentionally at-most-once: keep ingestion simple and never- // let concurrent task completion commit Kafka offsets out of order.- if let Err(error) = commit_message(&consumer, kafka_message) {- run_error = Some(error);- break;- }- match routed {- RoutedTrigger::Command(session_id, command) => {- let execution = harness.execute(session_id, command);- tasks.spawn(async move {- match execution.await {- Ok(()) => tracing::info!(%session_id, "executed an agent harness command"),- Err(error) => {- tracing::error!(error = ?error, %session_id, "failed to execute an agent harness command");+ let pending = match routed {+ RoutedTrigger::Command(session_id, command) => {+ tracing::Span::current()+ .record("agent.session.id", tracing::field::display(session_id));+ let event_type = match &command {+ HarnessCommand::Open(_) => "agent_trigger.new",+ HarnessCommand::Deliver(_) => "agent_trigger.existing",+ HarnessCommand::Delete => "agent_trigger.delete",+ HarnessCommand::SetSandboxSize(_) => {+ "agent_trigger.set_sandbox_size"}+ };+ tracing::Span::current().record("macro.event.type", event_type);+ let execution_span = tracing::info_span!(+ "harness.execute",+ agent.session.id = %session_id,+ agent.command.type = event_type,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ // `execute` admits synchronously, so entering here is+ // what carries this child context through the queue.+ let execution = execution_span+ .in_scope(|| harness.execute(session_id, command));+ PendingHarnessWork {+ session_id,+ span: execution_span,+ work: Box::pin(execution),+ description: "executed an agent harness command",}- });- }- RoutedTrigger::Announce(session_id, prompt) => {- let harness = harness.clone();- tasks.spawn(async move {- match harness.announce_external_prompt(session_id, prompt).await {- Ok(()) => tracing::info!(%session_id, "announced an external prompt"),- Err(error) => {- tracing::error!(error = ?error, %session_id, "failed to announce an external prompt");- }+ }+ RoutedTrigger::Announce(session_id, prompt) => {+ tracing::Span::current()+ .record("agent.session.id", tracing::field::display(session_id));+ tracing::Span::current()+ .record("macro.event.type", "agent_trigger.announce");+ let execution_span = tracing::info_span!(+ "harness.announce",+ agent.session.id = %session_id,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ let harness = harness.clone();+ PendingHarnessWork {+ session_id,+ span: execution_span,+ work: Box::pin(async move {+ harness.announce_external_prompt(session_id, prompt).await+ }),+ description: "announced an external prompt",}- });+ }+ };++ // Intentionally at-most-once: admission precedes commit, but+ // long-running harness work is independent of Kafka afterward.+ commit_message(&consumer, kafka_message)?;+ Ok(Some(pending))+ }+ .instrument(span.clone())+ .await;++ let pending = match processing {+ Ok(pending) => pending,+ Err(error) => {+ record_span_error(&span, &error);+ run_error = Some(error);+ break;}+ };+ if let Some(PendingHarnessWork { session_id, span, work, description }) = pending {+ tasks.spawn(async move {+ match work.instrument(span.clone()).await {+ Ok(()) => tracing::info!(%session_id, description, "agent harness work completed"),+ Err(error) => {+ record_span_error(&span, &error);+ tracing::error!(error = ?error, %session_id, "agent harness work failed");+ }+ }+ });}}Some(result) = tasks.join_next(), if !tasks.is_empty() => {@@ -409,7 +472,10 @@ async fn main() -> anyhow::Result<()> {http.abort();trigger.abort();- container_shutdown.shutdown_all().await;+ let stop_failures = container_shutdown.shutdown_all().await;+ if stop_failures > 0 {+ tracing::error!(stop_failures, "some sandboxes failed to stop");+ }while let Some(result) = tasks.join_next().await {if let Err(error) = result {@@ -417,11 +483,6 @@ async fn main() -> anyhow::Result<()> {}}- let stop_failures = container_shutdown.shutdown_all().await;- if stop_failures > 0 {- tracing::error!(stop_failures, "some sandboxes failed to stop");- }-match run_error {Some(error) => Err(error),None => Ok(()),diff --git a/services/agent_harness_service/src/trigger.rs b/services/agent_harness_service/src/trigger.rsindex d41d4e63f4..eaf9ce9bc3 100644--- a/services/agent_harness_service/src/trigger.rs+++ b/services/agent_harness_service/src/trigger.rs@@ -1,21 +1,22 @@//! Channel-message consumer that emits agent-session trigger events.use agent_session::outbound::postgres::PgAgentSessionRepo;+use agent_trigger::domain::processing::process_channel_event;use agent_trigger::domain::service::{AgentBotLookup, AgentTriggerService};-use anyhow::Context as _;use bot_id::BotId;use bots::domain::ports::BotRepo as _;use bots::outbound::pg_bots_repo::PgBotsRepo;-use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};-use kafka_util::{GroupName, KafkaEventConsumer};+use channels::domain::broker_events::ChannelMacroEvent;+use kafka_util::{GroupName, KafkaEventConsumer, consumer_span, record_span_error};use macro_event_broker::{- KafkaConsumerAdapter, KafkaEventPublisher, MacroEvent as _, MacroEventBroker as _,- MacroEventBrokerService, MacroEventCollection as _, MacroEventConsumerService,+ KafkaConsumerAdapter, KafkaEventPublisher, MacroEvent as _, MacroEventBrokerService,+ MacroEventCollection as _, MacroEventConsumerService,};use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};use sqlx::PgPool;use tokio::time::{Duration, sleep};+use tracing::Instrument as _;struct AgentTriggerConsumerGroup;@@ -87,38 +88,37 @@ async fn run(pool: PgPool, kafka_brokers: String) -> anyhow::Result<()> {continue;}};- let kafka_message = message.inner();- let event = match message.decode_payload() {- Ok(DeclaredChannelEvent::ChannelMacroEvent(event)) => event,- Err(error) => {- tracing::error!(- error = ?error,- partition = kafka_message.partition(),- offset = kafka_message.offset(),- "dropping undecodable channel event"- );- commit_message(&consumer, kafka_message)?;- continue;- }- };+ let span = consumer_span(message.inner(), AgentTriggerConsumerGroup::GROUP_NAME);+ let result = async {+ let kafka_message = message.inner();+ let event = match message.decode_payload() {+ Ok(DeclaredChannelEvent::ChannelMacroEvent(event)) => event,+ Err(error) => {+ record_span_error(&tracing::Span::current(), &error);+ tracing::error!(+ error = ?error,+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),+ "dropping undecodable channel event"+ );+ commit_message(&consumer, kafka_message)?;+ return Ok::<(), anyhow::Error>(());+ }+ };+ tracing::Span::current().record(+ "macro.event.id",+ tracing::field::display(event.event().event_id),+ );- let ChannelTopicEvent::MessagePosted(posted) = &event.event().event else {+ process_channel_event(&trigger, &publisher, &event).await?;commit_message(&consumer, kafka_message)?;- continue;- };- let yielded_events = trigger.evaluate(posted).await?;- if yielded_events.is_empty() {- tracing::debug!(message_id = %posted.message_id, "agent trigger yielded no event");+ Ok(())}- for yielded in yielded_events {- let json = serde_json::to_string(yielded.event())?;- tracing::info!(yielded_event = %json, "agent trigger yielded event");- let publish = publisher.send_event(&yielded)?;- publish- .await- .context("agent event publication task failed")??;+ .instrument(span.clone())+ .await;+ if let Err(error) = &result {+ record_span_error(&span, error);}-- commit_message(&consumer, kafka_message)?;+ result?;}}diff --git a/services/agent_trigger_service/src/main.rs b/services/agent_trigger_service/src/main.rsindex cec054db1e..33c45645d7 100644--- a/services/agent_trigger_service/src/main.rs+++ b/services/agent_trigger_service/src/main.rs@@ -3,22 +3,24 @@mod config;use agent_session::outbound::postgres::PgAgentSessionRepo;+use agent_trigger::domain::processing::process_channel_event;use agent_trigger::domain::service::{AgentBotLookup, AgentTriggerService};use anyhow::Context as _;use bots::domain::models::BotId;use bots::domain::ports::BotRepo as _;use bots::outbound::pg_bots_repo::PgBotsRepo;-use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};+use channels::domain::broker_events::ChannelMacroEvent;use config::Config;-use kafka_util::{GroupName, KafkaEventConsumer};+use kafka_util::{GroupName, KafkaEventConsumer, consumer_span, record_span_error};use macro_entrypoint::{MacroEntrypoint, shutdown_signal};use macro_event_broker::{- KafkaConsumerAdapter, KafkaEventPublisher, MacroEvent as _, MacroEventBroker as _,- MacroEventBrokerService, MacroEventCollection as _, MacroEventConsumerService,+ KafkaConsumerAdapter, KafkaEventPublisher, MacroEvent as _, MacroEventBrokerService,+ MacroEventCollection as _, MacroEventConsumerService,};use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};use sqlx::postgres::PgPoolOptions;+use tracing::Instrument as _;struct AgentTriggerConsumerGroup;@@ -51,9 +53,14 @@ impl AgentBotLookup for PgAgentBotLookup {}#[tokio::main]-#[tracing::instrument(err)]async fn main() -> anyhow::Result<()> {- MacroEntrypoint::default().init();+ let entrypoint = MacroEntrypoint::default().init();+ let result = run().await;+ entrypoint.shutdown();+ result+}++async fn run() -> anyhow::Result<()> {let config = Config::from_env()?;let pool = PgPoolOptions::new().min_connections(1)@@ -84,6 +91,7 @@ async fn main() -> anyhow::Result<()> {);let mut shutdown = std::pin::pin!(shutdown_signal());+ let mut run_error = None;loop {tokio::select! {() = &mut shutdown => {@@ -98,40 +106,44 @@ async fn main() -> anyhow::Result<()> {continue;}};- let kafka_message = message.inner();- let event = match message.decode_payload() {- Ok(DeclaredMacroEvent::ChannelMacroEvent(event)) => event,- Err(error) => {- tracing::error!(- error = ?error,- partition = kafka_message.partition(),- offset = kafka_message.offset(),- "dropping undecodable channel event"- );- commit_message(&consumer, kafka_message)?;- continue;- }- };-- let ChannelTopicEvent::MessagePosted(posted) = &event.event().event else {+ let span = consumer_span(message.inner(), AgentTriggerConsumerGroup::GROUP_NAME);+ let result = async {+ let kafka_message = message.inner();+ let event = match message.decode_payload() {+ Ok(DeclaredMacroEvent::ChannelMacroEvent(event)) => event,+ Err(error) => {+ record_span_error(&tracing::Span::current(), &error);+ tracing::error!(+ error = ?error,+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),+ "dropping undecodable channel event"+ );+ commit_message(&consumer, kafka_message)?;+ return Ok::<(), anyhow::Error>(());+ }+ };+ tracing::Span::current().record("macro.event.id", tracing::field::display(event.event().event_id));++ process_channel_event(&trigger, &publisher, &event).await?;commit_message(&consumer, kafka_message)?;- continue;- };- let yielded_events = trigger.evaluate(posted).await?;- if yielded_events.is_empty() {- tracing::debug!(message_id = %posted.message_id, "agent trigger yielded no event");+ Ok(())}- for yielded in yielded_events {- let json = serde_json::to_string(yielded.event())?;- tracing::info!(yielded_event = %json, "agent trigger yielded event");- let publish = publisher.send_event(&yielded)?;- publish.await.context("agent event publication task failed")??;+ .instrument(span.clone())+ .await;+ if let Err(error) = &result {+ record_span_error(&span, error);+ }+ if let Err(error) = result {+ run_error = Some(error);+ break;}-- commit_message(&consumer, kafka_message)?;}}}- Ok(())+ match run_error {+ Some(error) => Err(error),+ None => Ok(()),+ }}diff --git a/services/connection_gateway/Cargo.toml b/services/connection_gateway/Cargo.tomlindex 955323dc9c..117b0c1839 100644--- a/services/connection_gateway/Cargo.toml+++ b/services/connection_gateway/Cargo.toml@@ -57,10 +57,12 @@ macro_config = { path = "../../crates/macro_config" }macro_entrypoint = { path = "../../crates/macro_entrypoint" }macro_env = { path = "../../crates/macro_env" }macro_env_var = { path = "../../crates/macro_env_var" }+macro_tower_layers = { path = "../../crates/macro_tower_layers" }mockall.workspace = truemodel = { path = "../../crates/model" }model-entity = { path = "../../crates/model-entity" }models_bulk_upload = { path = "../../crates/models_bulk_upload" }+opentelemetry = { workspace = true, features = ["trace"] }redis = { workspace = true, features = ["aio","tokio-comp",@@ -78,6 +80,7 @@ tokio = { workspace = true }tower = { workspace = true }tower-http = { workspace = true }tracing = { workspace = true, features = ["attributes"] }+tracing-opentelemetry = { workspace = true }tungstenite = { workspace = true }url = { workspace = true }utoipa = { workspace = true, features = ["axum_extras"] }diff --git a/services/connection_gateway/src/api/connection.rs b/services/connection_gateway/src/api/connection.rsindex 8441f2b931..797a4b2b29 100644--- a/services/connection_gateway/src/api/connection.rs+++ b/services/connection_gateway/src/api/connection.rs@@ -2,7 +2,9 @@ use crate::{config::Config,context::{ApiContext, AppState, AuthorizationService},model::{- connection::ConnectionContext, message::OutgoingMessage, tracking::EntityConnectionExt,+ connection::ConnectionContext,+ message::{OutgoingMessage, record_span_error},+ tracking::EntityConnectionExt,},};use anyhow::Result;@@ -27,6 +29,8 @@ use model::user::UserContext;use model_entity::EntityType;use std::sync::Arc;use tokio::sync::mpsc::Receiver;+use tracing::Instrument as _;+use tracing_opentelemetry::OpenTelemetrySpanExt as _;mod messages;@@ -136,15 +140,37 @@ pub async fn forwarder(mut sink: SplitSink<WebSocket, AxumWebsocketMessage>,mut receiver: Receiver<OutgoingMessage>,) -> Result<()> {- while let Some(message) = receiver.recv().await {- if let Ok(msg) = message.try_into()- && let Err(err) = sink.send(msg).await- {- tracing::warn!(- error=?err,- "Failed to send message to WebSocket, client likely disconnected"- );- break;+ while let Some(mut message) = receiver.recv().await {+ let span = match &message {+ OutgoingMessage::Message(message) => {+ let span = tracing::info_span!(+ "connection_gateway.websocket_send",+ otel.kind = "producer",+ message_type = %message.message_type,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ if let Some(parent) = message.remote_trace_context() {+ let _ = span.set_parent(parent);+ }+ span+ }+ OutgoingMessage::Pong => tracing::Span::none(),+ };+ if let OutgoingMessage::Message(message) = &mut message {+ message.trace.clear();+ }++ if let Ok(msg) = message.try_into() {+ let result = sink.send(msg).instrument(span.clone()).await;+ if let Err(err) = result {+ record_span_error(&span, &err);+ tracing::warn!(+ error=?err,+ "Failed to send message to WebSocket, client likely disconnected"+ );+ break;+ }}}diff --git a/services/connection_gateway/src/api/message.rs b/services/connection_gateway/src/api/message.rsindex 9763352c65..bc6f0c4828 100644--- a/services/connection_gateway/src/api/message.rs+++ b/services/connection_gateway/src/api/message.rs@@ -61,10 +61,7 @@ pub async fn send_message_handler(let res = send_message_to_entity(&ctx,&entity,- Message {- message_type: body.message_type.clone(),- data: body.message.to_string(),- },+ Message::new(body.message_type.clone(), body.message.to_string()),redis_connection,).await@@ -110,10 +107,7 @@ pub async fn batch_send_message_handler(send_message_to_entity(&ctx,entity,- Message {- message_type: body.message_type.clone(),- data: body.message.to_string(),- },+ Message::new(body.message_type.clone(), body.message.to_string()),redis_connection,)}))@@ -165,10 +159,10 @@ pub async fn batch_send_unique_messages_handler(send_message_to_entity(&ctx,&message.entity,- Message {- message_type: message.message_type.clone(),- data: message.message_content.to_string(),- },+ Message::new(+ message.message_type.clone(),+ message.message_content.to_string(),+ ),redis_connection,)}))diff --git a/services/connection_gateway/src/main.rs b/services/connection_gateway/src/main.rsindex 6a35139c33..736b418520 100644--- a/services/connection_gateway/src/main.rs+++ b/services/connection_gateway/src/main.rs@@ -35,6 +35,7 @@ use macro_auth::middleware::decode_jwt::JwtValidationArgs;use macro_authorization::{InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState};use macro_entrypoint::MacroEntrypoint;use macro_env::Environment;+use macro_tower_layers::MacroRequestIdAndTracingLayer;use service::dynamodb::create_dynamo_db_connection_manager;use service::redis::poll_messages;use sqlx::postgres::PgPoolOptions;@@ -140,6 +141,7 @@ async fn main() -> Result<()> {Duration::from_secs(60),)),})+ .layer(MacroRequestIdAndTracingLayer::new(Duration::from_millis(200)).into_inner()).layer(cors);tracing::info!(diff --git a/services/connection_gateway/src/model/message.rs b/services/connection_gateway/src/model/message.rsindex 588bf213be..382152e5f7 100644--- a/services/connection_gateway/src/model/message.rs+++ b/services/connection_gateway/src/model/message.rs@@ -1,12 +1,49 @@use anyhow::Result;+use opentelemetry::trace::TraceContextExt as _;use serde::{Deserialize, Serialize};+use std::collections::HashMap;use stream::domain::StreamEvent;+use tracing_opentelemetry::OpenTelemetrySpanExt as _;#[derive(Serialize, Deserialize, Clone, Debug)]pub struct Message {#[serde(rename = "type")]pub message_type: String,pub data: String,+ #[serde(default, skip_serializing_if = "HashMap::is_empty")]+ pub(crate) trace: HashMap<String, String>,+}++impl Message {+ pub fn new(message_type: String, data: String) -> Self {+ Self {+ message_type,+ data,+ trace: HashMap::new(),+ }+ .with_current_trace_context()+ }++ pub fn with_current_trace_context(mut self) -> Self {+ self.trace.clear();+ let context = tracing::Span::current().context();+ opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.inject_context(&context, &mut self.trace);+ });+ self+ }++ pub fn remote_trace_context(&self) -> Option<opentelemetry::Context> {+ let context = opentelemetry::global::get_text_map_propagator(|propagator| {+ propagator.extract(&self.trace)+ });+ context.span().span_context().is_valid().then_some(context)+ }+}++pub(crate) fn record_span_error(span: &tracing::Span, error: &(impl std::fmt::Display + ?Sized)) {+ span.record("otel.status_code", "ERROR");+ span.record("otel.status_description", tracing::field::display(error));}static STREAM_EVENT_TYPE: &str = "stream_event";@@ -14,10 +51,7 @@ impl TryFrom<StreamEvent> for Message {type Error = anyhow::Error;fn try_from(value: StreamEvent) -> Result<Self, Self::Error> {serde_json::to_string(&value)- .map(|data| Self {- data,- message_type: STREAM_EVENT_TYPE.into(),- })+ .map(|data| Self::new(STREAM_EVENT_TYPE.into(), data)).map_err(anyhow::Error::from)}}diff --git a/services/connection_gateway/src/model/stream.rs b/services/connection_gateway/src/model/stream.rsindex 8122247b7c..05d06af73e 100644--- a/services/connection_gateway/src/model/stream.rs+++ b/services/connection_gateway/src/model/stream.rs@@ -6,11 +6,7 @@ const STREAM_MESSAGE_TYPE: &str = "stream";impl TryFrom<StreamItem> for OutgoingMessage {type Error = serde_json::Error;fn try_from(value: StreamItem) -> Result<Self, Self::Error> {- serde_json::to_string(&value).map(|data| {- OutgoingMessage::Message(Message {- message_type: STREAM_MESSAGE_TYPE.into(),- data,- })- })+ serde_json::to_string(&value)+ .map(|data| OutgoingMessage::Message(Message::new(STREAM_MESSAGE_TYPE.into(), data)))}}diff --git a/services/connection_gateway/src/service/redis.rs b/services/connection_gateway/src/service/redis.rsindex ba257f5ed2..5ab142e226 100644--- a/services/connection_gateway/src/service/redis.rs+++ b/services/connection_gateway/src/service/redis.rs@@ -1,8 +1,13 @@use anyhow::{Context, Result};use futures::StreamExt;use redis::{AsyncCommands, FromRedisValue, ParsingError, Value, aio::MultiplexedConnection};+use tracing::Instrument as _;+use tracing_opentelemetry::OpenTelemetrySpanExt as _;-use crate::{context::ApiContext, model::message::Message};+use crate::{+ context::ApiContext,+ model::message::{Message, record_span_error},+};pub const REDIS_CHANNEL: &str = "connection_gateway.messages";@@ -19,16 +24,31 @@ pub struct MessageWithConnection {/// will handle sending the message to the client correctly.pub async fn post_message(mut connection: MultiplexedConnection,- message: MessageWithConnection,+ mut message: MessageWithConnection,) -> Result<()> {- let message_json = serde_json::to_string(&message).context("Failed to serialize message")?;-- connection- .publish::<&str, &str, ()>(REDIS_CHANNEL, message_json.as_str())- .await- .context("Failed to publish message")?;-- Ok(())+ let span = tracing::info_span!(+ "connection_gateway.redis_publish",+ otel.kind = "producer",+ message_type = %message.message.message_type,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ let result = async {+ message.message = message.message.with_current_trace_context();+ let message_json =+ serde_json::to_string(&message).context("Failed to serialize message")?;++ connection+ .publish::<&str, &str, ()>(REDIS_CHANNEL, message_json.as_str())+ .await+ .context("Failed to publish message")+ }+ .instrument(span.clone())+ .await;+ if let Err(error) = &result {+ record_span_error(&span, error);+ }+ result}impl FromRedisValue for MessageWithConnection {@@ -55,7 +75,7 @@ pub async fn poll_messages(ctx: ApiContext) -> Result<()> {.context("Failed to subscribe to reddis channel")?;while let Some(maybe_message) = stream.next().await {- let message: MessageWithConnection =+ let mut message: MessageWithConnection =match maybe_message.get_payload::<MessageWithConnection>() {Ok(msg) => msg,Err(err) => {@@ -79,12 +99,26 @@ pub async fn poll_messages(ctx: ApiContext) -> Result<()> {connection_id = message.connection_id,"received message from redis, sending to connection");-- if let Err(err) = ctx- .connection_manager- .send_message(message.connection_id.as_str(), message.message)- .await- {+ let span = tracing::info_span!(+ "connection_gateway.redis_dispatch",+ otel.kind = "consumer",+ message_type = %message.message.message_type,+ otel.status_code = tracing::field::Empty,+ otel.status_description = tracing::field::Empty,+ );+ if let Some(parent) = message.message.remote_trace_context() {+ let _ = span.set_parent(parent);+ }+ let result = async {+ message.message = message.message.with_current_trace_context();+ ctx.connection_manager+ .send_message(message.connection_id.as_str(), message.message)+ .await+ }+ .instrument(span.clone())+ .await;+ if let Err(err) = result {+ record_span_error(&span, &err);tracing::error!(error=?err, "failed to send message");}}diff --git a/services/connection_gateway/src/service/tracker.rs b/services/connection_gateway/src/service/tracker.rsindex f7a6451f1d..2e6fdcbfd0 100644--- a/services/connection_gateway/src/service/tracker.rs+++ b/services/connection_gateway/src/service/tracker.rs@@ -150,14 +150,14 @@ pub struct UserTrackingChange {pub async fn notify_tracking_change(ctx: ConnectionContext<'_>, entity: &Entity<'_>) -> Result<()> {let users = get_users_for_entity(ctx, entity, Some(DEFAULT_TIMEOUT_THRESHOLD)).await?;- let message: Message = Message {- message_type: "user_tracking_change".to_string(),- data: serde_json::to_string(&UserTrackingChange {+ let message = Message::new(+ "user_tracking_change".to_string(),+ serde_json::to_string(&UserTrackingChange {entity_type: entity.entity_type,entity_id: entity.entity_id.to_string(),user_ids: users,})?,- };+ );let redis_connection = ctx.api_context.redis_connection.clone();send_message_to_entity(ctx, entity, message, redis_connection).await?;diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex afc418e199..3422c41102 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -167,7 +167,13 @@ maybe_env_vars! {#[tokio::main]async fn main() -> anyhow::Result<()> {- MacroEntrypoint::default().init();+ let entrypoint = MacroEntrypoint::default().init();+ let result = run().await;+ entrypoint.shutdown();+ result+}++async fn run() -> anyhow::Result<()> {let env = Environment::new_or_prod();let aws_config = macro_aws_config::get_macro_aws_config().await;