Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(agent_harness): trace a forwarded command across both replicas (#6082)
Wolf Mermelstein1a5282d04356
diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rsindex 0b75cb9a66..6b5627bffb 100644--- a/crates/agent_harness/src/domain/service.rs+++ b/crates/agent_harness/src/domain/service.rs@@ -665,26 +665,59 @@ where/// then its heartbeat going stale is what lets this replica take over. A/// manager that is alive but unreachable stays an error - executing/// locally anyway is how two actors end up on one session.- #[tracing::instrument(err, skip(self, command), fields(%session_id))]+ /// The routing decision is recorded on the span, not only logged: which of+ /// the three answers the lease gave, which peer it named, and whether the+ /// command left this process. Those are the fields you group by when a+ /// replica is mishandling commands, and a log line cannot be aggregated.+ #[tracing::instrument(+ err,+ skip(self, command),+ fields(+ %session_id,+ agent.session.management = tracing::field::Empty,+ agent.session.manager_replica = tracing::field::Empty,+ agent.command.forwarded = tracing::field::Empty,+ agent.command.stale_fallback = tracing::field::Empty,+ )+ )]async fn route_then_execute(&self,session_id: AgentSessionId,command: HarnessCommand,) -> Result<()> {+ let span = tracing::Span::current();// Open never routes: it is what creates the session row this routing// would read, and a fresh id has no manager to defer to.if matches!(command, HarnessCommand::Open(_)) {+ span.record("agent.session.management", "open");+ span.record("agent.command.forwarded", false);return self.execute(session_id, command).await;}let manager = match self.sessions.management(session_id).await? {- SessionManagement::Unmanaged | SessionManagement::Ours => {+ SessionManagement::Unmanaged => {+ span.record("agent.session.management", "unmanaged");+ span.record("agent.command.forwarded", false);+ return self.execute(session_id, command).await;+ }+ SessionManagement::Ours => {+ span.record("agent.session.management", "ours");+ span.record("agent.command.forwarded", false);return self.execute(session_id, command).await;}SessionManagement::Peer(manager) => manager,};+ span.record("agent.session.management", "peer");+ span.record(+ "agent.session.manager_replica",+ tracing::field::display(manager.replica),+ );let Some(address) = manager.address else {+ // Recorded false deliberately: the command stayed here, but as an+ // error rather than a local execution.+ span.record("agent.command.forwarded", false);return Err(HarnessError::ManagerUnreachable(session_id));};+ span.record("agent.command.forwarded", true);tracing::info!(%session_id, peer = %manager.replica, "forwarding an agent session command");match self.forwarder@@ -694,6 +727,10 @@ whereOk(()) => Ok(()),Err(forward_error) => match self.sessions.management(session_id).await? {SessionManagement::Unmanaged | SessionManagement::Ours => {+ // Worth aggregating rather than only logging: routine+ // fallbacks mean heartbeats are not keeping up, which is a+ // different problem from an occasional dead peer.+ span.record("agent.command.stale_fallback", true);tracing::warn!(error = ?forward_error,%session_id,diff --git a/crates/agent_harness/src/inbound/forward.rs b/crates/agent_harness/src/inbound/forward.rsindex 6e428695f0..8f69bb1bdc 100644--- a/crates/agent_harness/src/inbound/forward.rs+++ b/crates/agent_harness/src/inbound/forward.rs@@ -72,6 +72,21 @@ where}/// Execute a command a peer replica forwarded to this one.+///+/// Instrumented because this is the far half of the hop: the request carries+/// the sending replica's `traceparent`, which the tracing layer turns into a+/// parent link, so this span shows a forwarded command executing under the+/// span that routed it instead of as an unrelated root. `outcome` distinguishes+/// the three answers the sender can get without reading the response body.+#[tracing::instrument(+ name = "harness.forward.receive",+ skip_all,+ fields(+ agent.session.id = %session_id,+ agent.command.forwarded = true,+ agent.command.outcome = tracing::field::Empty,+ )+)]async fn forward_handler<Harness, Auth>(State(state): State<ForwardGatewayState<Harness, Auth>>,_caller: MacroAuthorizationExtractor<Auth, InternalOnly>,@@ -83,8 +98,12 @@ whereAuth: MacroAuthorizationService,{let session_id = AgentSessionId::new_from_uuid(session_id);+ let span = tracing::Span::current();match state.harness.execute_forwarded(session_id, command).await {- Ok(()) => StatusCode::NO_CONTENT.into_response(),+ Ok(()) => {+ span.record("agent.command.outcome", "executed");+ StatusCode::NO_CONTENT.into_response()+ }// Surfaced with its own status so the sender can tell "the actor is// not here after all" from an execution failure: the sender's// fallback re-reads the lease on any error, but a 409 is the signal@@ -92,12 +111,16 @@ whereErr(HarnessError::Disconnected(id))| Err(HarnessError::Session(agent_session::domain::error::AgentSessionError::Disconnected(id),- )) => (- StatusCode::CONFLICT,- format!("session {id} is not attached here"),- )- .into_response(),+ )) => {+ span.record("agent.command.outcome", "not_attached_here");+ (+ StatusCode::CONFLICT,+ format!("session {id} is not attached here"),+ )+ .into_response()+ }Err(error) => {+ span.record("agent.command.outcome", "failed");tracing::error!(error = ?error, %session_id, "a forwarded command failed");(StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response()}diff --git a/crates/agent_harness/src/outbound/forward.rs b/crates/agent_harness/src/outbound/forward.rsindex 87fb4c6d55..17f307107a 100644--- a/crates/agent_harness/src/outbound/forward.rs+++ b/crates/agent_harness/src/outbound/forward.rs@@ -43,7 +43,15 @@ impl HttpCommandForwarder {}impl CommandForwarder for HttpCommandForwarder {- #[tracing::instrument(err, skip(self, command), fields(%target, %session))]+ #[tracing::instrument(+ err,+ skip(self, command),+ fields(+ %target,+ %session,+ http.response.status_code = tracing::field::Empty,+ )+ )]async fn forward(&self,target: &ReplicaAddress,@@ -55,15 +63,23 @@ impl CommandForwarder for HttpCommandForwarder {target.as_str().trim_end_matches('/'),session);+ // The peer's inbound layer already extracts `traceparent` and parents+ // its request span to the remote span (see `macro_tower_layers`), so+ // injecting here is what makes a forwarded command one trace spanning+ // both replicas. Without it the hop is two unrelated traces.+ let mut headers = reqwest::header::HeaderMap::new();+ macro_tower_layers::inject_trace_headers(&mut headers);let response = self.client.post(url).header(INTERNAL_API_KEY_HEADER, &self.internal_api_key)+ .headers(headers).json(&command).send().await.map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;let status = response.status();+ tracing::Span::current().record("http.response.status_code", status.as_u16());if status.is_success() {return Ok(());}