Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(agent_harness): trace a forwarded command across both replicas (#6082)

Wolf Mermelstein1a5282d04356
diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rs
index 0b75cb9a66..6b5627bffb 100644
--- a/crates/agent_harness/src/domain/service.rs
+++ b/crates/agent_harness/src/domain/service.rs
@@ -665,26 +665,59 @@ where
/// then its heartbeat going stale is what lets this replica take over. A
/// manager that is alive but unreachable stays an error - executing
/// locally anyway is how two actors end up on one session.
- #[tracing::instrument(err, skip(self, command), fields(%session_id))]
+ /// The routing decision is recorded on the span, not only logged: which of
+ /// the three answers the lease gave, which peer it named, and whether the
+ /// command left this process. Those are the fields you group by when a
+ /// replica is mishandling commands, and a log line cannot be aggregated.
+ #[tracing::instrument(
+ err,
+ skip(self, command),
+ fields(
+ %session_id,
+ agent.session.management = tracing::field::Empty,
+ agent.session.manager_replica = tracing::field::Empty,
+ agent.command.forwarded = tracing::field::Empty,
+ agent.command.stale_fallback = tracing::field::Empty,
+ )
+ )]
async fn route_then_execute(
&self,
session_id: AgentSessionId,
command: HarnessCommand,
) -> Result<()> {
+ let span = tracing::Span::current();
// Open never routes: it is what creates the session row this routing
// would read, and a fresh id has no manager to defer to.
if matches!(command, HarnessCommand::Open(_)) {
+ span.record("agent.session.management", "open");
+ span.record("agent.command.forwarded", false);
return self.execute(session_id, command).await;
}
let manager = match self.sessions.management(session_id).await? {
- SessionManagement::Unmanaged | SessionManagement::Ours => {
+ SessionManagement::Unmanaged => {
+ span.record("agent.session.management", "unmanaged");
+ span.record("agent.command.forwarded", false);
+ return self.execute(session_id, command).await;
+ }
+ SessionManagement::Ours => {
+ span.record("agent.session.management", "ours");
+ span.record("agent.command.forwarded", false);
return self.execute(session_id, command).await;
}
SessionManagement::Peer(manager) => manager,
};
+ span.record("agent.session.management", "peer");
+ span.record(
+ "agent.session.manager_replica",
+ tracing::field::display(manager.replica),
+ );
let Some(address) = manager.address else {
+ // Recorded false deliberately: the command stayed here, but as an
+ // error rather than a local execution.
+ span.record("agent.command.forwarded", false);
return Err(HarnessError::ManagerUnreachable(session_id));
};
+ span.record("agent.command.forwarded", true);
tracing::info!(%session_id, peer = %manager.replica, "forwarding an agent session command");
match self
.forwarder
@@ -694,6 +727,10 @@ where
Ok(()) => Ok(()),
Err(forward_error) => match self.sessions.management(session_id).await? {
SessionManagement::Unmanaged | SessionManagement::Ours => {
+ // Worth aggregating rather than only logging: routine
+ // fallbacks mean heartbeats are not keeping up, which is a
+ // different problem from an occasional dead peer.
+ span.record("agent.command.stale_fallback", true);
tracing::warn!(
error = ?forward_error,
%session_id,
diff --git a/crates/agent_harness/src/inbound/forward.rs b/crates/agent_harness/src/inbound/forward.rs
index 6e428695f0..8f69bb1bdc 100644
--- a/crates/agent_harness/src/inbound/forward.rs
+++ b/crates/agent_harness/src/inbound/forward.rs
@@ -72,6 +72,21 @@ where
}
/// Execute a command a peer replica forwarded to this one.
+///
+/// Instrumented because this is the far half of the hop: the request carries
+/// the sending replica's `traceparent`, which the tracing layer turns into a
+/// parent link, so this span shows a forwarded command executing under the
+/// span that routed it instead of as an unrelated root. `outcome` distinguishes
+/// the three answers the sender can get without reading the response body.
+#[tracing::instrument(
+ name = "harness.forward.receive",
+ skip_all,
+ fields(
+ agent.session.id = %session_id,
+ agent.command.forwarded = true,
+ agent.command.outcome = tracing::field::Empty,
+ )
+)]
async fn forward_handler<Harness, Auth>(
State(state): State<ForwardGatewayState<Harness, Auth>>,
_caller: MacroAuthorizationExtractor<Auth, InternalOnly>,
@@ -83,8 +98,12 @@ where
Auth: MacroAuthorizationService,
{
let session_id = AgentSessionId::new_from_uuid(session_id);
+ let span = tracing::Span::current();
match state.harness.execute_forwarded(session_id, command).await {
- Ok(()) => StatusCode::NO_CONTENT.into_response(),
+ Ok(()) => {
+ span.record("agent.command.outcome", "executed");
+ StatusCode::NO_CONTENT.into_response()
+ }
// Surfaced with its own status so the sender can tell "the actor is
// not here after all" from an execution failure: the sender's
// fallback re-reads the lease on any error, but a 409 is the signal
@@ -92,12 +111,16 @@ where
Err(HarnessError::Disconnected(id))
| Err(HarnessError::Session(
agent_session::domain::error::AgentSessionError::Disconnected(id),
- )) => (
- StatusCode::CONFLICT,
- format!("session {id} is not attached here"),
- )
- .into_response(),
+ )) => {
+ span.record("agent.command.outcome", "not_attached_here");
+ (
+ StatusCode::CONFLICT,
+ format!("session {id} is not attached here"),
+ )
+ .into_response()
+ }
Err(error) => {
+ span.record("agent.command.outcome", "failed");
tracing::error!(error = ?error, %session_id, "a forwarded command failed");
(StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response()
}
diff --git a/crates/agent_harness/src/outbound/forward.rs b/crates/agent_harness/src/outbound/forward.rs
index 87fb4c6d55..17f307107a 100644
--- a/crates/agent_harness/src/outbound/forward.rs
+++ b/crates/agent_harness/src/outbound/forward.rs
@@ -43,7 +43,15 @@ impl HttpCommandForwarder {
}
impl CommandForwarder for HttpCommandForwarder {
- #[tracing::instrument(err, skip(self, command), fields(%target, %session))]
+ #[tracing::instrument(
+ err,
+ skip(self, command),
+ fields(
+ %target,
+ %session,
+ http.response.status_code = tracing::field::Empty,
+ )
+ )]
async fn forward(
&self,
target: &ReplicaAddress,
@@ -55,15 +63,23 @@ impl CommandForwarder for HttpCommandForwarder {
target.as_str().trim_end_matches('/'),
session
);
+ // The peer's inbound layer already extracts `traceparent` and parents
+ // its request span to the remote span (see `macro_tower_layers`), so
+ // injecting here is what makes a forwarded command one trace spanning
+ // both replicas. Without it the hop is two unrelated traces.
+ let mut headers = reqwest::header::HeaderMap::new();
+ macro_tower_layers::inject_trace_headers(&mut headers);
let response = self
.client
.post(url)
.header(INTERNAL_API_KEY_HEADER, &self.internal_api_key)
+ .headers(headers)
.json(&command)
.send()
.await
.map_err(|error| HarnessError::Forward(rootcause::report!(error).into_dynamic()))?;
let status = response.status();
+ tracing::Span::current().record("http.response.status_code", status.as_u16());
if status.is_success() {
return Ok(());
}