Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agent_session): make agent-session teardown, turns, and auto-naming observable (#6293)

Wolf Mermelstein1c56c506e0ee
diff --git a/crates/agent_harness/src/outbound/cursor/manager.rs b/crates/agent_harness/src/outbound/cursor/manager.rs
index 1317d49a27..a148054e17 100644
--- a/crates/agent_harness/src/outbound/cursor/manager.rs
+++ b/crates/agent_harness/src/outbound/cursor/manager.rs
@@ -330,9 +330,33 @@ where
// Recheck under the activity lock after inspecting the
// turn gate. A prompt frame arriving in that window
// changes the instant and prevents a stale idle reap.
- if *activity == observed_at
- && should_reap_cursor_pipe(activity.elapsed(), active_turn)
- {
+ let raced = *activity != observed_at;
+ let idle_ms = activity.elapsed().as_millis();
+ let reaped = !raced
+ && should_reap_cursor_pipe(activity.elapsed(), active_turn);
+ // Every tick, not just the reaping one. The inputs to
+ // this decision are what tell a pipe that died of
+ // idleness apart from one pulled out from under a
+ // live turn, and after the fact only the tick that
+ // fired is reconstructable - so the deadline being
+ // long expired while a turn held it open has to be
+ // visible on the ticks that did nothing.
+ tracing::debug!(
+ %session_id,
+ agent.pipe.idle_ms = idle_ms as u64,
+ agent.pipe.active_turn = active_turn,
+ agent.pipe.activity_raced = raced,
+ agent.pipe.reaped = reaped,
+ "cursor pipe idle check"
+ );
+ if reaped {
+ let _reap = tracing::info_span!(
+ "agent.pipe.reap",
+ agent.session.id = %session_id,
+ agent.pipe.idle_ms = idle_ms as u64,
+ agent.pipe.close_cause = "idle_timeout",
+ )
+ .entered();
tracing::info!(%session_id, "idle cursor session; closing its pipe");
reaper_shutdown.cancel();
break;
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rs
index 68ccad7910..3b0f90f45f 100644
--- a/crates/agent_session/src/domain/service.rs
+++ b/crates/agent_session/src/domain/service.rs
@@ -39,6 +39,7 @@ use macro_user_id::user_id::MacroUserIdStr;
use macro_uuid::Uuid;
use tokio::sync::{Mutex, mpsc, oneshot, watch};
use tokio_util::{sync::CancellationToken, task::TaskTracker};
+use tracing::Instrument as _;
use tracing::instrument::WithSubscriber as _;
use bots::domain::models::BotId;
@@ -625,6 +626,17 @@ where
})
}
+ /// The out-of-band disconnect: a session marked dead by its opener rather
+ /// than by its own actor stopping. Spanned separately from
+ /// `agent.session.disconnect` because the two write the same log frame
+ /// for entirely different reasons - this one means the runtime never came
+ /// up at all.
+ #[tracing::instrument(
+ name = "agent.session.mark_disconnected",
+ err,
+ skip(self),
+ fields(agent.session.id = %id),
+ )]
async fn mark_disconnected(&self, id: AgentSessionId) -> Result<()> {
let mut logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());
tokio::time::timeout(
@@ -785,39 +797,75 @@ fn spawn_initial_agent_session_rename<R, Rt, Namer>(
Rt: AgentSessionRealtime + Send + Sync + 'static,
Namer: AgentSessionNameGenerator + Send + Sync + 'static,
{
- tokio::spawn(async move {
- let result: std::result::Result<(), rootcause::Report> = async {
- let session = repo
- .get(id)
- .await
- .map_err(|error| rootcause::report!(error))?;
- let Some(name) = name_generator
- .generate_name(&session, &initial_prompt)
- .await?
- else {
- return Ok(());
- };
- let renamed = repo
- .set_name_if_default(id, &name)
- .await
- .map_err(|error| rootcause::report!(error))?;
- if !renamed {
- return Ok(());
+ // Detached, so without a span of its own this work has no session id and
+ // no link to the session that spawned it - which is why a rename that
+ // fails for every session in a deployment still looks like nothing at
+ // all. The outcome is recorded rather than logged because the failure is
+ // swallowed here by design: nothing downstream ever notices a session
+ // that kept its default name.
+ let span = tracing::info_span!(
+ "agent.session.rename",
+ agent.session.id = %id,
+ agent.rename.outcome = tracing::field::Empty,
+ );
+ tokio::spawn(
+ async move {
+ let span = tracing::Span::current();
+ let result: std::result::Result<&'static str, rootcause::Report> = async {
+ let session = repo
+ .get(id)
+ .await
+ .map_err(|error| rootcause::report!(error))?;
+ let Some(name) = name_generator
+ .generate_name(&session, &initial_prompt)
+ .await?
+ else {
+ return Ok("skipped_no_name");
+ };
+ let renamed = repo
+ .set_name_if_default(id, &name)
+ .await
+ .map_err(|error| rootcause::report!(error))?;
+ if !renamed {
+ return Ok("skipped_already_named");
+ }
+ realtime
+ .publish_renamed(AgentSessionRenamed {
+ agent_session_id: id,
+ name,
+ })
+ .await?;
+ Ok("renamed")
}
- realtime
- .publish_renamed(AgentSessionRenamed {
- agent_session_id: id,
- name,
- })
- .await?;
- Ok(())
- }
- .await;
+ .await;
- if let Err(error) = result {
- tracing::warn!(error = ?error, %id, "failed to auto-rename initial agent session");
+ match result {
+ Ok(outcome) => {
+ span.record("agent.rename.outcome", outcome);
+ }
+ Err(error) => {
+ span.record("agent.rename.outcome", "failed");
+ tracing::warn!(error = ?error, %id, "failed to auto-rename initial agent session");
+ }
+ }
}
- });
+ .instrument(span),
+ );
+}
+
+/// The telemetry shape of a log frame: its kind, and the status event it
+/// carries when it is one.
+///
+/// Status events are named individually because a session's visible state is
+/// projected from them and there are only a handful. ACP traffic is counted
+/// but never named - the frame's content is user data and never belongs in a
+/// span.
+fn frame_telemetry(content: &Message) -> (&'static str, Option<&str>) {
+ match content {
+ Message::ToServer(ToServerMessage::Event { event }) => ("event", Some(event.as_str())),
+ Message::ToServer(_) => ("acp_to_server", None),
+ Message::ToRuntime(_) => ("acp_to_runtime", None),
+ }
}
fn validate_agent_session_name(raw: &str) -> Result<&str> {
@@ -1065,17 +1113,32 @@ where
Rt: AgentSessionRealtime,
{
/// Push the frame just appended out to whoever is watching the session.
+ ///
+ /// The frame's kind rides along because this span is the only per-frame
+ /// signal a session emits: a status event here is what moves the
+ /// composer's whole notion of whether the agent is working, and without
+ /// naming it "a frame was published" answers nothing.
#[tracing::instrument(
name = "agent.session.realtime.publish",
err,
skip(self, agent_session_id, entry),
- fields(agent.session.id = %agent_session_id)
+ fields(
+ agent.session.id = %agent_session_id,
+ agent.log.frame_kind = tracing::field::Empty,
+ agent.log.event = tracing::field::Empty,
+ )
)]
async fn stream(
&mut self,
agent_session_id: AgentSessionId,
entry: StoredAgentSessionLog,
) -> std::result::Result<(), rootcause::Report> {
+ let span = tracing::Span::current();
+ let (frame_kind, event) = frame_telemetry(&entry.entry.content);
+ span.record("agent.log.frame_kind", frame_kind);
+ if let Some(event) = event {
+ span.record("agent.log.event", event);
+ }
self.realtime
.publish(LogAppended {
agent_session_id,
diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rs
index 301fa7fdf5..a9ebb3238a 100644
--- a/crates/agent_session/src/domain/session/actors.rs
+++ b/crates/agent_session/src/domain/session/actors.rs
@@ -408,6 +408,16 @@ where
});
}
Effect::TurnEnded { action_id } => {
+ // The counterpart to `agent.session.disconnect`: a turn
+ // that ended here reached its stop reason, so a session
+ // whose last turn has this span and no disconnect after
+ // it finished cleanly. One per turn.
+ let _ended = tracing::info_span!(
+ "agent.session.turn_ended",
+ agent.session.id = %self.machine.id(),
+ agent.action.id = %action_id,
+ )
+ .entered();
tracing::info!(
id = %self.machine.id(),
%action_id,
@@ -425,16 +435,34 @@ where
let _ = token.completed.send(result);
}
Effect::Stop { reason } => {
+ // The one place a session's `disconnected` event is
+ // written. The frame itself carries no cause, so a
+ // routine idle teardown and a runtime lost mid-turn are
+ // indistinguishable downstream - in the log, in the
+ // status projection, and in the composer that reads it.
+ // Recording the close reason here is what tells them
+ // apart after the fact.
+ let span = tracing::info_span!(
+ "agent.session.disconnect",
+ agent.session.id = %self.machine.id(),
+ agent.session.close_reason = %reason,
+ agent.session.disconnect_persisted = tracing::field::Empty,
+ );
let terminal_log = self.log(
None,
Message::ToServer(ToServerMessage::Event {
event: SystemEvent::Disconnected,
}),
);
- let result = tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, terminal_log).await;
+ let result = tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, terminal_log)
+ .instrument(span.clone())
+ .await;
match result {
- Ok(Ok(())) => {}
+ Ok(Ok(())) => {
+ span.record("agent.session.disconnect_persisted", "yes");
+ }
Ok(Err(error)) => {
+ span.record("agent.session.disconnect_persisted", "failed");
tracing::error!(
error = ?error,
id = %self.machine.id(),
@@ -442,6 +470,7 @@ where
);
}
Err(_) => {
+ span.record("agent.session.disconnect_persisted", "timed_out");
tracing::error!(
id = %self.machine.id(),
"agent session timed out persisting its disconnect"
diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rs
index 5f9d5f57d1..b2edda72ef 100644
--- a/crates/cursor_cloud_agents/src/domain/service.rs
+++ b/crates/cursor_cloud_agents/src/domain/service.rs
@@ -463,11 +463,53 @@ where
}
/// Run a prompt retaining its original ACP blocks in the native journal.
+ ///
+ /// This is the whole of a Cursor turn, and the ACP path calls straight
+ /// through here rather than through [`Self::prompt`] - so an
+ /// uninstrumented body leaves a turn with no span at all.
+ ///
+ /// The outcome is recorded on the way out rather than left to `err`.
+ /// A turn whose future is dropped - a pipe torn down under it - still
+ /// closes its span, with no error and no recorded outcome, which is
+ /// otherwise indistinguishable from a turn that ended normally. An unset
+ /// `agent.turn.outcome` is what tells those two apart.
+ #[tracing::instrument(
+ name = "agent.turn",
+ skip_all,
+ fields(
+ agent.acp.session_id = ?session_id,
+ cursor.agent.id = tracing::field::Empty,
+ cursor.run.id = tracing::field::Empty,
+ agent.turn.stop_reason = tracing::field::Empty,
+ agent.turn.outcome = tracing::field::Empty,
+ ),
+ err,
+ )]
pub async fn prompt_content(
&self,
session_id: &SessionId,
prompt: &str,
blocks: Vec<ContentBlock>,
+ ) -> Result<StopReason, SessionError> {
+ let outcome = self.run_turn(session_id, prompt, blocks).await;
+ let span = tracing::Span::current();
+ match &outcome {
+ Ok(stop_reason) => {
+ span.record("agent.turn.stop_reason", tracing::field::debug(stop_reason));
+ span.record("agent.turn.outcome", "completed");
+ }
+ Err(_) => {
+ span.record("agent.turn.outcome", "failed");
+ }
+ }
+ outcome
+ }
+
+ async fn run_turn(
+ &self,
+ session_id: &SessionId,
+ prompt: &str,
+ blocks: Vec<ContentBlock>,
) -> Result<StopReason, SessionError> {
let session = self.session(session_id)?;
// Wait behind an in-flight foreign-run mirror, so its frames and
@@ -609,6 +651,12 @@ where
// not observe/project the new run until every older run is reconciled.
self.backfill_foreign_runs(session_id, &session, &agent, Some(&run))
.await?;
+ // The turn span is the only place all three identities meet, and it
+ // is what makes a Macro session joinable to the cursor.com run that
+ // served it.
+ let span = tracing::Span::current();
+ span.record("cursor.agent.id", tracing::field::display(&agent));
+ span.record("cursor.run.id", tracing::field::display(&run));
tracing::info!(%agent, %run, "cursor run started");
let cancelled_before_the_run = {
let mut state = session.state.lock().expect("session state poisoned");
@@ -1105,6 +1153,27 @@ where
}
}
+ /// One poll of a running turn.
+ ///
+ /// Named explicitly so a rename of the client method underneath cannot
+ /// silently take the span with it: this loop is the only continuous
+ /// heartbeat a Cursor turn has, and a turn that stops polling is the
+ /// first evidence that something took it down.
+ ///
+ /// `debug` because the loop runs every [`POLL_INTERVAL`]; the turn span
+ /// above carries the outcome, this carries the liveness.
+ #[tracing::instrument(
+ name = "cursor.run.poll",
+ level = "debug",
+ skip_all,
+ fields(
+ agent.acp.session_id = ?session_id,
+ cursor.agent.id = %agent,
+ cursor.run.id = %run,
+ cursor.run.status = tracing::field::Empty,
+ ),
+ err,
+ )]
async fn poll_once(
&self,
session_id: &SessionId,
diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rs
index 963372690b..94747212b2 100644
--- a/crates/macro_entrypoint/src/lib.rs
+++ b/crates/macro_entrypoint/src/lib.rs
@@ -44,13 +44,30 @@ maybe_env_vars! {
/// `APP_SECRETS_JSON` (the way the rest of our config is injected, see `macro_env_var`) is ignored
/// and our tracing filter ends up wrong. This reads `RUST_LOG` the same way `macro_env_var` does —
/// `APP_SECRETS_JSON` first, then the process environment as a fallback.
+/// Both branches carry an INFO floor, for the same reason
+/// [`otel_trace_filter`] does. A builder with no default directive and an
+/// absent or empty `RUST_LOG` yields a filter with *zero* directives, which
+/// logs nothing at all - not even ERROR. A service deployed without an
+/// explicit `RUST_LOG` is then silent, while its spans keep flowing, so it
+/// reads as healthy-but-quiet rather than as misconfigured.
fn rust_log_env_filter() -> EnvFilter {
match RustLog::new() {
- Some(value) => EnvFilter::builder().parse_lossy(value),
- None => EnvFilter::from_default_env(),
+ Some(value) => rust_log_filter(Some(&value)),
+ // `from_env_lossy` reads `RUST_LOG` itself, so the process
+ // environment stays behind the builder rather than a direct
+ // `std::env::var` (CS-14).
+ None => rust_log_builder().from_env_lossy(),
}
}
+fn rust_log_builder() -> tracing_subscriber::filter::Builder {
+ EnvFilter::builder().with_default_directive(LevelFilter::INFO.into())
+}
+
+fn rust_log_filter(value: Option<&str>) -> EnvFilter {
+ rust_log_builder().parse_lossy(value.unwrap_or(""))
+}
+
/// Build an [`EnvFilter`] for the OpenTelemetry span exporter from `OTEL_TRACE_FILTER`.
///
/// Traces are filtered independently of `RUST_LOG` so that lowering log verbosity (e.g.
diff --git a/crates/macro_entrypoint/src/test.rs b/crates/macro_entrypoint/src/test.rs
index 87b0369134..0ea8f11363 100644
--- a/crates/macro_entrypoint/src/test.rs
+++ b/crates/macro_entrypoint/src/test.rs
@@ -248,3 +248,59 @@ fn otel_trace_filter_invalid_value_falls_back_to_info() {
assert_eq!(exporter.get_finished_spans().unwrap().len(), 1);
}
+
+#[test]
+fn absent_rust_log_still_logs_info() {
+ // A builder with no default directive and an empty value parses to zero
+ // directives, which logs nothing at all - the failure mode this floor
+ // exists to prevent. A service deployed without `RUST_LOG` must still
+ // emit its INFO lines, because those are what carry the decisions worth
+ // reconstructing after the fact.
+ for value in [None, Some("")] {
+ let logs = SharedWriter::default();
+ let subscriber = Registry::default().with(
+ tracing_subscriber::fmt::layer()
+ .with_ansi(false)
+ .with_writer(logs.clone())
+ .with_filter(rust_log_filter(value)),
+ );
+
+ tracing::subscriber::with_default(subscriber, || {
+ tracing::info!("info survives an unset RUST_LOG");
+ tracing::debug!("debug is still filtered");
+ });
+
+ let output = logs.contents();
+ assert!(
+ output.contains("info survives an unset RUST_LOG"),
+ "RUST_LOG={value:?} should keep INFO: {output:?}"
+ );
+ assert!(
+ !output.contains("debug is still filtered"),
+ "RUST_LOG={value:?} should not raise the floor to DEBUG: {output:?}"
+ );
+ }
+}
+
+#[test]
+fn explicit_rust_log_still_wins_over_the_floor() {
+ let logs = SharedWriter::default();
+ let subscriber = Registry::default().with(
+ tracing_subscriber::fmt::layer()
+ .with_ansi(false)
+ .with_writer(logs.clone())
+ .with_filter(rust_log_filter(Some("warn"))),
+ );
+
+ tracing::subscriber::with_default(subscriber, || {
+ tracing::warn!("warn is kept");
+ tracing::info!("info is filtered by an explicit directive");
+ });
+
+ let output = logs.contents();
+ assert!(output.contains("warn is kept"), "{output:?}");
+ assert!(
+ !output.contains("info is filtered by an explicit directive"),
+ "an explicit RUST_LOG must override the default floor: {output:?}"
+ );
+}
diff --git a/docs/AGENT_GUIDE/observability.md b/docs/AGENT_GUIDE/observability.md
index 6fd7a033f0..ef8e9d5503 100644
--- a/docs/AGENT_GUIDE/observability.md
+++ b/docs/AGENT_GUIDE/observability.md
@@ -67,3 +67,32 @@ trace context — timestamps + service are the only join for those.
(structured metadata), so prefer erroring *handlers* as log entry points.
- Frontend spans stop at the fetch: no spans for user interactions or the websocket-delivered
results, so async flows (AI edits applying, message fan-out) have no trace at all.
+
+## Agent sessions
+
+A session's lifetime is reconstructable from these spans. All of them carry
+`agent.session.id` — the Macro session UUID, and only ever that. The ACP-local
+session name (`cursor-acp-1`) is `agent.acp.session_id`; the two are different
+identifier spaces and must not be confused.
+
+| Span | Answers |
+| --- | --- |
+| `agent.turn` | Did a Cursor turn run, and how did it end? `agent.turn.stop_reason` / `agent.turn.outcome`, plus `cursor.agent.id` / `cursor.run.id`. |
+| `cursor.run.poll` | Is a turn still alive? One per poll, at DEBUG. |
+| `agent.session.turn_ended` | The turn reached a stop reason. |
+| `agent.session.disconnect` | The session's actor wrote a `disconnected` event, and `agent.session.close_reason` says why. |
+| `agent.session.mark_disconnected` | The session was marked dead by its opener because the runtime never came up. |
+| `agent.pipe.reap` | A Cursor pipe was closed for idleness, with `agent.pipe.idle_ms`. |
+| `agent.session.realtime.publish` | A frame reached watchers; `agent.log.event` names the status event when it is one. |
+| `agent.session.rename` | Auto-naming ran; `agent.rename.outcome` says whether it named, skipped, or failed. |
+
+Two things worth knowing when reading these:
+
+- **An unset `agent.turn.outcome` is a signal, not missing data.** `#[instrument(err)]`
+ records an error only on an `Err` return, so a turn whose future is *dropped* — a pipe torn
+ down under it — closes its span with no error and looks identical to a clean finish. The
+ outcome field is recorded explicitly on the way out; if it is absent, the turn did not
+ return.
+- **The idle-check DEBUG line fires every tick, not just the reaping one.** `agent.pipe.reaped`,
+ `agent.pipe.active_turn` and `agent.pipe.idle_ms` on the ticks that did *nothing* are what
+ show a deadline sitting long expired while a live turn held the pipe open.