Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(agent_session): make agent-session teardown, turns, and auto-naming observable (#6293)
Wolf Mermelstein1c56c506e0ee
diff --git a/crates/agent_harness/src/outbound/cursor/manager.rs b/crates/agent_harness/src/outbound/cursor/manager.rsindex 1317d49a27..a148054e17 100644--- a/crates/agent_harness/src/outbound/cursor/manager.rs+++ b/crates/agent_harness/src/outbound/cursor/manager.rs@@ -330,9 +330,33 @@ where// Recheck under the activity lock after inspecting the// turn gate. A prompt frame arriving in that window// changes the instant and prevents a stale idle reap.- if *activity == observed_at- && should_reap_cursor_pipe(activity.elapsed(), active_turn)- {+ let raced = *activity != observed_at;+ let idle_ms = activity.elapsed().as_millis();+ let reaped = !raced+ && should_reap_cursor_pipe(activity.elapsed(), active_turn);+ // Every tick, not just the reaping one. The inputs to+ // this decision are what tell a pipe that died of+ // idleness apart from one pulled out from under a+ // live turn, and after the fact only the tick that+ // fired is reconstructable - so the deadline being+ // long expired while a turn held it open has to be+ // visible on the ticks that did nothing.+ tracing::debug!(+ %session_id,+ agent.pipe.idle_ms = idle_ms as u64,+ agent.pipe.active_turn = active_turn,+ agent.pipe.activity_raced = raced,+ agent.pipe.reaped = reaped,+ "cursor pipe idle check"+ );+ if reaped {+ let _reap = tracing::info_span!(+ "agent.pipe.reap",+ agent.session.id = %session_id,+ agent.pipe.idle_ms = idle_ms as u64,+ agent.pipe.close_cause = "idle_timeout",+ )+ .entered();tracing::info!(%session_id, "idle cursor session; closing its pipe");reaper_shutdown.cancel();break;diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex 68ccad7910..3b0f90f45f 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -39,6 +39,7 @@ use macro_user_id::user_id::MacroUserIdStr;use macro_uuid::Uuid;use tokio::sync::{Mutex, mpsc, oneshot, watch};use tokio_util::{sync::CancellationToken, task::TaskTracker};+use tracing::Instrument as _;use tracing::instrument::WithSubscriber as _;use bots::domain::models::BotId;@@ -625,6 +626,17 @@ where})}+ /// The out-of-band disconnect: a session marked dead by its opener rather+ /// than by its own actor stopping. Spanned separately from+ /// `agent.session.disconnect` because the two write the same log frame+ /// for entirely different reasons - this one means the runtime never came+ /// up at all.+ #[tracing::instrument(+ name = "agent.session.mark_disconnected",+ err,+ skip(self),+ fields(agent.session.id = %id),+ )]async fn mark_disconnected(&self, id: AgentSessionId) -> Result<()> {let mut logs = LiveSessionLogWriter::new(self.repo.clone(), self.realtime.clone());tokio::time::timeout(@@ -785,39 +797,75 @@ fn spawn_initial_agent_session_rename<R, Rt, Namer>(Rt: AgentSessionRealtime + Send + Sync + 'static,Namer: AgentSessionNameGenerator + Send + Sync + 'static,{- tokio::spawn(async move {- let result: std::result::Result<(), rootcause::Report> = async {- let session = repo- .get(id)- .await- .map_err(|error| rootcause::report!(error))?;- let Some(name) = name_generator- .generate_name(&session, &initial_prompt)- .await?- else {- return Ok(());- };- let renamed = repo- .set_name_if_default(id, &name)- .await- .map_err(|error| rootcause::report!(error))?;- if !renamed {- return Ok(());+ // Detached, so without a span of its own this work has no session id and+ // no link to the session that spawned it - which is why a rename that+ // fails for every session in a deployment still looks like nothing at+ // all. The outcome is recorded rather than logged because the failure is+ // swallowed here by design: nothing downstream ever notices a session+ // that kept its default name.+ let span = tracing::info_span!(+ "agent.session.rename",+ agent.session.id = %id,+ agent.rename.outcome = tracing::field::Empty,+ );+ tokio::spawn(+ async move {+ let span = tracing::Span::current();+ let result: std::result::Result<&'static str, rootcause::Report> = async {+ let session = repo+ .get(id)+ .await+ .map_err(|error| rootcause::report!(error))?;+ let Some(name) = name_generator+ .generate_name(&session, &initial_prompt)+ .await?+ else {+ return Ok("skipped_no_name");+ };+ let renamed = repo+ .set_name_if_default(id, &name)+ .await+ .map_err(|error| rootcause::report!(error))?;+ if !renamed {+ return Ok("skipped_already_named");+ }+ realtime+ .publish_renamed(AgentSessionRenamed {+ agent_session_id: id,+ name,+ })+ .await?;+ Ok("renamed")}- realtime- .publish_renamed(AgentSessionRenamed {- agent_session_id: id,- name,- })- .await?;- Ok(())- }- .await;+ .await;- if let Err(error) = result {- tracing::warn!(error = ?error, %id, "failed to auto-rename initial agent session");+ match result {+ Ok(outcome) => {+ span.record("agent.rename.outcome", outcome);+ }+ Err(error) => {+ span.record("agent.rename.outcome", "failed");+ tracing::warn!(error = ?error, %id, "failed to auto-rename initial agent session");+ }+ }}- });+ .instrument(span),+ );+}++/// The telemetry shape of a log frame: its kind, and the status event it+/// carries when it is one.+///+/// Status events are named individually because a session's visible state is+/// projected from them and there are only a handful. ACP traffic is counted+/// but never named - the frame's content is user data and never belongs in a+/// span.+fn frame_telemetry(content: &Message) -> (&'static str, Option<&str>) {+ match content {+ Message::ToServer(ToServerMessage::Event { event }) => ("event", Some(event.as_str())),+ Message::ToServer(_) => ("acp_to_server", None),+ Message::ToRuntime(_) => ("acp_to_runtime", None),+ }}fn validate_agent_session_name(raw: &str) -> Result<&str> {@@ -1065,17 +1113,32 @@ whereRt: AgentSessionRealtime,{/// Push the frame just appended out to whoever is watching the session.+ ///+ /// The frame's kind rides along because this span is the only per-frame+ /// signal a session emits: a status event here is what moves the+ /// composer's whole notion of whether the agent is working, and without+ /// naming it "a frame was published" answers nothing.#[tracing::instrument(name = "agent.session.realtime.publish",err,skip(self, agent_session_id, entry),- fields(agent.session.id = %agent_session_id)+ fields(+ agent.session.id = %agent_session_id,+ agent.log.frame_kind = tracing::field::Empty,+ agent.log.event = tracing::field::Empty,+ ))]async fn stream(&mut self,agent_session_id: AgentSessionId,entry: StoredAgentSessionLog,) -> std::result::Result<(), rootcause::Report> {+ let span = tracing::Span::current();+ let (frame_kind, event) = frame_telemetry(&entry.entry.content);+ span.record("agent.log.frame_kind", frame_kind);+ if let Some(event) = event {+ span.record("agent.log.event", event);+ }self.realtime.publish(LogAppended {agent_session_id,diff --git a/crates/agent_session/src/domain/session/actors.rs b/crates/agent_session/src/domain/session/actors.rsindex 301fa7fdf5..a9ebb3238a 100644--- a/crates/agent_session/src/domain/session/actors.rs+++ b/crates/agent_session/src/domain/session/actors.rs@@ -408,6 +408,16 @@ where});}Effect::TurnEnded { action_id } => {+ // The counterpart to `agent.session.disconnect`: a turn+ // that ended here reached its stop reason, so a session+ // whose last turn has this span and no disconnect after+ // it finished cleanly. One per turn.+ let _ended = tracing::info_span!(+ "agent.session.turn_ended",+ agent.session.id = %self.machine.id(),+ agent.action.id = %action_id,+ )+ .entered();tracing::info!(id = %self.machine.id(),%action_id,@@ -425,16 +435,34 @@ wherelet _ = token.completed.send(result);}Effect::Stop { reason } => {+ // The one place a session's `disconnected` event is+ // written. The frame itself carries no cause, so a+ // routine idle teardown and a runtime lost mid-turn are+ // indistinguishable downstream - in the log, in the+ // status projection, and in the composer that reads it.+ // Recording the close reason here is what tells them+ // apart after the fact.+ let span = tracing::info_span!(+ "agent.session.disconnect",+ agent.session.id = %self.machine.id(),+ agent.session.close_reason = %reason,+ agent.session.disconnect_persisted = tracing::field::Empty,+ );let terminal_log = self.log(None,Message::ToServer(ToServerMessage::Event {event: SystemEvent::Disconnected,}),);- let result = tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, terminal_log).await;+ let result = tokio::time::timeout(COMMAND_DELIVERY_TIMEOUT, terminal_log)+ .instrument(span.clone())+ .await;match result {- Ok(Ok(())) => {}+ Ok(Ok(())) => {+ span.record("agent.session.disconnect_persisted", "yes");+ }Ok(Err(error)) => {+ span.record("agent.session.disconnect_persisted", "failed");tracing::error!(error = ?error,id = %self.machine.id(),@@ -442,6 +470,7 @@ where);}Err(_) => {+ span.record("agent.session.disconnect_persisted", "timed_out");tracing::error!(id = %self.machine.id(),"agent session timed out persisting its disconnect"diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rsindex 5f9d5f57d1..b2edda72ef 100644--- a/crates/cursor_cloud_agents/src/domain/service.rs+++ b/crates/cursor_cloud_agents/src/domain/service.rs@@ -463,11 +463,53 @@ where}/// Run a prompt retaining its original ACP blocks in the native journal.+ ///+ /// This is the whole of a Cursor turn, and the ACP path calls straight+ /// through here rather than through [`Self::prompt`] - so an+ /// uninstrumented body leaves a turn with no span at all.+ ///+ /// The outcome is recorded on the way out rather than left to `err`.+ /// A turn whose future is dropped - a pipe torn down under it - still+ /// closes its span, with no error and no recorded outcome, which is+ /// otherwise indistinguishable from a turn that ended normally. An unset+ /// `agent.turn.outcome` is what tells those two apart.+ #[tracing::instrument(+ name = "agent.turn",+ skip_all,+ fields(+ agent.acp.session_id = ?session_id,+ cursor.agent.id = tracing::field::Empty,+ cursor.run.id = tracing::field::Empty,+ agent.turn.stop_reason = tracing::field::Empty,+ agent.turn.outcome = tracing::field::Empty,+ ),+ err,+ )]pub async fn prompt_content(&self,session_id: &SessionId,prompt: &str,blocks: Vec<ContentBlock>,+ ) -> Result<StopReason, SessionError> {+ let outcome = self.run_turn(session_id, prompt, blocks).await;+ let span = tracing::Span::current();+ match &outcome {+ Ok(stop_reason) => {+ span.record("agent.turn.stop_reason", tracing::field::debug(stop_reason));+ span.record("agent.turn.outcome", "completed");+ }+ Err(_) => {+ span.record("agent.turn.outcome", "failed");+ }+ }+ outcome+ }++ async fn run_turn(+ &self,+ session_id: &SessionId,+ prompt: &str,+ blocks: Vec<ContentBlock>,) -> Result<StopReason, SessionError> {let session = self.session(session_id)?;// Wait behind an in-flight foreign-run mirror, so its frames and@@ -609,6 +651,12 @@ where// not observe/project the new run until every older run is reconciled.self.backfill_foreign_runs(session_id, &session, &agent, Some(&run)).await?;+ // The turn span is the only place all three identities meet, and it+ // is what makes a Macro session joinable to the cursor.com run that+ // served it.+ let span = tracing::Span::current();+ span.record("cursor.agent.id", tracing::field::display(&agent));+ span.record("cursor.run.id", tracing::field::display(&run));tracing::info!(%agent, %run, "cursor run started");let cancelled_before_the_run = {let mut state = session.state.lock().expect("session state poisoned");@@ -1105,6 +1153,27 @@ where}}+ /// One poll of a running turn.+ ///+ /// Named explicitly so a rename of the client method underneath cannot+ /// silently take the span with it: this loop is the only continuous+ /// heartbeat a Cursor turn has, and a turn that stops polling is the+ /// first evidence that something took it down.+ ///+ /// `debug` because the loop runs every [`POLL_INTERVAL`]; the turn span+ /// above carries the outcome, this carries the liveness.+ #[tracing::instrument(+ name = "cursor.run.poll",+ level = "debug",+ skip_all,+ fields(+ agent.acp.session_id = ?session_id,+ cursor.agent.id = %agent,+ cursor.run.id = %run,+ cursor.run.status = tracing::field::Empty,+ ),+ err,+ )]async fn poll_once(&self,session_id: &SessionId,diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rsindex 963372690b..94747212b2 100644--- a/crates/macro_entrypoint/src/lib.rs+++ b/crates/macro_entrypoint/src/lib.rs@@ -44,13 +44,30 @@ maybe_env_vars! {/// `APP_SECRETS_JSON` (the way the rest of our config is injected, see `macro_env_var`) is ignored/// and our tracing filter ends up wrong. This reads `RUST_LOG` the same way `macro_env_var` does —/// `APP_SECRETS_JSON` first, then the process environment as a fallback.+/// Both branches carry an INFO floor, for the same reason+/// [`otel_trace_filter`] does. A builder with no default directive and an+/// absent or empty `RUST_LOG` yields a filter with *zero* directives, which+/// logs nothing at all - not even ERROR. A service deployed without an+/// explicit `RUST_LOG` is then silent, while its spans keep flowing, so it+/// reads as healthy-but-quiet rather than as misconfigured.fn rust_log_env_filter() -> EnvFilter {match RustLog::new() {- Some(value) => EnvFilter::builder().parse_lossy(value),- None => EnvFilter::from_default_env(),+ Some(value) => rust_log_filter(Some(&value)),+ // `from_env_lossy` reads `RUST_LOG` itself, so the process+ // environment stays behind the builder rather than a direct+ // `std::env::var` (CS-14).+ None => rust_log_builder().from_env_lossy(),}}+fn rust_log_builder() -> tracing_subscriber::filter::Builder {+ EnvFilter::builder().with_default_directive(LevelFilter::INFO.into())+}++fn rust_log_filter(value: Option<&str>) -> EnvFilter {+ rust_log_builder().parse_lossy(value.unwrap_or(""))+}+/// Build an [`EnvFilter`] for the OpenTelemetry span exporter from `OTEL_TRACE_FILTER`.////// Traces are filtered independently of `RUST_LOG` so that lowering log verbosity (e.g.diff --git a/crates/macro_entrypoint/src/test.rs b/crates/macro_entrypoint/src/test.rsindex 87b0369134..0ea8f11363 100644--- a/crates/macro_entrypoint/src/test.rs+++ b/crates/macro_entrypoint/src/test.rs@@ -248,3 +248,59 @@ fn otel_trace_filter_invalid_value_falls_back_to_info() {assert_eq!(exporter.get_finished_spans().unwrap().len(), 1);}++#[test]+fn absent_rust_log_still_logs_info() {+ // A builder with no default directive and an empty value parses to zero+ // directives, which logs nothing at all - the failure mode this floor+ // exists to prevent. A service deployed without `RUST_LOG` must still+ // emit its INFO lines, because those are what carry the decisions worth+ // reconstructing after the fact.+ for value in [None, Some("")] {+ let logs = SharedWriter::default();+ let subscriber = Registry::default().with(+ tracing_subscriber::fmt::layer()+ .with_ansi(false)+ .with_writer(logs.clone())+ .with_filter(rust_log_filter(value)),+ );++ tracing::subscriber::with_default(subscriber, || {+ tracing::info!("info survives an unset RUST_LOG");+ tracing::debug!("debug is still filtered");+ });++ let output = logs.contents();+ assert!(+ output.contains("info survives an unset RUST_LOG"),+ "RUST_LOG={value:?} should keep INFO: {output:?}"+ );+ assert!(+ !output.contains("debug is still filtered"),+ "RUST_LOG={value:?} should not raise the floor to DEBUG: {output:?}"+ );+ }+}++#[test]+fn explicit_rust_log_still_wins_over_the_floor() {+ let logs = SharedWriter::default();+ let subscriber = Registry::default().with(+ tracing_subscriber::fmt::layer()+ .with_ansi(false)+ .with_writer(logs.clone())+ .with_filter(rust_log_filter(Some("warn"))),+ );++ tracing::subscriber::with_default(subscriber, || {+ tracing::warn!("warn is kept");+ tracing::info!("info is filtered by an explicit directive");+ });++ let output = logs.contents();+ assert!(output.contains("warn is kept"), "{output:?}");+ assert!(+ !output.contains("info is filtered by an explicit directive"),+ "an explicit RUST_LOG must override the default floor: {output:?}"+ );+}diff --git a/docs/AGENT_GUIDE/observability.md b/docs/AGENT_GUIDE/observability.mdindex 6fd7a033f0..ef8e9d5503 100644--- a/docs/AGENT_GUIDE/observability.md+++ b/docs/AGENT_GUIDE/observability.md@@ -67,3 +67,32 @@ trace context — timestamps + service are the only join for those.(structured metadata), so prefer erroring *handlers* as log entry points.- Frontend spans stop at the fetch: no spans for user interactions or the websocket-deliveredresults, so async flows (AI edits applying, message fan-out) have no trace at all.++## Agent sessions++A session's lifetime is reconstructable from these spans. All of them carry+`agent.session.id` — the Macro session UUID, and only ever that. The ACP-local+session name (`cursor-acp-1`) is `agent.acp.session_id`; the two are different+identifier spaces and must not be confused.++| Span | Answers |+| --- | --- |+| `agent.turn` | Did a Cursor turn run, and how did it end? `agent.turn.stop_reason` / `agent.turn.outcome`, plus `cursor.agent.id` / `cursor.run.id`. |+| `cursor.run.poll` | Is a turn still alive? One per poll, at DEBUG. |+| `agent.session.turn_ended` | The turn reached a stop reason. |+| `agent.session.disconnect` | The session's actor wrote a `disconnected` event, and `agent.session.close_reason` says why. |+| `agent.session.mark_disconnected` | The session was marked dead by its opener because the runtime never came up. |+| `agent.pipe.reap` | A Cursor pipe was closed for idleness, with `agent.pipe.idle_ms`. |+| `agent.session.realtime.publish` | A frame reached watchers; `agent.log.event` names the status event when it is one. |+| `agent.session.rename` | Auto-naming ran; `agent.rename.outcome` says whether it named, skipped, or failed. |++Two things worth knowing when reading these:++- **An unset `agent.turn.outcome` is a signal, not missing data.** `#[instrument(err)]`+ records an error only on an `Err` return, so a turn whose future is *dropped* — a pipe torn+ down under it — closes its span with no error and looks identical to a clean finish. The+ outcome field is recorded explicitly on the way out; if it is absent, the turn did not+ return.+- **The idle-check DEBUG line fires every tick, not just the reaping one.** `agent.pipe.reaped`,+ `agent.pipe.active_turn` and `agent.pipe.idle_ms` on the ticks that did *nothing* are what+ show a deadline sitting long expired while a live turn held the pipe open.