Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(cursor): let a stop reach a prompt stuck behind a foreign-run mirror (#7082)

Wolf Mermelstein82d42ecbd786
diff --git a/crates/agent_harness/src/domain/pending.rs b/crates/agent_harness/src/domain/pending.rs
index 86bd5a0a3c..80f7d8bcb9 100644
--- a/crates/agent_harness/src/domain/pending.rs
+++ b/crates/agent_harness/src/domain/pending.rs
@@ -96,6 +96,7 @@ mod test {
actor: None,
announce: None,
announcement_message_id: None,
+ dispatched_at: chrono::Utc::now(),
}
}
diff --git a/crates/agent_harness/src/domain/queue.rs b/crates/agent_harness/src/domain/queue.rs
index fecf4a3427..cde76d26cb 100644
--- a/crates/agent_harness/src/domain/queue.rs
+++ b/crates/agent_harness/src/domain/queue.rs
@@ -89,9 +89,20 @@ pub struct InFlightTurn {
/// [`SessionAnnouncer::resolve`](super::ports::SessionAnnouncer::resolve)).
/// Downstream events carry it under this name for both.
pub announcement_message_id: Option<Uuid>,
+ /// When delivery named this turn. What a prompt parked behind it, or
+ /// an idle check that keeps finding it, reports as the turn's age: a
+ /// turn hours old with nothing streaming is the shape of a wedged
+ /// session, and without this it looks exactly like a long one.
+ pub dispatched_at: DateTime<Utc>,
}
impl InFlightTurn {
+ /// How long this turn has been in flight.
+ #[must_use]
+ pub fn age(&self) -> chrono::Duration {
+ Utc::now().signed_duration_since(self.dispatched_at)
+ }
+
/// This turn as a session that died underneath it reports it.
#[must_use]
pub fn summary(&self) -> InFlightTurnSummary {
diff --git a/crates/agent_harness/src/domain/service/queue.rs b/crates/agent_harness/src/domain/service/queue.rs
index 1d79f10288..efa97e9779 100644
--- a/crates/agent_harness/src/domain/service/queue.rs
+++ b/crates/agent_harness/src/domain/service/queue.rs
@@ -633,6 +633,27 @@ where
}
let dispatched = if self.busy.is_pending(session_id) {
+ // Said out loud, with the turn it waits behind and that turn's
+ // age: a queue that never drains is otherwise silent on this
+ // replica, and the question "what is it waiting for, and since
+ // when" is the one that has to be answerable from the logs.
+ match self.busy.turn(session_id) {
+ Some(turn) => tracing::info!(
+ %session_id,
+ %action_id,
+ in_flight_turn = turn.turn.0,
+ in_flight_action_id = %turn.action_id,
+ in_flight_age_secs = turn.age().num_seconds(),
+ queued = self.queues.list(session_id).len(),
+ "action queued behind the turn in flight"
+ ),
+ None => tracing::info!(
+ %session_id,
+ %action_id,
+ queued = self.queues.list(session_id).len(),
+ "action queued behind a command admitted but not yet dispatched"
+ ),
+ }
Ok(())
} else {
// Marked before dispatching, not only once `dispatch_next`'s own
@@ -905,6 +926,7 @@ where
actor: entry.actor,
announce: entry.announce,
announcement_message_id: entry.announced,
+ dispatched_at: chrono::Utc::now(),
};
self.busy.mark_turn(session_id, turn.clone());
self.publish_lifecycle(session_id, |identity| {
diff --git a/crates/agent_harness/src/outbound/cursor/manager.rs b/crates/agent_harness/src/outbound/cursor/manager.rs
index fdc3859f6d..70a8b8279a 100644
--- a/crates/agent_harness/src/outbound/cursor/manager.rs
+++ b/crates/agent_harness/src/outbound/cursor/manager.rs
@@ -36,6 +36,7 @@ use cursor_cloud_agents::domain::ports::{
use cursor_cloud_agents::domain::service::CursorSessionService;
use cursor_cloud_agents::inbound::acp::{AcpNotifier, serve};
use futures::Stream;
+use tracing::Instrument as _;
use super::keys::CursorApiKeys;
use super::pipe::PipeTransport;
@@ -94,6 +95,38 @@ fn should_reap_cursor_pipe(idle: std::time::Duration, active_turn: bool, pending
idle >= CURSOR_IDLE_TIMEOUT && !active_turn && !pending
}
+/// How long a pipe may sit idle past its deadline, kept open only by a turn
+/// or an admitted command, before the idle check says so at `warn`.
+///
+/// The idle check logs its inputs every tick at `debug`, which production
+/// does not ship. A long Cursor run legitimately holds a pipe open for an
+/// hour with nothing moving through it, so the first half hour is nobody's
+/// business - but a pipe held open this long, and again every interval
+/// after, is either a very long run or a gate nobody will ever release, and
+/// the second was found only by noticing which sessions were *not* reaped.
+const HELD_OPEN_WARNING_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30 * 60);
+
+/// Whether this tick should warn that the pipe is held open, and the
+/// threshold the next warning waits for.
+///
+/// `next` is the idle duration the next warning fires at; a pipe that saw
+/// activity again resets it, so a session that is simply busy every day
+/// warns once per long stretch rather than once per process.
+fn held_open_warning(
+ idle: std::time::Duration,
+ held: bool,
+ next: std::time::Duration,
+) -> (bool, std::time::Duration) {
+ if !held || idle < CURSOR_IDLE_TIMEOUT {
+ return (false, HELD_OPEN_WARNING_INTERVAL);
+ }
+ if idle >= next {
+ (true, next + HELD_OPEN_WARNING_INTERVAL)
+ } else {
+ (false, next)
+ }
+}
+
/// The ref new agents start their work from.
const DEFAULT_STARTING_REF: &str = "main";
@@ -471,9 +504,18 @@ where
let observed = Arc::clone(&last_activity);
let reaper_shutdown = shutdown.clone();
let pending = self.pending.clone();
+ // The Macro session id on everything this task does. The mirror's
+ // own spans know only the ACP session id (`cursor-acp-1` for every
+ // hosted session), so without this a run it follows for an hour is
+ // unsearchable by the session it belongs to.
+ let background = tracing::info_span!(
+ "agent.session.background",
+ agent.session.id = %session_id,
+ );
tokio::spawn(async move {
let mut mirror = interval_from_now(FOREIGN_SYNC_INTERVAL);
let mut reaper = interval_from_now(CURSOR_IDLE_CHECK_INTERVAL);
+ let mut next_held_open_warning = HELD_OPEN_WARNING_INTERVAL;
loop {
tokio::select! {
() = pipe_closed.cancelled() => break,
@@ -530,11 +572,30 @@ where
reaper_shutdown.cancel();
break;
}
+ let (warn, next) = held_open_warning(
+ activity.elapsed(),
+ active_turn || pending_command,
+ next_held_open_warning,
+ );
+ next_held_open_warning = next;
+ if warn {
+ let in_flight = pending.turn(session_id);
+ tracing::warn!(
+ %session_id,
+ agent.pipe.idle_ms = idle_ms as u64,
+ agent.pipe.active_turn = active_turn,
+ agent.pipe.pending_command = pending_command,
+ in_flight_turn = in_flight.as_ref().map(|turn| turn.turn.0),
+ in_flight_action_id = in_flight.as_ref().map(|turn| tracing::field::display(turn.action_id)),
+ in_flight_age_secs = in_flight.as_ref().map(|turn| turn.age().num_seconds()),
+ "cursor pipe idle past its deadline but held open by a turn or an admitted command"
+ );
+ }
}
_ = mirror.tick() => sync_service.sync_foreign_runs().await,
}
}
- });
+ }.instrument(background));
let transport = PipeTransport::connect_recoverable(
ours,
move || {
diff --git a/crates/agent_harness/src/outbound/cursor/manager/test.rs b/crates/agent_harness/src/outbound/cursor/manager/test.rs
index 605e7492cc..4b61704d9e 100644
--- a/crates/agent_harness/src/outbound/cursor/manager/test.rs
+++ b/crates/agent_harness/src/outbound/cursor/manager/test.rs
@@ -840,6 +840,48 @@ fn a_pipe_is_not_idle_while_a_command_is_pending() {
assert!(!should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, false, true));
}
+/// The warning for a pipe kept open past its deadline: silent for the first
+/// half hour a turn holds it, then once per half hour after, and re-armed
+/// as soon as the pipe is either free or active again.
+#[test]
+fn a_pipe_held_open_past_its_deadline_warns_once_per_interval() {
+ let minute = std::time::Duration::from_secs(60);
+ let armed = HELD_OPEN_WARNING_INTERVAL;
+
+ // Idle but not yet past the deadline, or past it and free: nothing to say.
+ assert_eq!(held_open_warning(minute, true, armed), (false, armed));
+ assert_eq!(
+ held_open_warning(CURSOR_IDLE_TIMEOUT, false, armed),
+ (false, armed)
+ );
+
+ // Held past the deadline, but for less than the interval: not yet.
+ assert_eq!(
+ held_open_warning(CURSOR_IDLE_TIMEOUT, true, armed),
+ (false, armed)
+ );
+ assert_eq!(
+ held_open_warning(armed - minute, true, armed),
+ (false, armed)
+ );
+
+ // The interval reached: warn, and wait a whole interval for the next.
+ let (warn, next) = held_open_warning(armed, true, armed);
+ assert!(warn);
+ assert_eq!(next, armed * 2);
+ assert_eq!(
+ held_open_warning(armed + minute, true, next),
+ (false, next),
+ "the same stretch does not warn on every tick"
+ );
+ let (warn, next) = held_open_warning(armed * 2, true, next);
+ assert!(warn);
+ assert_eq!(next, armed * 3);
+
+ // Activity resumed: the threshold is re-armed for the next stretch.
+ assert_eq!(held_open_warning(minute, true, next), (false, armed));
+}
+
/// Teardown archives the agent on cursor.com and forgets the mapping; a
/// session that never minted an agent tears down without any API call.
#[tokio::test]
diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rs
index cd66040cd6..8a08d70274 100644
--- a/crates/cursor_cloud_agents/src/domain/service.rs
+++ b/crates/cursor_cloud_agents/src/domain/service.rs
@@ -163,6 +163,18 @@ const STREAM_RECONNECT_ATTEMPTS: usize = 5;
/// agent is drivable from cursor.com) before giving up, in poll intervals.
const BUSY_ATTEMPTS: usize = 450;
+/// How long a prompt waits for the turn gate while a background mirror is
+/// following a run started elsewhere, before giving up.
+///
+/// The same budget as [`BUSY_ATTEMPTS`], because it is the same wait seen
+/// from one step earlier: the mirror follows exactly the run that Cursor
+/// would answer `agent_busy` for. Observed live (prod, 2026-09-27): a mirror
+/// following a run Cursor never ended held the gate for hours, the prompt
+/// behind it never started, and everything queued after it never sent. The
+/// wait has to end somewhere the person can see.
+const GATE_WAIT_BUDGET: std::time::Duration =
+ std::time::Duration::from_secs(BUSY_ATTEMPTS as u64 * POLL_INTERVAL.as_secs());
+
/// How long a turn that saw no new artifacts waits before listing once more.
///
/// Cursor uploads a run's artifacts as the run finishes, so the listing can
@@ -404,6 +416,17 @@ struct SessionState {
/// whenever Cursor happens to end the stream. Replaced per turn, so a
/// cancel can never carry into the next one.
cancel: tokio_util::sync::CancellationToken,
+ /// Fired by cancel; awaited by the background mirror of a run started
+ /// elsewhere ([`CursorSessionService::sync_foreign_runs`]).
+ ///
+ /// Separate from `cancel`, which belongs to the turn: a mirror holds the
+ /// turn gate while a prompt may already be waiting behind it with its own
+ /// token installed, and a stop has to reach both. Replaced per mirror
+ /// sweep, so a stop can never carry into the next one. A stop is the only
+ /// thing that fires it — a prompt arriving behind the mirror waits for
+ /// it instead, because giving up on a run started from cursor.com would
+ /// leave that conversation's run marked abandoned and cancel it.
+ mirror_cancel: tokio_util::sync::CancellationToken,
/// The model this session's next run will use.
///
/// `None` means "whatever this user's own Cursor settings resolve to" —
@@ -757,6 +780,7 @@ where
cursor.run.id = tracing::field::Empty,
agent.turn.stop_reason = tracing::field::Empty,
agent.turn.outcome = tracing::field::Empty,
+ agent.turn.gate_wait_ms = tracing::field::Empty,
),
err,
)]
@@ -791,19 +815,71 @@ where
// this turn's never interleave — but never behind another turn: ACP
// makes a concurrent prompt the client's error, not a queue. The
// holder is told apart by active_run, which only turns set.
- let _turn = match session.turn_gate.try_lock() {
- Ok(guard) => guard,
- Err(_) => {
- if session
- .state
- .lock()
- .expect("session state poisoned")
- .active_run
- .is_some()
- {
- return Err(SessionError::TurnAlreadyActive(session_id.clone()));
+ let gate = session.turn_gate.try_lock().ok();
+ if gate.is_none()
+ && session
+ .state
+ .lock()
+ .expect("session state poisoned")
+ .active_run
+ .is_some()
+ {
+ return Err(SessionError::TurnAlreadyActive(session_id.clone()));
+ }
+ // This pending prompt owns cancellation before any wait it can be
+ // parked in — the gate, the model lookup, historical recovery. A stop
+ // has to land while the prompt is waiting behind a mirror, and the
+ // mirror can hold the gate for as long as the run it follows lives;
+ // installing the token after the gate would have that stop fire an
+ // old token and then be erased by the reset here. Only after the
+ // active-turn check, which must never clobber a live turn's token.
+ let cancel = {
+ let mut state = session.state.lock().expect("session state poisoned");
+ state.cancelled = false;
+ state.cancel = tokio_util::sync::CancellationToken::new();
+ state.cancel.clone()
+ };
+ let _turn = match gate {
+ Some(guard) => guard,
+ None => {
+ // The wait is on the turn span, not only in a log line: it
+ // is the one place a turn's time goes that no Cursor call
+ // accounts for, and a trace of a slow turn has to show it.
+ let waiting_since = std::time::Instant::now();
+ let span = tracing::Span::current();
+ tracing::info!(
+ "waiting for the turn gate behind a mirror of a run started elsewhere"
+ );
+ tokio::select! {
+ biased;
+ () = cancel.cancelled() => {
+ // Nothing to journal: the prompt never reached the
+ // gate, so it never became part of this session's
+ // record. The client's own transcript has it.
+ let waited_ms = waiting_since.elapsed().as_millis() as u64;
+ span.record("agent.turn.gate_wait_ms", waited_ms);
+ tracing::info!(waited_ms, "stopped while waiting for the turn gate");
+ return Ok(StopReason::Cancelled);
+ }
+ guard = session.turn_gate.lock() => {
+ let waited_ms = waiting_since.elapsed().as_millis() as u64;
+ span.record("agent.turn.gate_wait_ms", waited_ms);
+ tracing::info!(waited_ms, "acquired the turn gate");
+ guard
+ }
+ () = tokio::time::sleep(GATE_WAIT_BUDGET) => {
+ span.record("agent.turn.gate_wait_ms", GATE_WAIT_BUDGET.as_millis() as u64);
+ tracing::warn!(
+ waited_secs = GATE_WAIT_BUDGET.as_secs(),
+ "gave up waiting for the turn gate; a mirror is still following a run started elsewhere"
+ );
+ return Err(SessionError::Rejected(PromptRefusal::plain(
+ "Cursor is still busy with a run started outside Macro, and Macro \
+ stopped waiting for it. Send another message once that run has \
+ finished, or stop the session to interrupt it.",
+ )));
+ }
}
- session.turn_gate.lock().await
}
};
@@ -817,14 +893,6 @@ where
"Cursor session must load successfully before prompting"
)));
}
- // This pending prompt owns cancellation before any model lookup or
- // historical recovery can wait. Recovery cannot clear a received stop.
- let cancel = {
- let mut state = session.state.lock().expect("session state poisoned");
- state.cancelled = false;
- state.cancel = tokio_util::sync::CancellationToken::new();
- state.cancel.clone()
- };
// A failed model lookup proves no prompt was executed. Do it before
// reserving the durable intent, so load does not see false ambiguity.
let model = self.effective_model(session_id).await?;
@@ -836,7 +904,7 @@ where
.agent
.clone();
if let Some(agent) = &prior_agent {
- self.backfill_foreign_runs(session_id, &session, agent, None)
+ self.backfill_foreign_runs(session_id, &session, agent, None, &cancel)
.await?;
}
// Preserve original content before the provider creates remote work.
@@ -995,7 +1063,7 @@ where
}
// Acceptance is durable even if recovery of an older run fails. Do
// not observe/project the new run until every older run is reconciled.
- self.backfill_foreign_runs(session_id, &session, &agent, Some(&run))
+ self.backfill_foreign_runs(session_id, &session, &agent, Some(&run), &cancel)
.await?;
// The turn span is the only place all three identities meet, and it
// is what makes a Macro session joinable to the cursor.com run that
@@ -1124,6 +1192,11 @@ where
// the turn. The POST below is the notification that asks for that
// frame.
state.cancel.cancel();
+ // And the background mirror, if one is following a run started
+ // elsewhere: that run is cancelled below like any other, and a
+ // mirror left waiting for a `result` Cursor never sends would
+ // hold the turn gate — and every prompt behind it — for good.
+ state.mirror_cancel.cancel();
(state.agent.clone(), state.active_run.clone())
};
// No agent yet: the session's first prompt is still creating one, so
@@ -1923,12 +1996,22 @@ where
}
/// Catch up foreign runs through the same journal path as local prompts.
+ ///
+ /// `cancel` is the stop that ends a run still going: the turn's own token
+ /// when a prompt is catching up before it runs, the mirror's when the
+ /// host's sweep is. A run already finished is read to its end regardless
+ /// — the stream loop consults the token only where it waits — so a stop
+ /// never costs the recovery of history that is there to be had, only the
+ /// following of a run that has not ended. That run is left as a turn
+ /// leaves one it gave up on: interrupted, to be mirrored in once it does
+ /// end, and never blocking the prompt behind it again.
async fn backfill_foreign_runs(
&self,
session_id: &SessionId,
session: &Session,
agent: &CursorAgentId,
current_run: Option<&CursorRunId>,
+ cancel: &tokio_util::sync::CancellationToken,
) -> Result<bool, SessionError> {
self.ensure_journal(session_id, session).await?;
let last = session
@@ -1978,6 +2061,10 @@ where
.filter(|r| !r.status.is_terminal() && abandoned.contains(&r.id))
.map(|r| r.id.clone())
.collect();
+ let listing_status: std::collections::HashMap<_, _> = listings
+ .iter()
+ .map(|r| (r.id.clone(), r.status.clone()))
+ .collect();
let mut runs = Vec::new();
// A pending run omitted by the provider listing still has to recover.
for run in &pending {
@@ -1996,19 +2083,44 @@ where
let mirrored = !runs.is_empty();
for run in &runs {
if !reconciled.contains(run) {
- // A cancelled prior prompt must not cancel its recovery.
- self.ingest_run(
- session_id,
- session,
- agent,
- run,
- &tokio_util::sync::CancellationToken::new(),
- IngestMode {
- emit: false,
- ..IngestMode::LIVE
- },
- )
- .await?;
+ // Before the ingestion, not after: a run still going is
+ // followed to its end, however long that is, holding the
+ // turn gate the whole way. The first sign of one must not be
+ // a stream warning an hour in - it is this line, with the
+ // run's status as Cursor lists it right now.
+ let status = listing_status.get(run);
+ let still_running = status.is_some_and(|status| !status.is_terminal());
+ let started = std::time::Instant::now();
+ tracing::info!(
+ %agent,
+ %run,
+ cursor.run.status = ?status,
+ cursor.run.still_running = still_running,
+ cursor.run.listed = status.is_some(),
+ "recovering a run started elsewhere"
+ );
+ let outcome = self
+ .ingest_run(
+ session_id,
+ session,
+ agent,
+ run,
+ cancel,
+ IngestMode {
+ emit: false,
+ ..IngestMode::LIVE
+ },
+ )
+ .await;
+ tracing::info!(
+ %agent,
+ %run,
+ cursor.run.still_running = still_running,
+ elapsed_ms = started.elapsed().as_millis() as u64,
+ outcome = ?outcome.as_ref().map_err(ToString::to_string),
+ "finished recovering a run started elsewhere"
+ );
+ outcome?;
}
let complete = session
.state
@@ -2018,6 +2130,13 @@ where
.iter()
.any(|e| e.run.as_ref() == Some(run) && e.input == JournalInput::Reconciled);
if !complete {
+ // A stop ended the following, not a failure: what was
+ // captured is still shown, and the prompt behind this (if
+ // any) answers Cancelled rather than failing.
+ if cancel.is_cancelled() {
+ tracing::info!(%agent, %run, "stopped following a run started elsewhere");
+ break;
+ }
return Err(rootcause::report!("Cursor run {run} remains unreconciled").into());
}
}
@@ -2618,8 +2737,15 @@ where
let Some(agent) = agent else {
continue;
};
+ // Fresh per sweep: a stop that landed between sweeps has nothing
+ // left to end, and must not end the next one.
+ let cancel = {
+ let mut state = session.state.lock().expect("session state poisoned");
+ state.mirror_cancel = tokio_util::sync::CancellationToken::new();
+ state.mirror_cancel.clone()
+ };
if let Err(error) = self
- .backfill_foreign_runs(&session_id, &session, &agent, None)
+ .backfill_foreign_runs(&session_id, &session, &agent, None, &cancel)
.await
{
tracing::warn!(%session_id, %agent, %error, "could not mirror cursor.com runs");
diff --git a/crates/cursor_cloud_agents/src/domain/service/test.rs b/crates/cursor_cloud_agents/src/domain/service/test.rs
index 5d9d8df0ea..04cd61b77b 100644
--- a/crates/cursor_cloud_agents/src/domain/service/test.rs
+++ b/crates/cursor_cloud_agents/src/domain/service/test.rs
@@ -3732,3 +3732,222 @@ async fn a_resumed_stream_continues_the_captured_prefix_without_duplicating_it()
"the matched prefix is not captured a second time"
);
}
+
+/// Start a mirror of a run started elsewhere and hold it open.
+///
+/// The session has an agent whose only run, `R1`, Cursor still calls
+/// running; the mirror follows its stream and holds the turn gate for as
+/// long as the stream stays open. Returns the mirror's task and the stream
+/// the test feeds it.
+async fn mirror_following_a_live_run(
+ service: &Arc<Service>,
+ cursor: &FakeCursor,
+ id: &SessionId,
+) -> (tokio::task::JoinHandle<()>, crate::testing::ScriptSender) {
+ service.session(id).unwrap().state.lock().unwrap().agent = Some(CursorAgentId::new("agent"));
+ cursor.script_run_listings(vec![RunListing {
+ id: CursorRunId::new("R1"),
+ status: RunStatus::Running,
+ }]);
+ let foreign = cursor.script_stream();
+ foreign
+ .send(CursorEvent::Interaction(InteractionUpdate::UserMessage {
+ text: "asked over there".into(),
+ }))
+ .unwrap();
+ let mirror = tokio::spawn({
+ let service = Arc::clone(service);
+ async move { service.sync_foreign_runs().await }
+ });
+ tokio::time::timeout(std::time::Duration::from_secs(5), async {
+ loop {
+ if service
+ .journal
+ .read(id)
+ .await
+ .unwrap()
+ .iter()
+ .any(|e| matches!(e.input, JournalInput::Sse(_)))
+ {
+ break;
+ }
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("the mirror captures the foreign run");
+ assert!(service.has_active_turn(), "the mirror holds the turn gate");
+ (mirror, foreign)
+}
+
+/// The production hang (2026-09-27): a mirror following a run started from
+/// cursor.com held the turn gate, the prompt behind it waited with no way to
+/// be stopped, and everything queued after that prompt never sent.
+///
+/// A stop now ends the waiting prompt at once, as `Cancelled`, and asks
+/// Cursor to cancel the run the mirror was following - the same remote cancel
+/// any stop sends for a run this process did not start.
+#[tokio::test]
+async fn a_stop_ends_a_prompt_waiting_behind_a_mirror() {
+ let (service, cursor, _notifier) = service(None);
+ let id = service.new_session(Path::new(""), vec![]);
+ let (mirror, foreign) = mirror_following_a_live_run(&service, &cursor, &id).await;
+
+ let prompt = tokio::spawn({
+ let service = Arc::clone(&service);
+ let id = id.clone();
+ async move { service.prompt(&id, "queued behind the mirror").await }
+ });
+ // The prompt reaches the gate and parks there: it has no awaits before it.
+ for _ in 0..16 {
+ tokio::task::yield_now().await;
+ }
+ assert!(!prompt.is_finished(), "the prompt waits behind the mirror");
+
+ service.cancel(&id).await.expect("cancel works");
+ let stop = tokio::time::timeout(std::time::Duration::from_secs(5), prompt)
+ .await
+ .expect("the stop ends the wait")
+ .expect("task joins")
+ .expect("prompt resolves");
+ assert_eq!(stop, StopReason::Cancelled);
+ assert!(
+ cursor
+ .calls()
+ .iter()
+ .any(|call| matches!(call, CursorCall::CancelRun(_, run) if run.as_str() == "R1")),
+ "the run the mirror was following is asked to stop"
+ );
+ assert!(
+ !cursor
+ .calls()
+ .iter()
+ .any(|call| matches!(call, CursorCall::CreateRun(..))),
+ "a stopped prompt never reaches Cursor"
+ );
+
+ // Cursor answers the cancel with the run's end; the mirror reconciles it
+ // and lets go of the gate.
+ foreign.send(cancelled("R1")).unwrap();
+ foreign.send(CursorEvent::Done).unwrap();
+ tokio::time::timeout(std::time::Duration::from_secs(5), mirror)
+ .await
+ .expect("the mirror ends with the run")
+ .unwrap();
+ assert!(!service.has_active_turn());
+}
+
+/// A stop reaches the mirror itself, not only the prompt behind it.
+///
+/// Cursor was observed to keep a run at `RUNNING` after a cancel, with a
+/// stream that says nothing more. A mirror that waited for that run's
+/// `result` frame would hold the turn gate for good. Instead the stop ends
+/// the following at the mirror's next check, the run is left as one a turn
+/// gave up on, and the next prompt runs without waiting behind it.
+#[tokio::test(start_paused = true)]
+async fn a_stop_ends_a_mirror_following_a_run_cursor_never_ends() {
+ let (service, cursor, notifier) = service(None);
+ let id = service.new_session(Path::new(""), vec![]);
+ // The quiet-stream checks and the fallback poll each read the record,
+ // and every read says the run is still going.
+ for _ in 0..8 {
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Running,
+ text: None,
+ });
+ }
+ let (mirror, _foreign) = mirror_following_a_live_run(&service, &cursor, &id).await;
+
+ service.cancel(&id).await.expect("cancel works");
+
+ // The stream stays open and silent; the stop is what ends the mirror.
+ tokio::time::timeout(std::time::Duration::from_secs(600), mirror)
+ .await
+ .expect("the stop ends the mirror")
+ .unwrap();
+ assert!(!service.has_active_turn(), "the mirror let go of the gate");
+ let entries = service.journal.read(&id).await.unwrap();
+ assert!(
+ entries
+ .iter()
+ .any(|e| e.run.as_ref().map(CursorRunId::as_str) == Some("R1")
+ && matches!(e.input, JournalInput::Interrupted(_))),
+ "the run is recorded as one this session gave up on"
+ );
+ assert!(
+ !entries
+ .iter()
+ .any(|e| e.run.as_ref().map(CursorRunId::as_str) == Some("R1")
+ && e.input == JournalInput::Reconciled),
+ "an interrupted following is not reconciliation"
+ );
+ assert_eq!(
+ notifier.reloads(),
+ vec![id.clone()],
+ "what the mirror captured before the stop is still shown"
+ );
+
+ // The next prompt does not wait behind the run Cursor never ended: it is
+ // skipped as abandoned, and the prompt runs.
+ let events = cursor.script_stream();
+ events.send(finished("run-fake-1")).unwrap();
+ events.send(CursorEvent::Done).unwrap();
+ let stop = tokio::time::timeout(
+ std::time::Duration::from_secs(60),
+ service.prompt(&id, "next"),
+ )
+ .await
+ .expect("the prompt is not blocked by the abandoned run")
+ .expect("prompt resolves");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert!(
+ cursor
+ .calls()
+ .iter()
+ .any(|call| matches!(call, CursorCall::CreateRun(..))),
+ "the prompt reached Cursor"
+ );
+}
+
+/// A prompt behind a mirror does not wait forever, even with nobody to stop
+/// it: a mirror still following a run after the busy-agent budget ends the
+/// prompt with a refusal the person can read, instead of a turn that never
+/// ends and a queue that never drains.
+#[tokio::test(start_paused = true)]
+async fn a_prompt_gives_up_on_the_turn_gate_after_the_busy_budget() {
+ let (service, cursor, _notifier) = service(None);
+ let id = service.new_session(Path::new(""), vec![]);
+ // Enough record reads for every quiet check across the budget.
+ for _ in 0..2000 {
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Running,
+ text: None,
+ });
+ }
+ let (mirror, foreign) = mirror_following_a_live_run(&service, &cursor, &id).await;
+
+ let outcome = tokio::time::timeout(
+ GATE_WAIT_BUDGET + std::time::Duration::from_secs(60),
+ service.prompt(&id, "queued behind the mirror"),
+ )
+ .await
+ .expect("the wait is bounded");
+ assert!(
+ matches!(&outcome, Err(SessionError::Rejected(refusal)) if refusal.message.contains("started outside Macro")),
+ "the prompt is refused in the person's terms, got {outcome:?}"
+ );
+ assert!(
+ !cursor
+ .calls()
+ .iter()
+ .any(|call| matches!(call, CursorCall::CreateRun(..))),
+ "a refused prompt never reaches Cursor"
+ );
+ assert!(
+ service.has_active_turn(),
+ "giving up on the wait does not end the mirror"
+ );
+ drop(foreign);
+ mirror.abort();
+ let _ = mirror.await;
+}
diff --git a/docs/AGENT_GUIDE/observability.md b/docs/AGENT_GUIDE/observability.md
index 62e47713c3..f14f721d91 100644
--- a/docs/AGENT_GUIDE/observability.md
+++ b/docs/AGENT_GUIDE/observability.md
@@ -77,7 +77,8 @@ identifier spaces and must not be confused.
| Span | Answers |
| --- | --- |
-| `agent.turn` | Did a Cursor turn run, and how did it end? `agent.turn.stop_reason` / `agent.turn.outcome`, plus `cursor.agent.id` / `cursor.run.id`. |
+| `agent.turn` | Did a Cursor turn run, and how did it end? `agent.turn.stop_reason` / `agent.turn.outcome`, plus `cursor.agent.id` / `cursor.run.id`. `agent.turn.gate_wait_ms` is time spent waiting behind a mirror of a run started from cursor.com before the turn could begin. |
+| `agent.session.background` | The per-session task that mirrors cursor.com and reaps idle pipes. Every `cursor.run.ingest` a mirror runs sits under it, which is what makes a mirrored run searchable by Macro session id. |
| `cursor.run.poll` | Is a turn still alive? One per poll, at DEBUG. |
| `agent.session.turn_ended` | The connection's live fold closed the turn on a logged frame; carries `agent.turn.id`, `agent.turn.stop_reason`, and `agent.action.id` when a local prompt opened it. |
| `agent.session.disconnect` | The session's actor wrote a `disconnected` event, and `agent.session.close_reason` says why. |
@@ -95,4 +96,18 @@ Two things worth knowing when reading these:
return.
- **The idle-check DEBUG line fires every tick, not just the reaping one.** `agent.pipe.reaped`,
`agent.pipe.active_turn` and `agent.pipe.idle_ms` on the ticks that did *nothing* are what
- show a deadline sitting long expired while a live turn held the pipe open.
+ show a deadline sitting long expired while a live turn held the pipe open. Production does
+ not ship DEBUG, so the same condition is also said at WARN — `cursor pipe idle past its
+ deadline but held open by a turn or an admitted command`, with the in-flight turn's id and
+ age — once the pipe has been held open for half an hour, and once per half hour after.
+
+Log lines to search for when a session's queue is not draining, in the order a stuck
+prompt passes them:
+
+| Line | Where | Means |
+| --- | --- | --- |
+| `forwarding an agent session command` | the replica that took the request | The command left for the managing replica; this replica reports `Sent` regardless of what happens next. |
+| `executed a forwarded command` / `forwarded command failed` | the managing replica | The command arrived, and what it returned (`Completed` or `Queued`). Absent means the Redis hop lost it. |
+| `action queued behind the turn in flight` | the managing replica | Why the prompt is waiting: the in-flight turn, its action id, and `in_flight_age_secs`. |
+| `recovering a run started elsewhere` | the Cursor service | A mirror or a pre-prompt backfill is about to follow a cursor.com run; `cursor.run.still_running=true` means it will hold the turn gate until that run ends. `finished recovering …` closes it with `elapsed_ms`. |
+| `waiting for the turn gate behind a mirror …` / `acquired the turn gate` / `stopped while waiting …` / `gave up waiting for the turn gate` | the Cursor service, inside `agent.turn` | The prompt is parked behind that mirror, and how the wait ended. |
diff --git a/services/agent_harness_service/src/runtime_commands.rs b/services/agent_harness_service/src/runtime_commands.rs
index 620eea89df..b7a8d94acd 100644
--- a/services/agent_harness_service/src/runtime_commands.rs
+++ b/services/agent_harness_service/src/runtime_commands.rs
@@ -116,13 +116,23 @@ where
RuntimeCommandTarget::Harness(_) => false,
};
if selected {
- harness_service
+ // The forwarding replica answered its caller the moment it
+ // published, so this is the only record that the command
+ // arrived anywhere and what became of it. Logged on
+ // success too: a session whose queue never drains is
+ // debugged by asking whether its commands reached the
+ // replica that manages it.
+ match harness_service
.execute_forwarded(session, request.into_command())
.await
- .inspect_err(|error| {
+ {
+ Ok(outcome) => {
+ tracing::info!(%session, ?outcome, "executed a forwarded command");
+ }
+ Err(error) => {
tracing::error!(error = ?error, %session, "forwarded command failed");
- })
- .ok();
+ }
+ }
}
}
.instrument(span),