Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
cursor journal: fail fast on a superseded claim instead of blocking (#6327)
Wolf Mermelstein0071fb5d87d4
diff --git a/.sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json b/.sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.jsonsimilarity index 73%rename from .sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.jsonrename to .sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.jsonindex 870cdca190..2992fe05bf 100644--- a/.sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json+++ b/.sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE",+ "query": "SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE NOWAIT","describe": {"columns": [{@@ -19,5 +19,5 @@false]},- "hash": "9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab"+ "hash": "48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd"}diff --git a/crates/agent_harness/src/domain/mod.rs b/crates/agent_harness/src/domain/mod.rsindex fbc1b6464c..9f33d22ee2 100644--- a/crates/agent_harness/src/domain/mod.rs+++ b/crates/agent_harness/src/domain/mod.rs@@ -2,6 +2,9 @@ pub mod error;pub mod model;/// Fresh model discovery without creating an agent session.pub mod model_load;+/// A shared record of sessions with a command admitted but not yet resolved,+/// consulted by container managers' idle reapers.+pub mod pending;pub mod ports;/// The per-session queue of turn-occupying actions awaiting their turn.pub mod queue;diff --git a/crates/agent_harness/src/domain/pending.rs b/crates/agent_harness/src/domain/pending.rsnew file mode 100644index 0000000000..13bfe08e52--- /dev/null+++ b/crates/agent_harness/src/domain/pending.rs@@ -0,0 +1,145 @@+//! A shared record of sessions with a command admitted but not yet acted on,+//! consulted by container managers' idle reapers so they never pull a+//! session's transport out from under a command already on its way in.+//!+//! Built once at process wiring and handed to both [`crate::domain::service`]+//! (which marks and clears it) and every [`crate::domain::ports::ContainerManager`]+//! whose provider reaps idle transports (which only reads it). Neither side+//! needs the other's type, which is what lets the container managers be+//! constructed before the harness that owns them.+//!+//! A session is marked from the moment its command is admitted, which is+//! before any turn exists to describe - hence the two-stage value: admission+//! records only that something is coming, and dispatch fills in the turn it+//! opened. A reaper only ever asks the first question; the harness's own+//! lifecycle events need the second.++use std::sync::Arc;++use agent_session::domain::model::AgentSessionId;+use dashmap::DashMap;++use crate::domain::queue::InFlightTurn;++/// Cheap to clone; every clone shares the same underlying map.+#[derive(Clone, Default)]+pub struct PendingCommands(Arc<DashMap<AgentSessionId, Option<InFlightTurn>>>);++impl PendingCommands {+ /// An empty tracker: no session marked pending.+ #[must_use]+ pub fn new() -> Self {+ Self::default()+ }++ /// Record that `session` has a command admitted, before it has been+ /// dispatched and so before there is a turn to name.+ ///+ /// Only called for a session not already pending, so it can never+ /// overwrite a dispatched turn with an empty admission.+ pub fn admit(&self, session: AgentSessionId) {+ self.0.insert(session, None);+ }++ /// Record the turn a dispatched action opened for `session`.+ pub fn mark_turn(&self, session: AgentSessionId, turn: InFlightTurn) {+ self.0.insert(session, Some(turn));+ }++ /// Clear `session`'s mark, if any.+ pub fn clear(&self, session: AgentSessionId) {+ self.0.remove(&session);+ }++ /// Clear `session`'s mark and hand back the turn it had in flight, if it+ /// had got as far as dispatching one.+ pub fn take(&self, session: AgentSessionId) -> Option<InFlightTurn> {+ self.0.remove(&session).and_then(|(_, turn)| turn)+ }++ /// The turn `session` currently has in flight, if it has one.+ ///+ /// Absent both for a session with nothing pending and for one whose+ /// command is admitted but not yet dispatched - callers reporting on a+ /// turn have nothing to say in either case.+ #[must_use]+ pub fn turn(&self, session: AgentSessionId) -> Option<InFlightTurn> {+ self.0.get(&session).and_then(|entry| entry.clone())+ }++ /// Whether `session` currently has a command admitted and unresolved.+ ///+ /// A reaper's cue to leave the session's transport alone this tick even+ /// though it otherwise looks idle: something is already on its way to+ /// it, and closing the transport now would only race that delivery.+ #[must_use]+ pub fn is_pending(&self, session: AgentSessionId) -> bool {+ self.0.contains_key(&session)+ }+}++#[cfg(test)]+mod test {+ use agent_fold::domain::model::TurnId;+ use agent_runtime_protocol::domain::action::AgentActionId;++ use super::*;++ fn session_id(n: u128) -> AgentSessionId {+ AgentSessionId::new_from_uuid(macro_uuid::Uuid::from_u128(n))+ }++ fn turn() -> InFlightTurn {+ InFlightTurn {+ action_id: AgentActionId::mint(),+ turn: TurnId(7),+ actor: None,+ announcement_message_id: None,+ }+ }++ #[test]+ fn admits_and_clears() {+ let pending = PendingCommands::new();+ let id = session_id(1);+ assert!(!pending.is_pending(id));+ pending.admit(id);+ assert!(pending.is_pending(id));+ pending.clear(id);+ assert!(!pending.is_pending(id));+ }++ #[test]+ fn an_admitted_session_has_no_turn_until_dispatch() {+ let pending = PendingCommands::new();+ let id = session_id(2);+ pending.admit(id);+ assert!(+ pending.turn(id).is_none(),+ "admission alone names no turn, but is still pending"+ );+ let dispatched = turn();+ pending.mark_turn(id, dispatched.clone());+ assert_eq!(pending.turn(id).map(|t| t.turn), Some(dispatched.turn));+ }++ #[test]+ fn take_clears_and_returns_the_turn() {+ let pending = PendingCommands::new();+ let id = session_id(3);+ let dispatched = turn();+ pending.mark_turn(id, dispatched.clone());+ assert_eq!(pending.take(id).map(|t| t.turn), Some(dispatched.turn));+ assert!(!pending.is_pending(id));+ assert!(pending.take(id).is_none(), "a second take has nothing left");+ }++ #[test]+ fn clones_share_state() {+ let pending = PendingCommands::new();+ let clone = pending.clone();+ let id = session_id(4);+ pending.admit(id);+ assert!(clone.is_pending(id), "clones observe the same map");+ }+}diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rsindex fe1d52f8e2..a6a3ec4922 100644--- a/crates/agent_harness/src/domain/service.rs+++ b/crates/agent_harness/src/domain/service.rs@@ -49,6 +49,7 @@ use crate::domain::model::{AgentKind, AnnounceOrigin, AnnouncePrompt, CommandOutcome, DeliverAction, HarnessCommand,HarnessDefaults, OpenSession, SessionAnnouncement, SpawnContainer, is_macro_staff,};+use crate::domain::pending::PendingCommands;use crate::domain::ports::{AgentPromptComposer, ChannelPromptContext, CommandForwarder, ContainerManager,RuntimeConnections, SandboxEgressProvisioner, SessionAnnouncer,@@ -79,11 +80,19 @@ struct AgentHarnessInner</// Turn-occupying actions waiting for their session's running turn to/// end. In-memory beside the live actors this replica manages.queues: SessionQueues,- /// The sessions with a turn in flight, and which turn. Marked when a- /// turn-occupying action reaches the runtime, cleared by- /// `TurnEnded`/`SessionStopped`. Only ever touched from the session's- /// own command worker, which is what serializes it against dispatch.- busy: DashMap<AgentSessionId, InFlightTurn>,+ /// The sessions with a command admitted and not yet resolved, and which+ /// turn it opened once dispatch names one. Marked the moment a+ /// turn-occupying action is admitted (queue.rs's `enqueue_then_dispatch`),+ /// cleared by `TurnEnded`/`SessionStopped` or on admission failure. Only+ /// ever touched from the session's own command worker, which is what+ /// serializes it against dispatch.+ ///+ /// Shared (not private to this service) so a provider's idle reaper can+ /// read it before closing a session's transport: marking on admission+ /// rather than on delivery success is what closes the gap between "a+ /// command was handed to this session" and "the runtime has visibly+ /// started a turn", which a reaper watching only the latter cannot see.+ busy: PendingCommands,/// Where lifecycle facts go. Erased so it is not an eighth type parameter.lifecycle_publisher: Arc<dyn AgentSessionLifecyclePublisher>,}@@ -170,6 +179,7 @@ whereforwarder: impl CommandForwarder,defaults: impl Into<HarnessDefaults>,lifecycle_publisher: impl AgentSessionLifecyclePublisher,+ pending: PendingCommands,) -> Self {Self {inner: Arc::new(AgentHarnessInner {@@ -183,7 +193,7 @@ whereforwarder: Box::new(forwarder),defaults: defaults.into(),queues: SessionQueues::new(),- busy: DashMap::new(),+ busy: pending,lifecycle_publisher: Arc::new(lifecycle_publisher),}),workers: Arc::new(DashMap::new()),diff --git a/crates/agent_harness/src/domain/service/queue.rs b/crates/agent_harness/src/domain/service/queue.rsindex 8dc9e3e8eb..6e9f863266 100644--- a/crates/agent_harness/src/domain/service/queue.rs+++ b/crates/agent_harness/src/domain/service/queue.rs@@ -313,7 +313,7 @@ whereaction_id: fold_action_id,..}) => {- let ended = self.busy.remove(&session_id).map(|(_, turn)| turn);+ let ended = self.busy.take(session_id);// A turn end with no record: this replica restarted mid-turn// and the in-memory mark went with it, or the fold closed a// turn nobody here prompted. The queue still drains; only the@@ -372,7 +372,7 @@ whereOk(CommandOutcome::Completed)}HarnessCommand::SessionStopped { reason } => {- let in_flight = self.busy.remove(&session_id).map(|(_, turn)| turn);+ let in_flight = self.busy.take(session_id);self.publish_lifecycle(session_id, |identity| {AgentSessionLifecycleEvent::Stopped(SessionStoppedMetadata {identity,@@ -384,7 +384,7 @@ whereOk(CommandOutcome::Completed)}HarnessCommand::Turn(TurnSignal::ElicitationRaised { question, .. }) => {- let Some(turn) = self.busy.get(&session_id).map(|turn| turn.clone()) else {+ let Some(turn) = self.busy.turn(session_id) else {tracing::warn!(%session_id, "elicitation raised with no in-flight record");return Ok(CommandOutcome::Completed);};@@ -401,7 +401,7 @@ whereOk(CommandOutcome::Completed)}HarnessCommand::Turn(TurnSignal::ElicitationCleared { .. }) => {- let Some(turn) = self.busy.get(&session_id).map(|turn| turn.clone()) else {+ let Some(turn) = self.busy.turn(session_id) else {tracing::warn!(%session_id, "elicitation cleared with no in-flight record");return Ok(CommandOutcome::Completed);};@@ -427,7 +427,7 @@ where// session's entries will never dispatch, and leaving them// would leak them for the life of the process. The published// empty snapshot is the viewers' goodbye.- self.busy.remove(&session_id);+ self.busy.clear(session_id);self.queues.drop_session(session_id);self.publish_queue(session_id).await;match identity {@@ -477,10 +477,24 @@ wheresession_id,)?;- let dispatched = if self.busy.contains_key(&session_id) {+ let dispatched = if self.busy.is_pending(session_id) {Ok(())} else {- self.dispatch_next(session_id).await.map(drop)+ // Marked before dispatching, not only once `dispatch_next`'s own+ // delivery succeeds: this closes the window between "a command+ // was just handed off for this session" and "the runtime+ // visibly started a turn", which a reaper watching only the+ // latter cannot see. The turn itself is unknowable this early -+ // `dispatch_next` fills it in with `mark_turn` once delivery+ // names one - so this records only that something is coming. On+ // failure nothing actually started, so the mark comes back off+ // rather than sticking to a session with nothing running.+ self.busy.admit(session_id);+ let result = self.dispatch_next(session_id).await.map(drop);+ if result.is_err() {+ self.busy.clear(session_id);+ }+ result};self.publish_queue(session_id).await;dispatched?;@@ -599,7 +613,7 @@ whereactor: entry.actor,announcement_message_id: entry.announced,};- self.busy.insert(session_id, turn.clone());+ self.busy.mark_turn(session_id, turn.clone());self.publish_lifecycle(session_id, |identity| {AgentSessionLifecycleEvent::TurnStarted(TurnStartedMetadata {identity,diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rsindex f761eb0eb1..dbb2438229 100644--- a/crates/agent_harness/src/domain/service/test.rs+++ b/crates/agent_harness/src/domain/service/test.rs@@ -345,6 +345,7 @@ fn harness_with_signals(repo_url: "https://github.com/macro-inc/macro".to_owned(),},lifecycle.clone(),+ crate::domain::pending::PendingCommands::new(),);let (ended, ended_rx) = mpsc::unbounded_channel();turn_observer.bind(SignallingTurnObserver {@@ -1863,6 +1864,7 @@ async fn a_managed_session_opens_as_the_managed_default_bot() {).with_managed_bot(inmem_bot),NoopLifecyclePublisher,+ crate::domain::pending::PendingCommands::new(),);let session = service@@ -2308,6 +2310,7 @@ async fn commands_for_a_peer_managed_session_forward_through_redis() {repo_url: "https://github.com/macro-inc/macro".to_owned(),},NoopLifecyclePublisher,+ crate::domain::pending::PendingCommands::new(),);service@@ -2356,6 +2359,7 @@ async fn unmanaged_external_session_forwards_to_its_remote_harness() {repo_url: "https://github.com/macro-inc/macro".to_owned(),},NoopLifecyclePublisher,+ crate::domain::pending::PendingCommands::new(),);let session = service.open_external_session(open_external_request("/srv/agent"))diff --git a/crates/agent_harness/src/outbound/cursor/manager.rs b/crates/agent_harness/src/outbound/cursor/manager.rsindex a148054e17..2a54561ff8 100644--- a/crates/agent_harness/src/outbound/cursor/manager.rs+++ b/crates/agent_harness/src/outbound/cursor/manager.rs@@ -39,6 +39,7 @@ use super::keys::CursorApiKeys;use super::pipe::PipeTransport;use crate::domain::error::{HarnessError, Result};use crate::domain::model::SpawnContainer;+use crate::domain::pending::PendingCommands;use crate::domain::ports::ContainerManager;use crate::domain::sandbox::SandboxResizeEffect;use agent_session::domain::model::SandboxSize;@@ -85,8 +86,8 @@ const CURSOR_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(/// for reclaiming two idle tasks is nothing.const CURSOR_IDLE_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);-fn should_reap_cursor_pipe(idle: std::time::Duration, active_turn: bool) -> bool {- idle >= CURSOR_IDLE_TIMEOUT && !active_turn+fn should_reap_cursor_pipe(idle: std::time::Duration, active_turn: bool, pending: bool) -> bool {+ idle >= CURSOR_IDLE_TIMEOUT && !active_turn && !pending}/// The ref new agents start their work from.@@ -127,6 +128,10 @@ pub struct CursorContainerManager<Sessions, Keys> {repo: CursorRepoUrl,sessions: Sessions,journal_storage: JournalStorage,+ /// Sessions the harness has a command in flight for right now, shared+ /// with `AgentHarnessService` so the idle reaper below never closes a+ /// pipe a command is already on its way to.+ pending: PendingCommands,}/// Hosted sessions always use durable storage; tests select memory explicitly.@@ -169,6 +174,7 @@ wheresessions: Sessions,pool: sqlx::PgPool,replica: ReplicaId,+ pending: PendingCommands,) -> Self {Self {keys,@@ -176,6 +182,7 @@ whererepo,sessions,journal_storage: JournalStorage::Postgres { pool, replica },+ pending,}}@@ -192,6 +199,7 @@ whererepo,sessions,journal_storage: JournalStorage::Memory,+ pending: PendingCommands::new(),}}@@ -295,6 +303,12 @@ where);}let pipe_closed = tokio_util::sync::CancellationToken::new();+ // Cloned before the tasks below move `pipe_closed` itself: the+ // attachment built at the end of this function hands this same+ // signal to the session actor's command wait, so a command sent+ // right as - or just after - this pipe dies fails at once instead+ // of riding out its own separate timeout for nothing.+ let attachment_closed = pipe_closed.clone();let shutdown = tokio_util::sync::CancellationToken::new();let sync_service = Arc::clone(&service);let on_pipe_close = pipe_closed.clone();@@ -313,6 +327,7 @@ wherelet last_activity = Arc::new(std::sync::Mutex::new(tokio::time::Instant::now()));let observed = Arc::clone(&last_activity);let reaper_shutdown = shutdown.clone();+ let pending = self.pending.clone();tokio::spawn(async move {let mut mirror = interval_from_now(FOREIGN_SYNC_INTERVAL);let mut reaper = interval_from_now(CURSOR_IDLE_CHECK_INTERVAL);@@ -324,6 +339,12 @@ where.lock().unwrap_or_else(std::sync::PoisonError::into_inner);let active_turn = sync_service.has_active_turn();+ // Checked alongside `active_turn`: a command already+ // admitted for this session but not yet turn-active+ // (the harness marks this at admission, before+ // dispatch - see `queue::enqueue_then_dispatch`) is+ // exactly the case `active_turn` alone cannot see.+ let pending_command = pending.is_pending(session_id);let activity = last_activity.lock().unwrap_or_else(std::sync::PoisonError::into_inner);@@ -333,7 +354,11 @@ wherelet raced = *activity != observed_at;let idle_ms = activity.elapsed().as_millis();let reaped = !raced- && should_reap_cursor_pipe(activity.elapsed(), active_turn);+ && should_reap_cursor_pipe(+ activity.elapsed(),+ active_turn,+ pending_command,+ );// Every tick, not just the reaping one. The inputs to// this decision are what tell a pipe that died of// idleness apart from one pulled out from under a@@ -345,6 +370,7 @@ where%session_id,agent.pipe.idle_ms = idle_ms as u64,agent.pipe.active_turn = active_turn,+ agent.pipe.pending_command = pending_command,agent.pipe.activity_raced = raced,agent.pipe.reaped = reaped,"cursor pipe idle check"@@ -377,7 +403,8 @@ whereshutdown,Some(reload_rx),);- let mut attachment = agent_session::domain::connection::RuntimeAttachment::solo(transport);+ let mut attachment = agent_session::domain::connection::RuntimeAttachment::solo(transport)+ .with_closed(attachment_closed);if let Some(binding) = owner_binding {attachment = attachment.on_activate(binding);}diff --git a/crates/agent_harness/src/outbound/cursor/manager/test.rs b/crates/agent_harness/src/outbound/cursor/manager/test.rsindex 2a19c6cb21..6c43e67f9b 100644--- a/crates/agent_harness/src/outbound/cursor/manager/test.rs+++ b/crates/agent_harness/src/outbound/cursor/manager/test.rs@@ -733,8 +733,16 @@ async fn an_idle_pipe_is_shut_down() {#[test]fn a_pipe_is_not_idle_while_cursor_is_running_a_turn() {- assert!(!should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, true));- assert!(should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, false));+ assert!(!should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, true, false));+ assert!(should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, false, false));+}++#[test]+fn a_pipe_is_not_idle_while_a_command_is_pending() {+ // Same idle duration, no active turn yet - the case a command admitted+ // just before the reap tick looks like, before the runtime has had a+ // chance to mark a turn active.+ assert!(!should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, false, true));}/// Teardown archives the agent on cursor.com and forgets the mapping; adiff --git a/crates/agent_harness/src/outbound/daytona/manager.rs b/crates/agent_harness/src/outbound/daytona/manager.rsindex abb39819d9..2062c8070a 100644--- a/crates/agent_harness/src/outbound/daytona/manager.rs+++ b/crates/agent_harness/src/outbound/daytona/manager.rs@@ -15,6 +15,7 @@ use super::errors::DaytonaError;use super::types::{AnthropicApiKey, DaytonaSettings, Env, Labels, PortPreview, Snapshot};use crate::domain::error::{HarnessError, Result};use crate::domain::model::SpawnContainer;+use crate::domain::pending::PendingCommands;use crate::domain::ports::ContainerManager;use crate::domain::sandbox::{SandboxResizeEffect, SandboxResources, resize_effect_from_resources, resources,@@ -49,6 +50,10 @@ struct DaytonaContainerManagerState {shutdown_complete: CancellationToken,lifecycle: Mutex<ManagerLifecycle>,tasks: TaskTracker,+ /// Sessions the harness has a command in flight for right now; consulted+ /// by the idle reaper so it never stops a sandbox a command is already+ /// on its way to.+ pending: PendingCommands,}#[derive(Default)]@@ -57,17 +62,18 @@ struct ManagerLifecycle {}impl DaytonaContainerManagerState {- fn new() -> Self {+ fn new(pending: PendingCommands) -> Self {Self {containers: ManagedContainers::new(),shutdown: CancellationToken::new(),shutdown_complete: CancellationToken::new(),lifecycle: Mutex::new(ManagerLifecycle::default()),tasks: TaskTracker::new(),+ pending,}}- fn register(&self, id: DaytonaSandboxId) -> bool {+ fn register(&self, id: DaytonaSandboxId, session: AgentSessionId) -> bool {let lifecycle = self.lifecycle.lock()@@ -75,7 +81,7 @@ impl DaytonaContainerManagerState {if lifecycle.shutting_down {return false;}- self.containers.register(id);+ self.containers.register(id, session);true}}@@ -92,7 +98,7 @@ pub struct DaytonaContainerManager {impl DaytonaContainerManager {/// Build the manager from its settings.#[must_use]- pub fn new(settings: DaytonaSettings) -> Self {+ pub fn new(settings: DaytonaSettings, pending: PendingCommands) -> Self {let DaytonaSettings {api_url,api_key,@@ -100,7 +106,7 @@ impl DaytonaContainerManager {anthropic_api_key,} = settings;let client = DaytonaClient::new(api_url, api_key);- let managed = Arc::new(DaytonaContainerManagerState::new());+ let managed = Arc::new(DaytonaContainerManagerState::new(pending));managed.tasks.spawn(reap_idle_containers(client.clone(), managed.clone()));@@ -386,7 +392,7 @@ impl ContainerManager for DaytonaContainerManager {.map_err(unavailable)?,);tracing::info!(sandbox_id = %id.as_str(), session = %session_id, "sandbox created");- if !self.managed.register(id.clone()) {+ if !self.managed.register(id.clone(), session_id) {if !stop_sandbox(&self.client, &id, "shutdown during sandbox creation").await {self.managed.containers@@ -473,7 +479,7 @@ impl ContainerManager for DaytonaContainerManager {HarnessError::Container(format!("session {session} has no sandbox to resume"))})?,);- if !self.managed.register(id.clone()) {+ if !self.managed.register(id.clone(), session) {return Err(HarnessError::Container("the container manager is shutting down".to_owned(),));@@ -557,7 +563,12 @@ async fn reap_idle_containers(client: DaytonaClient, managed: Arc<DaytonaContainbiased;() = managed.shutdown.cancelled() => return,_ = interval.tick() => {- let stale = managed.containers.reap_stale(Instant::now(), IDLE_TIMEOUT);+ let stale = managed.containers.reap_stale(Instant::now(), IDLE_TIMEOUT, |id| {+ managed+ .containers+ .session_of(id)+ .is_some_and(|session| managed.pending.is_pending(session))+ });stop_reaped(&client, &managed, stale).await;}}diff --git a/crates/agent_harness/src/outbound/managed_containers.rs b/crates/agent_harness/src/outbound/managed_containers.rsindex 88ec03296e..efb8504892 100644--- a/crates/agent_harness/src/outbound/managed_containers.rs+++ b/crates/agent_harness/src/outbound/managed_containers.rs@@ -1,6 +1,7 @@use std::hash::Hash;use std::time::{Duration, Instant};+use agent_session::domain::model::AgentSessionId;use dashmap::DashMap;#[cfg(test)]@@ -15,6 +16,10 @@ enum ContainerState {pub(crate) struct ManagedContainers<Id> {entries: DashMap<Id, ContainerState>,+ /// The session each container id was registered for, so a reaper -+ /// which only ever sees `Id` - can ask whether the harness has a+ /// command in flight for it.+ sessions: DashMap<Id, AgentSessionId>,}impl<Id> ManagedContainers<Id>@@ -24,11 +29,18 @@ wherepub(crate) fn new() -> Self {Self {entries: DashMap::new(),+ sessions: DashMap::new(),}}- pub(crate) fn register(&self, id: Id) {- self.entries.insert(id, ContainerState::Pending);+ pub(crate) fn register(&self, id: Id, session: AgentSessionId) {+ self.entries.insert(id.clone(), ContainerState::Pending);+ self.sessions.insert(id, session);+ }++ /// The session `id` was registered for, if it is still tracked.+ pub(crate) fn session_of(&self, id: &Id) -> Option<AgentSessionId> {+ self.sessions.get(id).map(|entry| *entry)}pub(crate) fn activate(&self, id: &Id, now: Instant) -> bool {@@ -50,16 +62,31 @@ where}pub(crate) fn remove(&self, id: &Id) -> bool {+ self.sessions.remove(id);self.entries.remove(id).is_some()}- pub(crate) fn reap_stale(&self, now: Instant, max_idle: Duration) -> Vec<Id> {+ /// Stop-candidate ids idle at least `max_idle`, excluding any `skip`+ /// answers true for.+ ///+ /// `skip` is checked before a candidate is claimed as stale: a session+ /// with a command already admitted but not yet turn-active - the harness+ /// marks this at admission, before the runtime visibly starts a turn -+ /// looks idle by activity timestamp alone, and reaping it now would only+ /// race that command's delivery.+ pub(crate) fn reap_stale(+ &self,+ now: Instant,+ max_idle: Duration,+ skip: impl Fn(&Id) -> bool,+ ) -> Vec<Id> {let candidates = self.entries.iter().filter_map(|entry| match entry.value() {ContainerState::Active { last_activity }- if now.saturating_duration_since(*last_activity) >= max_idle =>+ if now.saturating_duration_since(*last_activity) >= max_idle+ && !skip(entry.key()) =>{Some(entry.key().clone())}@@ -73,6 +100,7 @@ where};if let ContainerState::Active { last_activity } = *state&& now.saturating_duration_since(last_activity) >= max_idle+ && !skip(&id){*state = ContainerState::Stopping { last_activity };stale.push(id);diff --git a/crates/agent_harness/src/outbound/managed_containers/test.rs b/crates/agent_harness/src/outbound/managed_containers/test.rsindex ac780535a1..37e2558ff5 100644--- a/crates/agent_harness/src/outbound/managed_containers/test.rs+++ b/crates/agent_harness/src/outbound/managed_containers/test.rs@@ -1,7 +1,15 @@use super::*;+fn session(n: u128) -> AgentSessionId {+ AgentSessionId::new_from_uuid(macro_uuid::Uuid::from_u128(n))+}++fn never_skip<Id>(_: &Id) -> bool {+ false+}+fn activate(managed: &ManagedContainers<&str>, id: &'static str, last_activity: Instant) {- managed.register(id);+ managed.register(id, session(1));assert!(managed.activate(&id, last_activity));}@@ -17,7 +25,10 @@ fn reap_stale_returns_only_containers_idle_for_the_limit() {activate(&managed, "stale", seconds_ago(now, 300));activate(&managed, "active", seconds_ago(now, 299));- assert_eq!(managed.reap_stale(now, Duration::from_secs(300)), ["stale"]);+ assert_eq!(+ managed.reap_stale(now, Duration::from_secs(300), never_skip),+ ["stale"]+ );}#[test]@@ -28,21 +39,29 @@ fn recording_activity_keeps_a_container_alive() {managed.record_activity(&"active", now);- assert!(managed.reap_stale(now, Duration::from_secs(300)).is_empty());+ assert!(+ managed+ .reap_stale(now, Duration::from_secs(300), never_skip)+ .is_empty()+ );}#[test]fn pending_and_stopping_containers_are_not_reaped_again() {let managed = ManagedContainers::new();let now = Instant::now();- managed.register("pending");+ managed.register("pending", session(1));activate(&managed, "stopping", seconds_ago(now, 300));assert_eq!(- managed.reap_stale(now, Duration::from_secs(300)),+ managed.reap_stale(now, Duration::from_secs(300), never_skip),["stopping"]);- assert!(managed.reap_stale(now, Duration::ZERO).is_empty());+ assert!(+ managed+ .reap_stale(now, Duration::ZERO, never_skip)+ .is_empty()+ );}#[test]@@ -51,12 +70,34 @@ fn activity_during_a_failed_stop_is_preserved() {let now = Instant::now();activate(&managed, "active", seconds_ago(now, 300));assert_eq!(- managed.reap_stale(now, Duration::from_secs(300)),+ managed.reap_stale(now, Duration::from_secs(300), never_skip),["active"]);managed.record_activity(&"active", now);managed.finish_stop(&"active", false);- assert!(managed.reap_stale(now, Duration::from_secs(300)).is_empty());+ assert!(+ managed+ .reap_stale(now, Duration::from_secs(300), never_skip)+ .is_empty()+ );+}++#[test]+fn a_session_with_a_pending_command_is_not_reaped() {+ let managed = ManagedContainers::new();+ let now = Instant::now();+ let pending_session = session(2);+ managed.register("busy", pending_session);+ assert!(managed.activate(&"busy", seconds_ago(now, 300)));++ assert!(+ managed+ .reap_stale(now, Duration::from_secs(300), |id| managed+ .session_of(id)+ .is_some_and(|s| s == pending_session))+ .is_empty(),+ "a container whose session has a pending command must not be reaped"+ );}diff --git a/crates/agent_session/src/domain/connection.rs b/crates/agent_session/src/domain/connection.rsindex 35ad521a10..f9321825bf 100644--- a/crates/agent_session/src/domain/connection.rs+++ b/crates/agent_session/src/domain/connection.rs@@ -150,6 +150,12 @@ pub struct RuntimeAttachment<Connector> {/// fresh at each attach - the set follows what the owner has connected/// *now*, not what they had connected when the session was created.pub(crate) mcp_servers: Vec<McpServer>,+ /// Cancelled by the connector the moment its transport ends, for+ /// connectors that can tell - most can, via whatever already tears their+ /// pipe or socket down. `None` for a connector with no such signal;+ /// callers waiting on a reply then fall back to their own timeout alone,+ /// exactly as before this existed.+ pub(crate) closed: Option<CancellationToken>,}/// Activate attachment-owned resources with the exact acquired ownership claim.@@ -168,6 +174,7 @@ impl<Connector> RuntimeAttachment<Connector> {connector,handshake,mcp_servers: Vec::new(),+ closed: None,}}@@ -178,6 +185,17 @@ impl<Connector> RuntimeAttachment<Connector> {self}+ /// Wire up the connector's own transport-closed signal, when it has one.+ ///+ /// Lets a command waiting on this session's reply fail the moment the+ /// transport is known to be gone, rather than only once its own+ /// wall-clock timeout separately elapses.+ #[must_use]+ pub fn with_closed(mut self, closed: CancellationToken) -> Self {+ self.closed = Some(closed);+ self+ }+/// Change the carrier without losing its handshake or activation lifecycle.pub fn map_transport<T>(self, map: impl FnOnce(Connector) -> T) -> RuntimeAttachment<T> {RuntimeAttachment {@@ -185,6 +203,7 @@ impl<Connector> RuntimeAttachment<Connector> {handshake: self.handshake,mcp_servers: self.mcp_servers,activation: self.activation,+ closed: self.closed,}}@@ -361,6 +380,9 @@ where// External runtimes hold no egress environment; the sessions// they serve are not handed proxied MCP servers.mcp_servers: Vec::new(),+ // This connection already tracks its own end (`evict`, or the+ // router task finishing) - the same signal `closed()` awaits.+ closed: Some(self.closed.clone()),}}diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex 8f682449ee..5cace53e36 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -86,6 +86,12 @@ struct ActiveSession {marker: Arc<()>,deleting: bool,stopping: bool,+ /// Cancelled by the connector the moment its transport ends, when it+ /// offers one - not every connector does. `deliver_action` races this+ /// alongside its reply wait so a command sent to a session whose+ /// transport is already known to be gone fails immediately instead of+ /// waiting out the full command timeout for nothing.+ transport_closed: Option<CancellationToken>,}type ActiveSessions = DashMap<AgentSessionId, ActiveSession>;@@ -350,6 +356,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {marker: marker.clone(),deleting: false,stopping: false,+ transport_closed: None,});Ok(AttachReservation {active: self.active.clone(),@@ -390,6 +397,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {activate(claim)?;}active.commands = Some(commands.clone());+ active.transport_closed = attachment.closed.clone();drop(active);let (marker, stopped_tx) = reservation.commit();@@ -435,10 +443,10 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {action: AgentAction,action_id: AgentActionId,) -> Result<()> {- let commands = self+ let (commands, transport_closed) = self.active.get(&id)- .and_then(|entry| entry.commands.clone())+ .and_then(|entry| Some((entry.commands.clone()?, entry.transport_closed.clone()))).ok_or(AgentSessionError::Disconnected(id))?;let (completed, result) = oneshot::channel();@@ -471,10 +479,26 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {// can only notice by hitting its own much longer internal deadline// instead of promptly.drop(commands);- match tokio::time::timeout(HANDSHAKE_TIMEOUT, result).await {- Ok(Ok(result)) => result,- Ok(Err(_)) => Err(AgentSessionError::Disconnected(id)),- Err(_elapsed) => {+ // A transport that has already ended has no reply coming, however+ // long we wait: race the connector's own closed signal (when it has+ // one) alongside the reply so that case fails at once instead of+ // riding out the rest of `HANDSHAKE_TIMEOUT` for nothing. A+ // connector with no such signal just never resolves this side of+ // the select, unchanged from before.+ let transport_closed_first = async {+ match &transport_closed {+ Some(closed) => closed.cancelled().await,+ None => std::future::pending().await,+ }+ };+ let timed_out = tokio::select! {+ res = tokio::time::timeout(HANDSHAKE_TIMEOUT, result) => Some(res),+ () = transport_closed_first => None,+ };+ match timed_out {+ Some(Ok(Ok(result))) => result,+ Some(Ok(Err(_))) => Err(AgentSessionError::Disconnected(id)),+ Some(Err(_elapsed)) => {// The actor is presumably still stuck in the handshake, so it// is stopped directly - the same "drop the sender, the actor// notices and tears itself down" mechanism `close_session`@@ -488,6 +512,18 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {tracing::warn!(%id, "agent session command timed out waiting for the ACP handshake");Err(AgentSessionError::Disconnected(id))}+ None => {+ // The connector's transport already ended - no wall-clock+ // wait can help, so stop the actor the same way a timeout+ // does and fail now.+ let (stopped, marker) = self.begin_stop(id, false);+ Self::wait_stopped(stopped).await;+ self.active.remove_if(&id, |_, active| {+ Arc::ptr_eq(&active.marker, &marker) && !active.deleting+ });+ tracing::info!(%id, "agent session command failed fast: transport already closed");+ Err(AgentSessionError::Disconnected(id))+ }}}@@ -516,6 +552,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {marker: marker.clone(),deleting,stopping: true,+ transport_closed: None,});(stopped, marker)}diff --git a/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs b/crates/cursor_cloud_agents/src/outbound/postgres_journal.rsindex 8d3408d1d9..05964f7734 100644--- a/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs+++ b/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs@@ -6,6 +6,20 @@ use agent_session::domain::model::{AgentSessionId, ManagerFence, ReplicaId};use futures::future::BoxFuture;use sqlx::PgPool;+/// Postgres' `lock_not_available` SQLSTATE, the error `FOR UPDATE NOWAIT`+/// raises instead of blocking. <https://www.postgresql.org/docs/current/errcodes-appendix.html>+const LOCK_NOT_AVAILABLE: &str = "55P03";++/// Whether `error` is exactly the row already being locked by another+/// transaction - the one outcome `NOWAIT` exists to turn into an immediate+/// error rather than a wait.+fn is_lock_not_available(error: &sqlx::Error) -> bool {+ matches!(+ error,+ sqlx::Error::Database(db_err) if db_err.code().as_deref() == Some(LOCK_NOT_AVAILABLE)+ )+}+/// Bound to exactly one authorized host session and its current management/// claim. A takeover updates the same locked row and invalidates this writer.#[derive(Debug)]@@ -53,12 +67,33 @@ impl PgCursorJournal {// Dispatch fencing cannot stop an already-running stream or mirror// poll. Hold the same row takeover updates until the journal commits;// checking the claim before opening this transaction would race.+ //+ // NOWAIT, not a plain FOR UPDATE: a writer whose claim is being+ // superseded (a resume/reattach bumping this same row's fence) has no+ // business waiting for that other transaction to commit - it has+ // already lost, whatever the wait would answer. Blocking here used to+ // leave a stale writer's read/append hanging until some caller far+ // above gave up on an unrelated wall-clock timeout (tens of seconds)+ // and force-tore the session down from outside; failing the lock+ // immediately turns that into a fencing error on the spot, which is+ // the caller's cue to reattach right away instead of stalling first.let expected = *self.fence.get().ok_or_else(|| rootcause::report!("Cursor journal attachment is not activated"))?;- let current = sqlx::query_scalar!("SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE", self.session.as_uuid(), self.replica.as_uuid())- .fetch_optional(&mut **tx).await.map_err(|e| rootcause::report!(e))?+ let current = sqlx::query_scalar!("SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE NOWAIT", self.session.as_uuid(), self.replica.as_uuid())+ .fetch_optional(&mut **tx).await.map_err(|e| {+ if is_lock_not_available(&e) {+ // Someone else - almost always a superseding claim - holds+ // this row right now. We would lose that race even if we+ // waited, so report it exactly like a lost fence.+ rootcause::report!("Cursor journal writer fenced out")+ } else {+ // Every other database failure keeps its type, SQLSTATE+ // and source chain, as elsewhere in this file.+ rootcause::report!(e).into_dynamic()+ }+ })?.ok_or_else(|| rootcause::report!("Cursor journal writer fenced out"))?;if ManagerFence(current) != expected {return Err(rootcause::report!("Cursor journal writer fenced out"));diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rsindex c54adb4fa7..a592c9f633 100644--- a/services/agent_harness_service/src/main.rs+++ b/services/agent_harness_service/src/main.rs@@ -247,6 +247,14 @@ async fn run() -> anyhow::Result<()> {replica,);+ // Sessions with a command admitted but not yet resolved - shared with+ // every provider whose idle reaper closes a session's transport on its+ // own schedule, so a reaper never pulls the transport out from under a+ // command already on its way in. Built before the container managers+ // below (which read it) and handed to the harness after them (which+ // marks and clears it); nothing else needs to know its type.+ let pending_commands = agent_harness::domain::pending::PendingCommands::new();+// Containers: the sandbox provider (local Docker when a developer has// opted in, Daytona otherwise) plus Cursor cloud agents for the `@cursor`// bot, routed per session.@@ -285,12 +293,15 @@ async fn run() -> anyhow::Result<()> {"DAYTONA_API_KEY is unset: Daytona-backed sandboxes are unarmed; external agent sessions are unaffected");}- HarnessContainers::Daytona(DaytonaContainerManager::new(DaytonaSettings {- api_url: config.daytona_api_url.clone(),- api_key: DaytonaApiKeySecret::new(config.daytona_api_key.clone()),- snapshot: Snapshot::new(config.daytona_snapshot.clone()),- anthropic_api_key,- }))+ HarnessContainers::Daytona(DaytonaContainerManager::new(+ DaytonaSettings {+ api_url: config.daytona_api_url.clone(),+ api_key: DaytonaApiKeySecret::new(config.daytona_api_key.clone()),+ snapshot: Snapshot::new(config.daytona_snapshot.clone()),+ anthropic_api_key,+ },+ pending_commands.clone(),+ ))};let container_shutdown = sandbox.clone();@@ -418,6 +429,7 @@ async fn run() -> anyhow::Result<()> {session_repo.clone(),pool.clone(),replica,+ pending_commands.clone(),);// Fixed system agents retain their deployment defaults. User/team agents// are resolved from agent_configs for every trigger so newly-created or@@ -549,6 +561,7 @@ async fn run() -> anyhow::Result<()> {RedisCommandForwarder::new(redis.clone()),defaults,Arc::clone(&lifecycle_publisher),+ pending_commands,));// Close the loop: turn ends observed by the session actors drain the// harness's prompt queue.