Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

cursor journal: fail fast on a superseded claim instead of blocking (#6327)

Wolf Mermelstein0071fb5d87d4
diff --git a/.sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json b/.sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json
similarity index 73%
rename from .sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json
rename to .sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json
index 870cdca190..2992fe05bf 100644
--- a/.sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json
+++ b/.sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE",
+ "query": "SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE NOWAIT",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
false
]
},
- "hash": "9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab"
+ "hash": "48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd"
}
diff --git a/crates/agent_harness/src/domain/mod.rs b/crates/agent_harness/src/domain/mod.rs
index fbc1b6464c..9f33d22ee2 100644
--- a/crates/agent_harness/src/domain/mod.rs
+++ b/crates/agent_harness/src/domain/mod.rs
@@ -2,6 +2,9 @@ pub mod error;
pub mod model;
/// Fresh model discovery without creating an agent session.
pub mod model_load;
+/// A shared record of sessions with a command admitted but not yet resolved,
+/// consulted by container managers' idle reapers.
+pub mod pending;
pub mod ports;
/// The per-session queue of turn-occupying actions awaiting their turn.
pub mod queue;
diff --git a/crates/agent_harness/src/domain/pending.rs b/crates/agent_harness/src/domain/pending.rs
new file mode 100644
index 0000000000..13bfe08e52
--- /dev/null
+++ b/crates/agent_harness/src/domain/pending.rs
@@ -0,0 +1,145 @@
+//! A shared record of sessions with a command admitted but not yet acted on,
+//! consulted by container managers' idle reapers so they never pull a
+//! session's transport out from under a command already on its way in.
+//!
+//! Built once at process wiring and handed to both [`crate::domain::service`]
+//! (which marks and clears it) and every [`crate::domain::ports::ContainerManager`]
+//! whose provider reaps idle transports (which only reads it). Neither side
+//! needs the other's type, which is what lets the container managers be
+//! constructed before the harness that owns them.
+//!
+//! A session is marked from the moment its command is admitted, which is
+//! before any turn exists to describe - hence the two-stage value: admission
+//! records only that something is coming, and dispatch fills in the turn it
+//! opened. A reaper only ever asks the first question; the harness's own
+//! lifecycle events need the second.
+
+use std::sync::Arc;
+
+use agent_session::domain::model::AgentSessionId;
+use dashmap::DashMap;
+
+use crate::domain::queue::InFlightTurn;
+
+/// Cheap to clone; every clone shares the same underlying map.
+#[derive(Clone, Default)]
+pub struct PendingCommands(Arc<DashMap<AgentSessionId, Option<InFlightTurn>>>);
+
+impl PendingCommands {
+ /// An empty tracker: no session marked pending.
+ #[must_use]
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Record that `session` has a command admitted, before it has been
+ /// dispatched and so before there is a turn to name.
+ ///
+ /// Only called for a session not already pending, so it can never
+ /// overwrite a dispatched turn with an empty admission.
+ pub fn admit(&self, session: AgentSessionId) {
+ self.0.insert(session, None);
+ }
+
+ /// Record the turn a dispatched action opened for `session`.
+ pub fn mark_turn(&self, session: AgentSessionId, turn: InFlightTurn) {
+ self.0.insert(session, Some(turn));
+ }
+
+ /// Clear `session`'s mark, if any.
+ pub fn clear(&self, session: AgentSessionId) {
+ self.0.remove(&session);
+ }
+
+ /// Clear `session`'s mark and hand back the turn it had in flight, if it
+ /// had got as far as dispatching one.
+ pub fn take(&self, session: AgentSessionId) -> Option<InFlightTurn> {
+ self.0.remove(&session).and_then(|(_, turn)| turn)
+ }
+
+ /// The turn `session` currently has in flight, if it has one.
+ ///
+ /// Absent both for a session with nothing pending and for one whose
+ /// command is admitted but not yet dispatched - callers reporting on a
+ /// turn have nothing to say in either case.
+ #[must_use]
+ pub fn turn(&self, session: AgentSessionId) -> Option<InFlightTurn> {
+ self.0.get(&session).and_then(|entry| entry.clone())
+ }
+
+ /// Whether `session` currently has a command admitted and unresolved.
+ ///
+ /// A reaper's cue to leave the session's transport alone this tick even
+ /// though it otherwise looks idle: something is already on its way to
+ /// it, and closing the transport now would only race that delivery.
+ #[must_use]
+ pub fn is_pending(&self, session: AgentSessionId) -> bool {
+ self.0.contains_key(&session)
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use agent_fold::domain::model::TurnId;
+ use agent_runtime_protocol::domain::action::AgentActionId;
+
+ use super::*;
+
+ fn session_id(n: u128) -> AgentSessionId {
+ AgentSessionId::new_from_uuid(macro_uuid::Uuid::from_u128(n))
+ }
+
+ fn turn() -> InFlightTurn {
+ InFlightTurn {
+ action_id: AgentActionId::mint(),
+ turn: TurnId(7),
+ actor: None,
+ announcement_message_id: None,
+ }
+ }
+
+ #[test]
+ fn admits_and_clears() {
+ let pending = PendingCommands::new();
+ let id = session_id(1);
+ assert!(!pending.is_pending(id));
+ pending.admit(id);
+ assert!(pending.is_pending(id));
+ pending.clear(id);
+ assert!(!pending.is_pending(id));
+ }
+
+ #[test]
+ fn an_admitted_session_has_no_turn_until_dispatch() {
+ let pending = PendingCommands::new();
+ let id = session_id(2);
+ pending.admit(id);
+ assert!(
+ pending.turn(id).is_none(),
+ "admission alone names no turn, but is still pending"
+ );
+ let dispatched = turn();
+ pending.mark_turn(id, dispatched.clone());
+ assert_eq!(pending.turn(id).map(|t| t.turn), Some(dispatched.turn));
+ }
+
+ #[test]
+ fn take_clears_and_returns_the_turn() {
+ let pending = PendingCommands::new();
+ let id = session_id(3);
+ let dispatched = turn();
+ pending.mark_turn(id, dispatched.clone());
+ assert_eq!(pending.take(id).map(|t| t.turn), Some(dispatched.turn));
+ assert!(!pending.is_pending(id));
+ assert!(pending.take(id).is_none(), "a second take has nothing left");
+ }
+
+ #[test]
+ fn clones_share_state() {
+ let pending = PendingCommands::new();
+ let clone = pending.clone();
+ let id = session_id(4);
+ pending.admit(id);
+ assert!(clone.is_pending(id), "clones observe the same map");
+ }
+}
diff --git a/crates/agent_harness/src/domain/service.rs b/crates/agent_harness/src/domain/service.rs
index fe1d52f8e2..a6a3ec4922 100644
--- a/crates/agent_harness/src/domain/service.rs
+++ b/crates/agent_harness/src/domain/service.rs
@@ -49,6 +49,7 @@ use crate::domain::model::{
AgentKind, AnnounceOrigin, AnnouncePrompt, CommandOutcome, DeliverAction, HarnessCommand,
HarnessDefaults, OpenSession, SessionAnnouncement, SpawnContainer, is_macro_staff,
};
+use crate::domain::pending::PendingCommands;
use crate::domain::ports::{
AgentPromptComposer, ChannelPromptContext, CommandForwarder, ContainerManager,
RuntimeConnections, SandboxEgressProvisioner, SessionAnnouncer,
@@ -79,11 +80,19 @@ struct AgentHarnessInner<
/// Turn-occupying actions waiting for their session's running turn to
/// end. In-memory beside the live actors this replica manages.
queues: SessionQueues,
- /// The sessions with a turn in flight, and which turn. Marked when a
- /// turn-occupying action reaches the runtime, cleared by
- /// `TurnEnded`/`SessionStopped`. Only ever touched from the session's
- /// own command worker, which is what serializes it against dispatch.
- busy: DashMap<AgentSessionId, InFlightTurn>,
+ /// The sessions with a command admitted and not yet resolved, and which
+ /// turn it opened once dispatch names one. Marked the moment a
+ /// turn-occupying action is admitted (queue.rs's `enqueue_then_dispatch`),
+ /// cleared by `TurnEnded`/`SessionStopped` or on admission failure. Only
+ /// ever touched from the session's own command worker, which is what
+ /// serializes it against dispatch.
+ ///
+ /// Shared (not private to this service) so a provider's idle reaper can
+ /// read it before closing a session's transport: marking on admission
+ /// rather than on delivery success is what closes the gap between "a
+ /// command was handed to this session" and "the runtime has visibly
+ /// started a turn", which a reaper watching only the latter cannot see.
+ busy: PendingCommands,
/// Where lifecycle facts go. Erased so it is not an eighth type parameter.
lifecycle_publisher: Arc<dyn AgentSessionLifecyclePublisher>,
}
@@ -170,6 +179,7 @@ where
forwarder: impl CommandForwarder,
defaults: impl Into<HarnessDefaults>,
lifecycle_publisher: impl AgentSessionLifecyclePublisher,
+ pending: PendingCommands,
) -> Self {
Self {
inner: Arc::new(AgentHarnessInner {
@@ -183,7 +193,7 @@ where
forwarder: Box::new(forwarder),
defaults: defaults.into(),
queues: SessionQueues::new(),
- busy: DashMap::new(),
+ busy: pending,
lifecycle_publisher: Arc::new(lifecycle_publisher),
}),
workers: Arc::new(DashMap::new()),
diff --git a/crates/agent_harness/src/domain/service/queue.rs b/crates/agent_harness/src/domain/service/queue.rs
index 8dc9e3e8eb..6e9f863266 100644
--- a/crates/agent_harness/src/domain/service/queue.rs
+++ b/crates/agent_harness/src/domain/service/queue.rs
@@ -313,7 +313,7 @@ where
action_id: fold_action_id,
..
}) => {
- let ended = self.busy.remove(&session_id).map(|(_, turn)| turn);
+ let ended = self.busy.take(session_id);
// A turn end with no record: this replica restarted mid-turn
// and the in-memory mark went with it, or the fold closed a
// turn nobody here prompted. The queue still drains; only the
@@ -372,7 +372,7 @@ where
Ok(CommandOutcome::Completed)
}
HarnessCommand::SessionStopped { reason } => {
- let in_flight = self.busy.remove(&session_id).map(|(_, turn)| turn);
+ let in_flight = self.busy.take(session_id);
self.publish_lifecycle(session_id, |identity| {
AgentSessionLifecycleEvent::Stopped(SessionStoppedMetadata {
identity,
@@ -384,7 +384,7 @@ where
Ok(CommandOutcome::Completed)
}
HarnessCommand::Turn(TurnSignal::ElicitationRaised { question, .. }) => {
- let Some(turn) = self.busy.get(&session_id).map(|turn| turn.clone()) else {
+ let Some(turn) = self.busy.turn(session_id) else {
tracing::warn!(%session_id, "elicitation raised with no in-flight record");
return Ok(CommandOutcome::Completed);
};
@@ -401,7 +401,7 @@ where
Ok(CommandOutcome::Completed)
}
HarnessCommand::Turn(TurnSignal::ElicitationCleared { .. }) => {
- let Some(turn) = self.busy.get(&session_id).map(|turn| turn.clone()) else {
+ let Some(turn) = self.busy.turn(session_id) else {
tracing::warn!(%session_id, "elicitation cleared with no in-flight record");
return Ok(CommandOutcome::Completed);
};
@@ -427,7 +427,7 @@ where
// session's entries will never dispatch, and leaving them
// would leak them for the life of the process. The published
// empty snapshot is the viewers' goodbye.
- self.busy.remove(&session_id);
+ self.busy.clear(session_id);
self.queues.drop_session(session_id);
self.publish_queue(session_id).await;
match identity {
@@ -477,10 +477,24 @@ where
session_id,
)?;
- let dispatched = if self.busy.contains_key(&session_id) {
+ let dispatched = if self.busy.is_pending(session_id) {
Ok(())
} else {
- self.dispatch_next(session_id).await.map(drop)
+ // Marked before dispatching, not only once `dispatch_next`'s own
+ // delivery succeeds: this closes the window between "a command
+ // was just handed off for this session" and "the runtime
+ // visibly started a turn", which a reaper watching only the
+ // latter cannot see. The turn itself is unknowable this early -
+ // `dispatch_next` fills it in with `mark_turn` once delivery
+ // names one - so this records only that something is coming. On
+ // failure nothing actually started, so the mark comes back off
+ // rather than sticking to a session with nothing running.
+ self.busy.admit(session_id);
+ let result = self.dispatch_next(session_id).await.map(drop);
+ if result.is_err() {
+ self.busy.clear(session_id);
+ }
+ result
};
self.publish_queue(session_id).await;
dispatched?;
@@ -599,7 +613,7 @@ where
actor: entry.actor,
announcement_message_id: entry.announced,
};
- self.busy.insert(session_id, turn.clone());
+ self.busy.mark_turn(session_id, turn.clone());
self.publish_lifecycle(session_id, |identity| {
AgentSessionLifecycleEvent::TurnStarted(TurnStartedMetadata {
identity,
diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rs
index f761eb0eb1..dbb2438229 100644
--- a/crates/agent_harness/src/domain/service/test.rs
+++ b/crates/agent_harness/src/domain/service/test.rs
@@ -345,6 +345,7 @@ fn harness_with_signals(
repo_url: "https://github.com/macro-inc/macro".to_owned(),
},
lifecycle.clone(),
+ crate::domain::pending::PendingCommands::new(),
);
let (ended, ended_rx) = mpsc::unbounded_channel();
turn_observer.bind(SignallingTurnObserver {
@@ -1863,6 +1864,7 @@ async fn a_managed_session_opens_as_the_managed_default_bot() {
)
.with_managed_bot(inmem_bot),
NoopLifecyclePublisher,
+ crate::domain::pending::PendingCommands::new(),
);
let session = service
@@ -2308,6 +2310,7 @@ async fn commands_for_a_peer_managed_session_forward_through_redis() {
repo_url: "https://github.com/macro-inc/macro".to_owned(),
},
NoopLifecyclePublisher,
+ crate::domain::pending::PendingCommands::new(),
);
service
@@ -2356,6 +2359,7 @@ async fn unmanaged_external_session_forwards_to_its_remote_harness() {
repo_url: "https://github.com/macro-inc/macro".to_owned(),
},
NoopLifecyclePublisher,
+ crate::domain::pending::PendingCommands::new(),
);
let session = service
.open_external_session(open_external_request("/srv/agent"))
diff --git a/crates/agent_harness/src/outbound/cursor/manager.rs b/crates/agent_harness/src/outbound/cursor/manager.rs
index a148054e17..2a54561ff8 100644
--- a/crates/agent_harness/src/outbound/cursor/manager.rs
+++ b/crates/agent_harness/src/outbound/cursor/manager.rs
@@ -39,6 +39,7 @@ use super::keys::CursorApiKeys;
use super::pipe::PipeTransport;
use crate::domain::error::{HarnessError, Result};
use crate::domain::model::SpawnContainer;
+use crate::domain::pending::PendingCommands;
use crate::domain::ports::ContainerManager;
use crate::domain::sandbox::SandboxResizeEffect;
use agent_session::domain::model::SandboxSize;
@@ -85,8 +86,8 @@ const CURSOR_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(
/// for reclaiming two idle tasks is nothing.
const CURSOR_IDLE_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
-fn should_reap_cursor_pipe(idle: std::time::Duration, active_turn: bool) -> bool {
- idle >= CURSOR_IDLE_TIMEOUT && !active_turn
+fn should_reap_cursor_pipe(idle: std::time::Duration, active_turn: bool, pending: bool) -> bool {
+ idle >= CURSOR_IDLE_TIMEOUT && !active_turn && !pending
}
/// The ref new agents start their work from.
@@ -127,6 +128,10 @@ pub struct CursorContainerManager<Sessions, Keys> {
repo: CursorRepoUrl,
sessions: Sessions,
journal_storage: JournalStorage,
+ /// Sessions the harness has a command in flight for right now, shared
+ /// with `AgentHarnessService` so the idle reaper below never closes a
+ /// pipe a command is already on its way to.
+ pending: PendingCommands,
}
/// Hosted sessions always use durable storage; tests select memory explicitly.
@@ -169,6 +174,7 @@ where
sessions: Sessions,
pool: sqlx::PgPool,
replica: ReplicaId,
+ pending: PendingCommands,
) -> Self {
Self {
keys,
@@ -176,6 +182,7 @@ where
repo,
sessions,
journal_storage: JournalStorage::Postgres { pool, replica },
+ pending,
}
}
@@ -192,6 +199,7 @@ where
repo,
sessions,
journal_storage: JournalStorage::Memory,
+ pending: PendingCommands::new(),
}
}
@@ -295,6 +303,12 @@ where
);
}
let pipe_closed = tokio_util::sync::CancellationToken::new();
+ // Cloned before the tasks below move `pipe_closed` itself: the
+ // attachment built at the end of this function hands this same
+ // signal to the session actor's command wait, so a command sent
+ // right as - or just after - this pipe dies fails at once instead
+ // of riding out its own separate timeout for nothing.
+ let attachment_closed = pipe_closed.clone();
let shutdown = tokio_util::sync::CancellationToken::new();
let sync_service = Arc::clone(&service);
let on_pipe_close = pipe_closed.clone();
@@ -313,6 +327,7 @@ where
let last_activity = Arc::new(std::sync::Mutex::new(tokio::time::Instant::now()));
let observed = Arc::clone(&last_activity);
let reaper_shutdown = shutdown.clone();
+ let pending = self.pending.clone();
tokio::spawn(async move {
let mut mirror = interval_from_now(FOREIGN_SYNC_INTERVAL);
let mut reaper = interval_from_now(CURSOR_IDLE_CHECK_INTERVAL);
@@ -324,6 +339,12 @@ where
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let active_turn = sync_service.has_active_turn();
+ // Checked alongside `active_turn`: a command already
+ // admitted for this session but not yet turn-active
+ // (the harness marks this at admission, before
+ // dispatch - see `queue::enqueue_then_dispatch`) is
+ // exactly the case `active_turn` alone cannot see.
+ let pending_command = pending.is_pending(session_id);
let activity = last_activity
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
@@ -333,7 +354,11 @@ where
let raced = *activity != observed_at;
let idle_ms = activity.elapsed().as_millis();
let reaped = !raced
- && should_reap_cursor_pipe(activity.elapsed(), active_turn);
+ && should_reap_cursor_pipe(
+ activity.elapsed(),
+ active_turn,
+ pending_command,
+ );
// Every tick, not just the reaping one. The inputs to
// this decision are what tell a pipe that died of
// idleness apart from one pulled out from under a
@@ -345,6 +370,7 @@ where
%session_id,
agent.pipe.idle_ms = idle_ms as u64,
agent.pipe.active_turn = active_turn,
+ agent.pipe.pending_command = pending_command,
agent.pipe.activity_raced = raced,
agent.pipe.reaped = reaped,
"cursor pipe idle check"
@@ -377,7 +403,8 @@ where
shutdown,
Some(reload_rx),
);
- let mut attachment = agent_session::domain::connection::RuntimeAttachment::solo(transport);
+ let mut attachment = agent_session::domain::connection::RuntimeAttachment::solo(transport)
+ .with_closed(attachment_closed);
if let Some(binding) = owner_binding {
attachment = attachment.on_activate(binding);
}
diff --git a/crates/agent_harness/src/outbound/cursor/manager/test.rs b/crates/agent_harness/src/outbound/cursor/manager/test.rs
index 2a19c6cb21..6c43e67f9b 100644
--- a/crates/agent_harness/src/outbound/cursor/manager/test.rs
+++ b/crates/agent_harness/src/outbound/cursor/manager/test.rs
@@ -733,8 +733,16 @@ async fn an_idle_pipe_is_shut_down() {
#[test]
fn a_pipe_is_not_idle_while_cursor_is_running_a_turn() {
- assert!(!should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, true));
- assert!(should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, false));
+ assert!(!should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, true, false));
+ assert!(should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, false, false));
+}
+
+#[test]
+fn a_pipe_is_not_idle_while_a_command_is_pending() {
+ // Same idle duration, no active turn yet - the case a command admitted
+ // just before the reap tick looks like, before the runtime has had a
+ // chance to mark a turn active.
+ assert!(!should_reap_cursor_pipe(CURSOR_IDLE_TIMEOUT, false, true));
}
/// Teardown archives the agent on cursor.com and forgets the mapping; a
diff --git a/crates/agent_harness/src/outbound/daytona/manager.rs b/crates/agent_harness/src/outbound/daytona/manager.rs
index abb39819d9..2062c8070a 100644
--- a/crates/agent_harness/src/outbound/daytona/manager.rs
+++ b/crates/agent_harness/src/outbound/daytona/manager.rs
@@ -15,6 +15,7 @@ use super::errors::DaytonaError;
use super::types::{AnthropicApiKey, DaytonaSettings, Env, Labels, PortPreview, Snapshot};
use crate::domain::error::{HarnessError, Result};
use crate::domain::model::SpawnContainer;
+use crate::domain::pending::PendingCommands;
use crate::domain::ports::ContainerManager;
use crate::domain::sandbox::{
SandboxResizeEffect, SandboxResources, resize_effect_from_resources, resources,
@@ -49,6 +50,10 @@ struct DaytonaContainerManagerState {
shutdown_complete: CancellationToken,
lifecycle: Mutex<ManagerLifecycle>,
tasks: TaskTracker,
+ /// Sessions the harness has a command in flight for right now; consulted
+ /// by the idle reaper so it never stops a sandbox a command is already
+ /// on its way to.
+ pending: PendingCommands,
}
#[derive(Default)]
@@ -57,17 +62,18 @@ struct ManagerLifecycle {
}
impl DaytonaContainerManagerState {
- fn new() -> Self {
+ fn new(pending: PendingCommands) -> Self {
Self {
containers: ManagedContainers::new(),
shutdown: CancellationToken::new(),
shutdown_complete: CancellationToken::new(),
lifecycle: Mutex::new(ManagerLifecycle::default()),
tasks: TaskTracker::new(),
+ pending,
}
}
- fn register(&self, id: DaytonaSandboxId) -> bool {
+ fn register(&self, id: DaytonaSandboxId, session: AgentSessionId) -> bool {
let lifecycle = self
.lifecycle
.lock()
@@ -75,7 +81,7 @@ impl DaytonaContainerManagerState {
if lifecycle.shutting_down {
return false;
}
- self.containers.register(id);
+ self.containers.register(id, session);
true
}
}
@@ -92,7 +98,7 @@ pub struct DaytonaContainerManager {
impl DaytonaContainerManager {
/// Build the manager from its settings.
#[must_use]
- pub fn new(settings: DaytonaSettings) -> Self {
+ pub fn new(settings: DaytonaSettings, pending: PendingCommands) -> Self {
let DaytonaSettings {
api_url,
api_key,
@@ -100,7 +106,7 @@ impl DaytonaContainerManager {
anthropic_api_key,
} = settings;
let client = DaytonaClient::new(api_url, api_key);
- let managed = Arc::new(DaytonaContainerManagerState::new());
+ let managed = Arc::new(DaytonaContainerManagerState::new(pending));
managed
.tasks
.spawn(reap_idle_containers(client.clone(), managed.clone()));
@@ -386,7 +392,7 @@ impl ContainerManager for DaytonaContainerManager {
.map_err(unavailable)?,
);
tracing::info!(sandbox_id = %id.as_str(), session = %session_id, "sandbox created");
- if !self.managed.register(id.clone()) {
+ if !self.managed.register(id.clone(), session_id) {
if !stop_sandbox(&self.client, &id, "shutdown during sandbox creation").await {
self.managed
.containers
@@ -473,7 +479,7 @@ impl ContainerManager for DaytonaContainerManager {
HarnessError::Container(format!("session {session} has no sandbox to resume"))
})?,
);
- if !self.managed.register(id.clone()) {
+ if !self.managed.register(id.clone(), session) {
return Err(HarnessError::Container(
"the container manager is shutting down".to_owned(),
));
@@ -557,7 +563,12 @@ async fn reap_idle_containers(client: DaytonaClient, managed: Arc<DaytonaContain
biased;
() = managed.shutdown.cancelled() => return,
_ = interval.tick() => {
- let stale = managed.containers.reap_stale(Instant::now(), IDLE_TIMEOUT);
+ let stale = managed.containers.reap_stale(Instant::now(), IDLE_TIMEOUT, |id| {
+ managed
+ .containers
+ .session_of(id)
+ .is_some_and(|session| managed.pending.is_pending(session))
+ });
stop_reaped(&client, &managed, stale).await;
}
}
diff --git a/crates/agent_harness/src/outbound/managed_containers.rs b/crates/agent_harness/src/outbound/managed_containers.rs
index 88ec03296e..efb8504892 100644
--- a/crates/agent_harness/src/outbound/managed_containers.rs
+++ b/crates/agent_harness/src/outbound/managed_containers.rs
@@ -1,6 +1,7 @@
use std::hash::Hash;
use std::time::{Duration, Instant};
+use agent_session::domain::model::AgentSessionId;
use dashmap::DashMap;
#[cfg(test)]
@@ -15,6 +16,10 @@ enum ContainerState {
pub(crate) struct ManagedContainers<Id> {
entries: DashMap<Id, ContainerState>,
+ /// The session each container id was registered for, so a reaper -
+ /// which only ever sees `Id` - can ask whether the harness has a
+ /// command in flight for it.
+ sessions: DashMap<Id, AgentSessionId>,
}
impl<Id> ManagedContainers<Id>
@@ -24,11 +29,18 @@ where
pub(crate) fn new() -> Self {
Self {
entries: DashMap::new(),
+ sessions: DashMap::new(),
}
}
- pub(crate) fn register(&self, id: Id) {
- self.entries.insert(id, ContainerState::Pending);
+ pub(crate) fn register(&self, id: Id, session: AgentSessionId) {
+ self.entries.insert(id.clone(), ContainerState::Pending);
+ self.sessions.insert(id, session);
+ }
+
+ /// The session `id` was registered for, if it is still tracked.
+ pub(crate) fn session_of(&self, id: &Id) -> Option<AgentSessionId> {
+ self.sessions.get(id).map(|entry| *entry)
}
pub(crate) fn activate(&self, id: &Id, now: Instant) -> bool {
@@ -50,16 +62,31 @@ where
}
pub(crate) fn remove(&self, id: &Id) -> bool {
+ self.sessions.remove(id);
self.entries.remove(id).is_some()
}
- pub(crate) fn reap_stale(&self, now: Instant, max_idle: Duration) -> Vec<Id> {
+ /// Stop-candidate ids idle at least `max_idle`, excluding any `skip`
+ /// answers true for.
+ ///
+ /// `skip` is checked before a candidate is claimed as stale: a session
+ /// with a command already admitted but not yet turn-active - the harness
+ /// marks this at admission, before the runtime visibly starts a turn -
+ /// looks idle by activity timestamp alone, and reaping it now would only
+ /// race that command's delivery.
+ pub(crate) fn reap_stale(
+ &self,
+ now: Instant,
+ max_idle: Duration,
+ skip: impl Fn(&Id) -> bool,
+ ) -> Vec<Id> {
let candidates = self
.entries
.iter()
.filter_map(|entry| match entry.value() {
ContainerState::Active { last_activity }
- if now.saturating_duration_since(*last_activity) >= max_idle =>
+ if now.saturating_duration_since(*last_activity) >= max_idle
+ && !skip(entry.key()) =>
{
Some(entry.key().clone())
}
@@ -73,6 +100,7 @@ where
};
if let ContainerState::Active { last_activity } = *state
&& now.saturating_duration_since(last_activity) >= max_idle
+ && !skip(&id)
{
*state = ContainerState::Stopping { last_activity };
stale.push(id);
diff --git a/crates/agent_harness/src/outbound/managed_containers/test.rs b/crates/agent_harness/src/outbound/managed_containers/test.rs
index ac780535a1..37e2558ff5 100644
--- a/crates/agent_harness/src/outbound/managed_containers/test.rs
+++ b/crates/agent_harness/src/outbound/managed_containers/test.rs
@@ -1,7 +1,15 @@
use super::*;
+fn session(n: u128) -> AgentSessionId {
+ AgentSessionId::new_from_uuid(macro_uuid::Uuid::from_u128(n))
+}
+
+fn never_skip<Id>(_: &Id) -> bool {
+ false
+}
+
fn activate(managed: &ManagedContainers<&str>, id: &'static str, last_activity: Instant) {
- managed.register(id);
+ managed.register(id, session(1));
assert!(managed.activate(&id, last_activity));
}
@@ -17,7 +25,10 @@ fn reap_stale_returns_only_containers_idle_for_the_limit() {
activate(&managed, "stale", seconds_ago(now, 300));
activate(&managed, "active", seconds_ago(now, 299));
- assert_eq!(managed.reap_stale(now, Duration::from_secs(300)), ["stale"]);
+ assert_eq!(
+ managed.reap_stale(now, Duration::from_secs(300), never_skip),
+ ["stale"]
+ );
}
#[test]
@@ -28,21 +39,29 @@ fn recording_activity_keeps_a_container_alive() {
managed.record_activity(&"active", now);
- assert!(managed.reap_stale(now, Duration::from_secs(300)).is_empty());
+ assert!(
+ managed
+ .reap_stale(now, Duration::from_secs(300), never_skip)
+ .is_empty()
+ );
}
#[test]
fn pending_and_stopping_containers_are_not_reaped_again() {
let managed = ManagedContainers::new();
let now = Instant::now();
- managed.register("pending");
+ managed.register("pending", session(1));
activate(&managed, "stopping", seconds_ago(now, 300));
assert_eq!(
- managed.reap_stale(now, Duration::from_secs(300)),
+ managed.reap_stale(now, Duration::from_secs(300), never_skip),
["stopping"]
);
- assert!(managed.reap_stale(now, Duration::ZERO).is_empty());
+ assert!(
+ managed
+ .reap_stale(now, Duration::ZERO, never_skip)
+ .is_empty()
+ );
}
#[test]
@@ -51,12 +70,34 @@ fn activity_during_a_failed_stop_is_preserved() {
let now = Instant::now();
activate(&managed, "active", seconds_ago(now, 300));
assert_eq!(
- managed.reap_stale(now, Duration::from_secs(300)),
+ managed.reap_stale(now, Duration::from_secs(300), never_skip),
["active"]
);
managed.record_activity(&"active", now);
managed.finish_stop(&"active", false);
- assert!(managed.reap_stale(now, Duration::from_secs(300)).is_empty());
+ assert!(
+ managed
+ .reap_stale(now, Duration::from_secs(300), never_skip)
+ .is_empty()
+ );
+}
+
+#[test]
+fn a_session_with_a_pending_command_is_not_reaped() {
+ let managed = ManagedContainers::new();
+ let now = Instant::now();
+ let pending_session = session(2);
+ managed.register("busy", pending_session);
+ assert!(managed.activate(&"busy", seconds_ago(now, 300)));
+
+ assert!(
+ managed
+ .reap_stale(now, Duration::from_secs(300), |id| managed
+ .session_of(id)
+ .is_some_and(|s| s == pending_session))
+ .is_empty(),
+ "a container whose session has a pending command must not be reaped"
+ );
}
diff --git a/crates/agent_session/src/domain/connection.rs b/crates/agent_session/src/domain/connection.rs
index 35ad521a10..f9321825bf 100644
--- a/crates/agent_session/src/domain/connection.rs
+++ b/crates/agent_session/src/domain/connection.rs
@@ -150,6 +150,12 @@ pub struct RuntimeAttachment<Connector> {
/// fresh at each attach - the set follows what the owner has connected
/// *now*, not what they had connected when the session was created.
pub(crate) mcp_servers: Vec<McpServer>,
+ /// Cancelled by the connector the moment its transport ends, for
+ /// connectors that can tell - most can, via whatever already tears their
+ /// pipe or socket down. `None` for a connector with no such signal;
+ /// callers waiting on a reply then fall back to their own timeout alone,
+ /// exactly as before this existed.
+ pub(crate) closed: Option<CancellationToken>,
}
/// Activate attachment-owned resources with the exact acquired ownership claim.
@@ -168,6 +174,7 @@ impl<Connector> RuntimeAttachment<Connector> {
connector,
handshake,
mcp_servers: Vec::new(),
+ closed: None,
}
}
@@ -178,6 +185,17 @@ impl<Connector> RuntimeAttachment<Connector> {
self
}
+ /// Wire up the connector's own transport-closed signal, when it has one.
+ ///
+ /// Lets a command waiting on this session's reply fail the moment the
+ /// transport is known to be gone, rather than only once its own
+ /// wall-clock timeout separately elapses.
+ #[must_use]
+ pub fn with_closed(mut self, closed: CancellationToken) -> Self {
+ self.closed = Some(closed);
+ self
+ }
+
/// Change the carrier without losing its handshake or activation lifecycle.
pub fn map_transport<T>(self, map: impl FnOnce(Connector) -> T) -> RuntimeAttachment<T> {
RuntimeAttachment {
@@ -185,6 +203,7 @@ impl<Connector> RuntimeAttachment<Connector> {
handshake: self.handshake,
mcp_servers: self.mcp_servers,
activation: self.activation,
+ closed: self.closed,
}
}
@@ -361,6 +380,9 @@ where
// External runtimes hold no egress environment; the sessions
// they serve are not handed proxied MCP servers.
mcp_servers: Vec::new(),
+ // This connection already tracks its own end (`evict`, or the
+ // router task finishing) - the same signal `closed()` awaits.
+ closed: Some(self.closed.clone()),
}
}
diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rs
index 8f682449ee..5cace53e36 100644
--- a/crates/agent_session/src/domain/service.rs
+++ b/crates/agent_session/src/domain/service.rs
@@ -86,6 +86,12 @@ struct ActiveSession {
marker: Arc<()>,
deleting: bool,
stopping: bool,
+ /// Cancelled by the connector the moment its transport ends, when it
+ /// offers one - not every connector does. `deliver_action` races this
+ /// alongside its reply wait so a command sent to a session whose
+ /// transport is already known to be gone fails immediately instead of
+ /// waiting out the full command timeout for nothing.
+ transport_closed: Option<CancellationToken>,
}
type ActiveSessions = DashMap<AgentSessionId, ActiveSession>;
@@ -350,6 +356,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
marker: marker.clone(),
deleting: false,
stopping: false,
+ transport_closed: None,
});
Ok(AttachReservation {
active: self.active.clone(),
@@ -390,6 +397,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
activate(claim)?;
}
active.commands = Some(commands.clone());
+ active.transport_closed = attachment.closed.clone();
drop(active);
let (marker, stopped_tx) = reservation.commit();
@@ -435,10 +443,10 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
action: AgentAction,
action_id: AgentActionId,
) -> Result<()> {
- let commands = self
+ let (commands, transport_closed) = self
.active
.get(&id)
- .and_then(|entry| entry.commands.clone())
+ .and_then(|entry| Some((entry.commands.clone()?, entry.transport_closed.clone())))
.ok_or(AgentSessionError::Disconnected(id))?;
let (completed, result) = oneshot::channel();
@@ -471,10 +479,26 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
// can only notice by hitting its own much longer internal deadline
// instead of promptly.
drop(commands);
- match tokio::time::timeout(HANDSHAKE_TIMEOUT, result).await {
- Ok(Ok(result)) => result,
- Ok(Err(_)) => Err(AgentSessionError::Disconnected(id)),
- Err(_elapsed) => {
+ // A transport that has already ended has no reply coming, however
+ // long we wait: race the connector's own closed signal (when it has
+ // one) alongside the reply so that case fails at once instead of
+ // riding out the rest of `HANDSHAKE_TIMEOUT` for nothing. A
+ // connector with no such signal just never resolves this side of
+ // the select, unchanged from before.
+ let transport_closed_first = async {
+ match &transport_closed {
+ Some(closed) => closed.cancelled().await,
+ None => std::future::pending().await,
+ }
+ };
+ let timed_out = tokio::select! {
+ res = tokio::time::timeout(HANDSHAKE_TIMEOUT, result) => Some(res),
+ () = transport_closed_first => None,
+ };
+ match timed_out {
+ Some(Ok(Ok(result))) => result,
+ Some(Ok(Err(_))) => Err(AgentSessionError::Disconnected(id)),
+ Some(Err(_elapsed)) => {
// The actor is presumably still stuck in the handshake, so it
// is stopped directly - the same "drop the sender, the actor
// notices and tears itself down" mechanism `close_session`
@@ -488,6 +512,18 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
tracing::warn!(%id, "agent session command timed out waiting for the ACP handshake");
Err(AgentSessionError::Disconnected(id))
}
+ None => {
+ // The connector's transport already ended - no wall-clock
+ // wait can help, so stop the actor the same way a timeout
+ // does and fail now.
+ let (stopped, marker) = self.begin_stop(id, false);
+ Self::wait_stopped(stopped).await;
+ self.active.remove_if(&id, |_, active| {
+ Arc::ptr_eq(&active.marker, &marker) && !active.deleting
+ });
+ tracing::info!(%id, "agent session command failed fast: transport already closed");
+ Err(AgentSessionError::Disconnected(id))
+ }
}
}
@@ -516,6 +552,7 @@ impl<R, Folds, Rt, Namer> AgentSessionServiceImpl<R, Folds, Rt, Namer> {
marker: marker.clone(),
deleting,
stopping: true,
+ transport_closed: None,
});
(stopped, marker)
}
diff --git a/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs b/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs
index 8d3408d1d9..05964f7734 100644
--- a/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs
+++ b/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs
@@ -6,6 +6,20 @@ use agent_session::domain::model::{AgentSessionId, ManagerFence, ReplicaId};
use futures::future::BoxFuture;
use sqlx::PgPool;
+/// Postgres' `lock_not_available` SQLSTATE, the error `FOR UPDATE NOWAIT`
+/// raises instead of blocking. <https://www.postgresql.org/docs/current/errcodes-appendix.html>
+const LOCK_NOT_AVAILABLE: &str = "55P03";
+
+/// Whether `error` is exactly the row already being locked by another
+/// transaction - the one outcome `NOWAIT` exists to turn into an immediate
+/// error rather than a wait.
+fn is_lock_not_available(error: &sqlx::Error) -> bool {
+ matches!(
+ error,
+ sqlx::Error::Database(db_err) if db_err.code().as_deref() == Some(LOCK_NOT_AVAILABLE)
+ )
+}
+
/// Bound to exactly one authorized host session and its current management
/// claim. A takeover updates the same locked row and invalidates this writer.
#[derive(Debug)]
@@ -53,12 +67,33 @@ impl PgCursorJournal {
// Dispatch fencing cannot stop an already-running stream or mirror
// poll. Hold the same row takeover updates until the journal commits;
// checking the claim before opening this transaction would race.
+ //
+ // NOWAIT, not a plain FOR UPDATE: a writer whose claim is being
+ // superseded (a resume/reattach bumping this same row's fence) has no
+ // business waiting for that other transaction to commit - it has
+ // already lost, whatever the wait would answer. Blocking here used to
+ // leave a stale writer's read/append hanging until some caller far
+ // above gave up on an unrelated wall-clock timeout (tens of seconds)
+ // and force-tore the session down from outside; failing the lock
+ // immediately turns that into a fencing error on the spot, which is
+ // the caller's cue to reattach right away instead of stalling first.
let expected = *self
.fence
.get()
.ok_or_else(|| rootcause::report!("Cursor journal attachment is not activated"))?;
- let current = sqlx::query_scalar!("SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE", self.session.as_uuid(), self.replica.as_uuid())
- .fetch_optional(&mut **tx).await.map_err(|e| rootcause::report!(e))?
+ let current = sqlx::query_scalar!("SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE NOWAIT", self.session.as_uuid(), self.replica.as_uuid())
+ .fetch_optional(&mut **tx).await.map_err(|e| {
+ if is_lock_not_available(&e) {
+ // Someone else - almost always a superseding claim - holds
+ // this row right now. We would lose that race even if we
+ // waited, so report it exactly like a lost fence.
+ rootcause::report!("Cursor journal writer fenced out")
+ } else {
+ // Every other database failure keeps its type, SQLSTATE
+ // and source chain, as elsewhere in this file.
+ rootcause::report!(e).into_dynamic()
+ }
+ })?
.ok_or_else(|| rootcause::report!("Cursor journal writer fenced out"))?;
if ManagerFence(current) != expected {
return Err(rootcause::report!("Cursor journal writer fenced out"));
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index c54adb4fa7..a592c9f633 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -247,6 +247,14 @@ async fn run() -> anyhow::Result<()> {
replica,
);
+ // Sessions with a command admitted but not yet resolved - shared with
+ // every provider whose idle reaper closes a session's transport on its
+ // own schedule, so a reaper never pulls the transport out from under a
+ // command already on its way in. Built before the container managers
+ // below (which read it) and handed to the harness after them (which
+ // marks and clears it); nothing else needs to know its type.
+ let pending_commands = agent_harness::domain::pending::PendingCommands::new();
+
// Containers: the sandbox provider (local Docker when a developer has
// opted in, Daytona otherwise) plus Cursor cloud agents for the `@cursor`
// bot, routed per session.
@@ -285,12 +293,15 @@ async fn run() -> anyhow::Result<()> {
"DAYTONA_API_KEY is unset: Daytona-backed sandboxes are unarmed; external agent sessions are unaffected"
);
}
- HarnessContainers::Daytona(DaytonaContainerManager::new(DaytonaSettings {
- api_url: config.daytona_api_url.clone(),
- api_key: DaytonaApiKeySecret::new(config.daytona_api_key.clone()),
- snapshot: Snapshot::new(config.daytona_snapshot.clone()),
- anthropic_api_key,
- }))
+ HarnessContainers::Daytona(DaytonaContainerManager::new(
+ DaytonaSettings {
+ api_url: config.daytona_api_url.clone(),
+ api_key: DaytonaApiKeySecret::new(config.daytona_api_key.clone()),
+ snapshot: Snapshot::new(config.daytona_snapshot.clone()),
+ anthropic_api_key,
+ },
+ pending_commands.clone(),
+ ))
};
let container_shutdown = sandbox.clone();
@@ -418,6 +429,7 @@ async fn run() -> anyhow::Result<()> {
session_repo.clone(),
pool.clone(),
replica,
+ pending_commands.clone(),
);
// Fixed system agents retain their deployment defaults. User/team agents
// are resolved from agent_configs for every trigger so newly-created or
@@ -549,6 +561,7 @@ async fn run() -> anyhow::Result<()> {
RedisCommandForwarder::new(redis.clone()),
defaults,
Arc::clone(&lifecycle_publisher),
+ pending_commands,
));
// Close the loop: turn ends observed by the session actors drain the
// harness's prompt queue.