Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(cursor): wait out sibling row locks in the journal instead of fencing (#6346)
Wolf Mermelsteindadec88017b9
diff --git a/.sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json b/.sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.jsonsimilarity index 73%rename from .sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.jsonrename to .sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.jsonindex 2992fe05bf..870cdca190 100644--- a/.sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json+++ b/.sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE NOWAIT",+ "query": "SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE","describe": {"columns": [{@@ -19,5 +19,5 @@false]},- "hash": "48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd"+ "hash": "9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab"}diff --git a/crates/cursor_cloud_agents/src/api.rs b/crates/cursor_cloud_agents/src/api.rsindex 71595ecf64..9e3d6f01bb 100644--- a/crates/cursor_cloud_agents/src/api.rs+++ b/crates/cursor_cloud_agents/src/api.rs@@ -296,6 +296,7 @@ impl CursorClient {}impl CursorAgents for CursorClient {+ #[tracing::instrument(skip(self), err)]async fn raw_result(&self,agent: &CursorAgentId,@@ -478,6 +479,9 @@ const STREAM_CONNECT_ATTEMPTS: usize = 5;const STREAM_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(400);impl RunStream for CursorClient {+ // Covers connecting the stream, retries included; reading it belongs to+ // the caller's `cursor.run.ingest` span.+ #[tracing::instrument(skip(self), err)]async fn raw_stream(&self,agent: &CursorAgentId,diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rsindex b2edda72ef..63c9f6bb50 100644--- a/crates/cursor_cloud_agents/src/domain/service.rs+++ b/crates/cursor_cloud_agents/src/domain/service.rs@@ -910,6 +910,24 @@ where/// One ordered path for live, foreign, and hydration ingestion. Reconnect/// starts at the beginning and verifies the captured content prefix. No/// local sequence is sent to Cursor as a remote resume token.+ ///+ /// One span per ingestion attempt: this is where a turn or a replay spends+ /// its time between the run being listed and its records being journaled,+ /// and the Cursor stream and poll calls inside it are what a stalled+ /// handshake or turn was otherwise waiting on invisibly.+ #[tracing::instrument(+ name = "cursor.run.ingest",+ skip_all,+ fields(+ agent.acp.session_id = ?session_id,+ cursor.agent.id = %agent,+ cursor.run.id = %run,+ cursor.ingest.emit = mode.emit,+ cursor.ingest.strict = mode.strict,+ cursor.ingest.attempt = mode.attempt,+ ),+ err,+ )]async fn ingest_run(&self,session_id: &SessionId,@@ -1511,6 +1529,15 @@ where/// Reconstruct the entire session before allowing a successful load reply./// The returned guard serializes the reply itself with every live writer.+ ///+ /// The runtime-side counterpart of the harness's handshake span: a+ /// `session/load` that times out upstream is spent in here.+ #[tracing::instrument(+ name = "cursor.session.replay",+ skip(self),+ fields(agent.acp.session_id = ?id),+ err,+ )]pub async fn replay_session(&self, id: &SessionId) -> Result<ReplayGuard, SessionError> {let session = self.session(id)?;let gate = Arc::clone(&session.turn_gate).lock_owned().await;diff --git a/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs b/crates/cursor_cloud_agents/src/outbound/postgres_journal.rsindex 05964f7734..2030686067 100644--- a/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs+++ b/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs@@ -5,20 +5,7 @@ use agent_client_protocol::schema::v1::SessionId;use agent_session::domain::model::{AgentSessionId, ManagerFence, ReplicaId};use futures::future::BoxFuture;use sqlx::PgPool;--/// Postgres' `lock_not_available` SQLSTATE, the error `FOR UPDATE NOWAIT`-/// raises instead of blocking. <https://www.postgresql.org/docs/current/errcodes-appendix.html>-const LOCK_NOT_AVAILABLE: &str = "55P03";--/// Whether `error` is exactly the row already being locked by another-/// transaction - the one outcome `NOWAIT` exists to turn into an immediate-/// error rather than a wait.-fn is_lock_not_available(error: &sqlx::Error) -> bool {- matches!(- error,- sqlx::Error::Database(db_err) if db_err.code().as_deref() == Some(LOCK_NOT_AVAILABLE)- )-}+use tracing::Instrument;/// Bound to exactly one authorized host session and its current management/// claim. A takeover updates the same locked row and invalidates this writer.@@ -60,6 +47,10 @@ impl PgCursorJournal {}Ok(())}+ /// Its own span because its duration is the row-lock wait: the one+ /// number that tells a contended row apart from a slow database or a+ /// slow Cursor API when a journal write looks stuck.+ #[tracing::instrument(name = "cursor.journal.lock_owner", skip_all)]async fn lock_owner(&self,tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,@@ -68,32 +59,23 @@ impl PgCursorJournal {// poll. Hold the same row takeover updates until the journal commits;// checking the claim before opening this transaction would race.//- // NOWAIT, not a plain FOR UPDATE: a writer whose claim is being- // superseded (a resume/reattach bumping this same row's fence) has no- // business waiting for that other transaction to commit - it has- // already lost, whatever the wait would answer. Blocking here used to- // leave a stale writer's read/append hanging until some caller far- // above gave up on an unrelated wall-clock timeout (tens of seconds)- // and force-tore the session down from outside; failing the lock- // immediately turns that into a fencing error on the spot, which is- // the caller's cue to reattach right away instead of stalling first.+ // A plain, blocking FOR UPDATE - never NOWAIT. This row is shared+ // with writers that are not takeovers at all: the session actor's+ // fenced log write locks it for every frame it stores, and during a+ // streaming turn that write runs concurrently with the very append+ // that produced the frame. Contention here is therefore routine and+ // says nothing about the claim; only the fence comparison below does.+ // Treating "someone else holds the row" as "fenced out" failed every+ // streaming turn the moment its first text delta was logged. A+ // superseding claim is still detected promptly: its single-statement+ // transaction commits in milliseconds, after which this select+ // re-reads the row and the fence no longer matches.let expected = *self.fence.get().ok_or_else(|| rootcause::report!("Cursor journal attachment is not activated"))?;- let current = sqlx::query_scalar!("SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE NOWAIT", self.session.as_uuid(), self.replica.as_uuid())- .fetch_optional(&mut **tx).await.map_err(|e| {- if is_lock_not_available(&e) {- // Someone else - almost always a superseding claim - holds- // this row right now. We would lose that race even if we- // waited, so report it exactly like a lost fence.- rootcause::report!("Cursor journal writer fenced out")- } else {- // Every other database failure keeps its type, SQLSTATE- // and source chain, as elsewhere in this file.- rootcause::report!(e).into_dynamic()- }- })?+ let current = sqlx::query_scalar!("SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE", self.session.as_uuid(), self.replica.as_uuid())+ .fetch_optional(&mut **tx).await.map_err(|e| rootcause::report!(e))?.ok_or_else(|| rootcause::report!("Cursor journal writer fenced out"))?;if ManagerFence(current) != expected {return Err(rootcause::report!("Cursor journal writer fenced out"));@@ -106,6 +88,10 @@ impl CursorJournal for PgCursorJournal {&'a self,_session: &'a SessionId,) -> BoxFuture<'a, Result<Vec<JournalEntry>, rootcause::Report>> {+ let span = tracing::info_span!(+ "cursor.journal.read",+ agent.session.id = %self.session,+ );Box::pin(async move {// A read is scoped by the bound host identity, never by a caller's// ACP ID, which is only unique within a transport.@@ -124,7 +110,7 @@ impl CursorJournal for PgCursorJournal {})}).collect()- })+ }.instrument(span))}fn append<'a>(&'a self,@@ -133,6 +119,12 @@ impl CursorJournal for PgCursorJournal {run: Option<&'a CursorRunId>,input: &'a JournalInput,) -> BoxFuture<'a, Result<JournalEntry, rootcause::Report>> {+ let span = tracing::info_span!(+ "cursor.journal.append",+ agent.session.id = %self.session,+ cursor.run.id = run.map(tracing::field::display),+ cursor.journal.expected_sequence = expected,+ );Box::pin(async move {let mut tx = self.pool.begin().await.map_err(|e| rootcause::report!(e))?;self.lock_owner(&mut tx).await?;@@ -154,7 +146,7 @@ impl CursorJournal for PgCursorJournal {run: run.cloned(),input: input.clone(),})- })+ }.instrument(span))}}