Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(cursor): wait out sibling row locks in the journal instead of fencing (#6346)

Wolf Mermelsteindadec88017b9
diff --git a/.sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json b/.sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json
similarity index 73%
rename from .sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json
rename to .sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json
index 2992fe05bf..870cdca190 100644
--- a/.sqlx/query-48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd.json
+++ b/.sqlx/query-9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE NOWAIT",
+ "query": "SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
false
]
},
- "hash": "48d7470b990204f9ca6bc0340a375a6915c5d72ae0ffc8f11b9716d2b41f97fd"
+ "hash": "9e595b74206ab19e5d5fe85e7e730254799266365a092d62e246c0eddf6a63ab"
}
diff --git a/crates/cursor_cloud_agents/src/api.rs b/crates/cursor_cloud_agents/src/api.rs
index 71595ecf64..9e3d6f01bb 100644
--- a/crates/cursor_cloud_agents/src/api.rs
+++ b/crates/cursor_cloud_agents/src/api.rs
@@ -296,6 +296,7 @@ impl CursorClient {
}
impl CursorAgents for CursorClient {
+ #[tracing::instrument(skip(self), err)]
async fn raw_result(
&self,
agent: &CursorAgentId,
@@ -478,6 +479,9 @@ const STREAM_CONNECT_ATTEMPTS: usize = 5;
const STREAM_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(400);
impl RunStream for CursorClient {
+ // Covers connecting the stream, retries included; reading it belongs to
+ // the caller's `cursor.run.ingest` span.
+ #[tracing::instrument(skip(self), err)]
async fn raw_stream(
&self,
agent: &CursorAgentId,
diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rs
index b2edda72ef..63c9f6bb50 100644
--- a/crates/cursor_cloud_agents/src/domain/service.rs
+++ b/crates/cursor_cloud_agents/src/domain/service.rs
@@ -910,6 +910,24 @@ where
/// One ordered path for live, foreign, and hydration ingestion. Reconnect
/// starts at the beginning and verifies the captured content prefix. No
/// local sequence is sent to Cursor as a remote resume token.
+ ///
+ /// One span per ingestion attempt: this is where a turn or a replay spends
+ /// its time between the run being listed and its records being journaled,
+ /// and the Cursor stream and poll calls inside it are what a stalled
+ /// handshake or turn was otherwise waiting on invisibly.
+ #[tracing::instrument(
+ name = "cursor.run.ingest",
+ skip_all,
+ fields(
+ agent.acp.session_id = ?session_id,
+ cursor.agent.id = %agent,
+ cursor.run.id = %run,
+ cursor.ingest.emit = mode.emit,
+ cursor.ingest.strict = mode.strict,
+ cursor.ingest.attempt = mode.attempt,
+ ),
+ err,
+ )]
async fn ingest_run(
&self,
session_id: &SessionId,
@@ -1511,6 +1529,15 @@ where
/// Reconstruct the entire session before allowing a successful load reply.
/// The returned guard serializes the reply itself with every live writer.
+ ///
+ /// The runtime-side counterpart of the harness's handshake span: a
+ /// `session/load` that times out upstream is spent in here.
+ #[tracing::instrument(
+ name = "cursor.session.replay",
+ skip(self),
+ fields(agent.acp.session_id = ?id),
+ err,
+ )]
pub async fn replay_session(&self, id: &SessionId) -> Result<ReplayGuard, SessionError> {
let session = self.session(id)?;
let gate = Arc::clone(&session.turn_gate).lock_owned().await;
diff --git a/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs b/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs
index 05964f7734..2030686067 100644
--- a/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs
+++ b/crates/cursor_cloud_agents/src/outbound/postgres_journal.rs
@@ -5,20 +5,7 @@ use agent_client_protocol::schema::v1::SessionId;
use agent_session::domain::model::{AgentSessionId, ManagerFence, ReplicaId};
use futures::future::BoxFuture;
use sqlx::PgPool;
-
-/// Postgres' `lock_not_available` SQLSTATE, the error `FOR UPDATE NOWAIT`
-/// raises instead of blocking. <https://www.postgresql.org/docs/current/errcodes-appendix.html>
-const LOCK_NOT_AVAILABLE: &str = "55P03";
-
-/// Whether `error` is exactly the row already being locked by another
-/// transaction - the one outcome `NOWAIT` exists to turn into an immediate
-/// error rather than a wait.
-fn is_lock_not_available(error: &sqlx::Error) -> bool {
- matches!(
- error,
- sqlx::Error::Database(db_err) if db_err.code().as_deref() == Some(LOCK_NOT_AVAILABLE)
- )
-}
+use tracing::Instrument;
/// Bound to exactly one authorized host session and its current management
/// claim. A takeover updates the same locked row and invalidates this writer.
@@ -60,6 +47,10 @@ impl PgCursorJournal {
}
Ok(())
}
+ /// Its own span because its duration is the row-lock wait: the one
+ /// number that tells a contended row apart from a slow database or a
+ /// slow Cursor API when a journal write looks stuck.
+ #[tracing::instrument(name = "cursor.journal.lock_owner", skip_all)]
async fn lock_owner(
&self,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
@@ -68,32 +59,23 @@ impl PgCursorJournal {
// poll. Hold the same row takeover updates until the journal commits;
// checking the claim before opening this transaction would race.
//
- // NOWAIT, not a plain FOR UPDATE: a writer whose claim is being
- // superseded (a resume/reattach bumping this same row's fence) has no
- // business waiting for that other transaction to commit - it has
- // already lost, whatever the wait would answer. Blocking here used to
- // leave a stale writer's read/append hanging until some caller far
- // above gave up on an unrelated wall-clock timeout (tens of seconds)
- // and force-tore the session down from outside; failing the lock
- // immediately turns that into a fencing error on the spot, which is
- // the caller's cue to reattach right away instead of stalling first.
+ // A plain, blocking FOR UPDATE - never NOWAIT. This row is shared
+ // with writers that are not takeovers at all: the session actor's
+ // fenced log write locks it for every frame it stores, and during a
+ // streaming turn that write runs concurrently with the very append
+ // that produced the frame. Contention here is therefore routine and
+ // says nothing about the claim; only the fence comparison below does.
+ // Treating "someone else holds the row" as "fenced out" failed every
+ // streaming turn the moment its first text delta was logged. A
+ // superseding claim is still detected promptly: its single-statement
+ // transaction commits in milliseconds, after which this select
+ // re-reads the row and the fence no longer matches.
let expected = *self
.fence
.get()
.ok_or_else(|| rootcause::report!("Cursor journal attachment is not activated"))?;
- let current = sqlx::query_scalar!("SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE NOWAIT", self.session.as_uuid(), self.replica.as_uuid())
- .fetch_optional(&mut **tx).await.map_err(|e| {
- if is_lock_not_available(&e) {
- // Someone else - almost always a superseding claim - holds
- // this row right now. We would lose that race even if we
- // waited, so report it exactly like a lost fence.
- rootcause::report!("Cursor journal writer fenced out")
- } else {
- // Every other database failure keeps its type, SQLSTATE
- // and source chain, as elsewhere in this file.
- rootcause::report!(e).into_dynamic()
- }
- })?
+ let current = sqlx::query_scalar!("SELECT manager_fence FROM agent_session WHERE id = $1 AND manager_replica_id = $2 FOR UPDATE", self.session.as_uuid(), self.replica.as_uuid())
+ .fetch_optional(&mut **tx).await.map_err(|e| rootcause::report!(e))?
.ok_or_else(|| rootcause::report!("Cursor journal writer fenced out"))?;
if ManagerFence(current) != expected {
return Err(rootcause::report!("Cursor journal writer fenced out"));
@@ -106,6 +88,10 @@ impl CursorJournal for PgCursorJournal {
&'a self,
_session: &'a SessionId,
) -> BoxFuture<'a, Result<Vec<JournalEntry>, rootcause::Report>> {
+ let span = tracing::info_span!(
+ "cursor.journal.read",
+ agent.session.id = %self.session,
+ );
Box::pin(async move {
// A read is scoped by the bound host identity, never by a caller's
// ACP ID, which is only unique within a transport.
@@ -124,7 +110,7 @@ impl CursorJournal for PgCursorJournal {
})
})
.collect()
- })
+ }.instrument(span))
}
fn append<'a>(
&'a self,
@@ -133,6 +119,12 @@ impl CursorJournal for PgCursorJournal {
run: Option<&'a CursorRunId>,
input: &'a JournalInput,
) -> BoxFuture<'a, Result<JournalEntry, rootcause::Report>> {
+ let span = tracing::info_span!(
+ "cursor.journal.append",
+ agent.session.id = %self.session,
+ cursor.run.id = run.map(tracing::field::display),
+ cursor.journal.expected_sequence = expected,
+ );
Box::pin(async move {
let mut tx = self.pool.begin().await.map_err(|e| rootcause::report!(e))?;
self.lock_owner(&mut tx).await?;
@@ -154,7 +146,7 @@ impl CursorJournal for PgCursorJournal {
run: run.cloned(),
input: input.clone(),
})
- })
+ }.instrument(span))
}
}