Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(cursor): resume a dropped run stream instead of polling out the run (#6459)
Wolf Mermelstein39591d759b54
diff --git a/crates/agent_harness/src/outbound/cursor/manager.rs b/crates/agent_harness/src/outbound/cursor/manager.rsindex 4eb3b3a54b..2de5a33844 100644--- a/crates/agent_harness/src/outbound/cursor/manager.rs+++ b/crates/agent_harness/src/outbound/cursor/manager.rs@@ -702,15 +702,18 @@ where&self,agent: &CursorAgentId,run: &CursorRunId,+ resume_from: Option<&str>,) -> std::result::Result<- impl Stream<- Item = std::result::Result<- cursor_cloud_agents::domain::journal::NativeRecord,- rootcause::Report,- >,- > + Send,- rootcause::Report,+ cursor_cloud_agents::domain::ports::ConnectedStream<+ impl Stream<+ Item = std::result::Result<+ cursor_cloud_agents::domain::journal::NativeRecord,+ rootcause::Report,+ >,+ > + Send,+ >,+ cursor_cloud_agents::domain::ports::StreamConnectError,> {- self.client.raw_stream(agent, run).await+ self.client.raw_stream(agent, run, resume_from).await}}diff --git a/crates/cursor_cloud_agents/src/api.rs b/crates/cursor_cloud_agents/src/api.rsindex 3b0f75e189..0ff440d3c3 100644--- a/crates/cursor_cloud_agents/src/api.rs+++ b/crates/cursor_cloud_agents/src/api.rs@@ -30,7 +30,7 @@ use crate::domain::model::{CursorAgentId, CursorModel, CursorRunId, McpServer, ModelChoice, ModelParam, ModelVariant,RepoUrl, RunListing,};-use crate::domain::ports::{CursorAgents, RunStream};+use crate::domain::ports::{ConnectedStream, CursorAgents, RunStream, StreamConnectError};use futures::{Stream, StreamExt as _};use sse_core::SseEvent;use std::collections::VecDeque;@@ -549,73 +549,93 @@ const STREAM_CONNECT_ATTEMPTS: usize = 5;/// the second or so between a run's creation and its stream existing.const STREAM_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(400);+/// Cursor's header naming how long a dropped stream stays resumable.+const RETENTION_HEADER: &str = "x-cursor-stream-retention-seconds";++/// The SSE resume header. Not in `http`'s constant set, so it is spelled here.+const LAST_EVENT_ID_HEADER: &str = "last-event-id";+impl RunStream for CursorClient {// Covers connecting the stream, retries included; reading it belongs to// the caller's `cursor.run.ingest` span.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self), fields(cursor.stream.resumed = resume_from.is_some()))]async fn raw_stream(&self,agent: &CursorAgentId,run: &CursorRunId,+ resume_from: Option<&str>,) -> Result<- impl Stream<Item = Result<crate::domain::journal::NativeRecord, rootcause::Report>> + Send,- rootcause::Report,+ ConnectedStream<+ impl Stream<Item = Result<crate::domain::journal::NativeRecord, rootcause::Report>> + Send,+ >,+ StreamConnectError,> {- // Only retry failed HTTP connects. Successful SSE records, including- // stream_unavailable, must reach the domain journal before inspection.+ // Only retry an unavailable stream. Successful SSE records, including+ // a `stream_unavailable` record, must reach the domain journal before+ // inspection, and a rejected or expired resume position is the+ // domain's decision to make, not a thing to sit and retry.for attempt in 1..=STREAM_CONNECT_ATTEMPTS {- match self.connect_stream(agent, run).await {+ match self.connect_stream(agent, run, resume_from).await {Ok(stream) => return Ok(stream),Err(StreamConnectError::Unavailable(_)) if attempt < STREAM_CONNECT_ATTEMPTS => {tokio::time::sleep(STREAM_RETRY_DELAY).await}- Err(StreamConnectError::Unavailable(message)) => {- return Err(rootcause::report!("Cursor stream unavailable: {message}"));- }- Err(StreamConnectError::Other(error)) => return Err(error),+ Err(error) => return Err(error),}}unreachable!("connect attempts are nonzero")}}-/// Why one stream connect did not produce a stream: the endpoint saying the-/// stream is not there (retryable — it appears seconds after run creation),-/// or anything else (not).-enum StreamConnectError {- /// `stream_unavailable`, as an HTTP status. Carries the server's message.- Unavailable(String),- /// Every other failure.- Other(rootcause::Report),-}-impl CursorClient {async fn connect_stream(&self,agent: &CursorAgentId,run: &CursorRunId,+ resume_from: Option<&str>,) -> Result<- impl Stream<Item = Result<crate::domain::journal::NativeRecord, rootcause::Report>>- + Send- + use<>,+ ConnectedStream<+ impl Stream<Item = Result<crate::domain::journal::NativeRecord, rootcause::Report>>+ + Send+ + use<>,+ >,StreamConnectError,> {- let response = self+ let mut request = self.http.get(self.url(&format!("/v1/agents/{agent}/runs/{run}/stream"))).basic_auth(self.config.api_key.expose(), Some(""))- .header(reqwest::header::ACCEPT, "text/event-stream")+ .header(reqwest::header::ACCEPT, "text/event-stream");+ if let Some(resume_from) = resume_from {+ request = request.header(LAST_EVENT_ID_HEADER, resume_from);+ }+ let response = request.send().await.map_err(|error| StreamConnectError::Other(rootcause::report!(error).into()))?;let status = response.status();+ // Read before the body is consumed; a resumed stream's window is the+ // only thing that says whether the next drop is recoverable at all.+ let retention_seconds = response+ .headers()+ .get(RETENTION_HEADER)+ .and_then(|value| value.to_str().ok())+ .and_then(|value| value.parse().ok());if !status.is_success() {let text = response.text().await.unwrap_or_default();+ let detail = format!("{status}: {text}");if text.contains("stream_unavailable") {- return Err(StreamConnectError::Unavailable(format!("{status}: {text}")));+ return Err(StreamConnectError::Unavailable(detail));+ }+ if status == reqwest::StatusCode::BAD_REQUEST && text.contains("invalid_last_event_id")+ {+ return Err(StreamConnectError::InvalidResumePosition(detail));+ }+ if status == reqwest::StatusCode::GONE || text.contains("stream_expired") {+ return Err(StreamConnectError::Expired(detail));}return Err(StreamConnectError::Other(rootcause::report!(- "cursor stream -> {status}: {text}"+ "cursor stream -> {detail}")));}@@ -636,7 +656,7 @@ impl CursorClient {VecDeque::new(),recording,);- Ok(futures::stream::try_unfold(+ let records = futures::stream::try_unfold(state,|(mut bytes, mut decoder, mut pending, mut recording)| async move {loop {@@ -665,7 +685,7 @@ impl CursorClient {}};let SseEvent::Message(message) = record else {- continue; // `retry:`; nothing reconnects yet+ continue; // `retry:`; the domain drives reconnects};pending.push_back(Ok(crate::domain::journal::NativeRecord {event: message.event.into_owned(),@@ -681,6 +701,10 @@ impl CursorClient {}}},- ))+ );+ Ok(ConnectedStream {+ records,+ retention_seconds,+ })}}diff --git a/crates/cursor_cloud_agents/src/domain/journal.rs b/crates/cursor_cloud_agents/src/domain/journal.rsindex 5c3133ce47..6e5433aa2d 100644--- a/crates/cursor_cloud_agents/src/domain/journal.rs+++ b/crates/cursor_cloud_agents/src/domain/journal.rs@@ -48,6 +48,24 @@ pub enum JournalInput {PromptAborted(i64),/// Transport failure, retained without prematurely closing running tools.TransportError(String),+ /// The run's stream broke or went silent and a reconnect was attempted.+ ///+ /// Journaled rather than merely logged because the journal is the durable+ /// record of what the transport did: a transcript with a gap in it is only+ /// explicable if the gap's cause is in the same ordered record as the+ /// content around it, and a later replay of the session has no other way+ /// to know a reconnect happened here. Like+ /// [`JournalInput::TransportError`] it projects to nothing — it is a fact+ /// about the connection, never about the conversation.+ StreamInterrupted {+ /// What broke, in the transport's own words.+ reason: String,+ /// The last provider event id captured before the break, if any; what+ /// the reconnect resumed from.+ last_event_id: Option<String>,+ /// Which reconnect attempt this interruption started, from one.+ attempt: u32,+ },/// Raw complete provider message, including unknown payloads.Sse(NativeRecord),/// Original successful polling response body.@@ -201,7 +219,8 @@ impl ReplayMachine {| JournalInput::Reconciled| JournalInput::PromptAccepted(_)| JournalInput::PromptAborted(_)- | JournalInput::TransportError(_) => Ok(Vec::new()),+ | JournalInput::TransportError(_)+ | JournalInput::StreamInterrupted { .. } => Ok(Vec::new()),}}fn event(diff --git a/crates/cursor_cloud_agents/src/domain/journal/test.rs b/crates/cursor_cloud_agents/src/domain/journal/test.rsindex c30810b9ee..9f3cb40c93 100644--- a/crates/cursor_cloud_agents/src/domain/journal/test.rs+++ b/crates/cursor_cloud_agents/src/domain/journal/test.rs@@ -363,3 +363,39 @@ fn divergent_terminal_text_keeps_the_captured_answer_and_the_terminal_facts() {);}}++/// A stream interruption is a fact about the connection, not the conversation:+/// it is durable so a gap in a transcript is explicable, and it must add+/// nothing to the transcript it explains — on capture or on any later replay.+#[test]+fn a_stream_interruption_projects_to_nothing() {+ let run = CursorRunId::new("run");+ let interruption = JournalInput::StreamInterrupted {+ reason: "error decoding response body".into(),+ last_event_id: Some("1713033006000-0".into()),+ attempt: 2,+ };+ let mut machine = ReplayMachine::default();+ assert!(+ machine+ .push(+ Some(&run),+ &JournalInput::Sse(crate::testing::raw_record(CursorEvent::Assistant {+ text: "before".into()+ }))+ )+ .unwrap()+ .len()+ == 1+ );+ assert!(machine.push(Some(&run), &interruption).unwrap().is_empty());+ assert!(machine.push(None, &interruption).unwrap().is_empty());+ assert!(!machine.complete(&run), "no outcome was invented either");+ // Durability is the point, so the variant has to survive the journal's+ // own encoding unchanged.+ let encoded = serde_json::to_string(&interruption).unwrap();+ assert_eq!(+ serde_json::from_str::<JournalInput>(&encoded).unwrap(),+ interruption+ );+}diff --git a/crates/cursor_cloud_agents/src/domain/ports.rs b/crates/cursor_cloud_agents/src/domain/ports.rsindex a6b2192a62..e51033933b 100644--- a/crates/cursor_cloud_agents/src/domain/ports.rs+++ b/crates/cursor_cloud_agents/src/domain/ports.rs@@ -92,6 +92,51 @@ pub trait CursorAgents: Sync {) -> impl Future<Output = Result<Vec<RunListing>, rootcause::Report>> + Send;}+/// One connected run stream, with what the provider said about resuming it.+pub struct ConnectedStream<Records> {+ /// Complete native records, captured before decoding or translation.+ pub records: Records,+ /// The provider's `X-Cursor-Stream-Retention-Seconds`, when it sent one.+ ///+ /// How long a dropped stream stays resumable. Not enforced here — the+ /// provider answers an expired resume with+ /// [`StreamConnectError::Expired`] — but worth logging, because a run+ /// whose tool call outlives the window can never be resumed and the+ /// number is the only warning of that.+ pub retention_seconds: Option<u64>,+}++/// Why one connect did not produce a stream.+///+/// The domain acts differently on each: an unavailable stream is retried, an+/// invalid resume position is retried without one, and an expired stream is+/// gone for good and leaves polling as the only way to learn the outcome.+#[derive(Debug)]+pub enum StreamConnectError {+ /// `stream_unavailable`: the stream is not there (yet). Carries the+ /// provider's message.+ Unavailable(String),+ /// `invalid_last_event_id`: the resume position is not this run's.+ InvalidResumePosition(String),+ /// `stream_expired`: the retention window closed behind us.+ Expired(String),+ /// Every other failure.+ Other(rootcause::Report),+}++impl std::fmt::Display for StreamConnectError {+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::Unavailable(message) => write!(formatter, "Cursor stream unavailable: {message}"),+ Self::InvalidResumePosition(message) => {+ write!(formatter, "Cursor rejected the resume position: {message}")+ }+ Self::Expired(message) => write!(formatter, "Cursor stream expired: {message}"),+ Self::Other(report) => write!(formatter, "{report}"),+ }+ }+}+/// Observe a run as a stream of native SSE records.////// The stream ends when the server closes it — normally just after a@@ -99,15 +144,23 @@ pub trait CursorAgents: Sync {/// [`CursorEvent::Result`](super::event::CursorEvent::Result) must treat the run's outcome as unknown rather/// than successful.pub trait RunStream: Sync {- /// Complete native records, captured before decoding or translation.+ /// Connect a run's stream, optionally resuming after an event id.+ ///+ /// `resume_from` is an id observed on an earlier record of this same run,+ /// passed back verbatim: the provider's ids are opaque and a consumer that+ /// parses or invents one gets [`StreamConnectError::InvalidResumePosition`].+ /// `None` connects from the beginning of what the provider still retains.fn raw_stream(&self,agent: &CursorAgentId,run: &CursorRunId,+ resume_from: Option<&str>,) -> impl Future<Output = Result<- impl Stream<Item = Result<super::journal::NativeRecord, rootcause::Report>> + Send,- rootcause::Report,+ ConnectedStream<+ impl Stream<Item = Result<super::journal::NativeRecord, rootcause::Report>> + Send,+ >,+ StreamConnectError,>,> + Send;}diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rsindex ce239e3377..475ef99134 100644--- a/crates/cursor_cloud_agents/src/domain/service.rs+++ b/crates/cursor_cloud_agents/src/domain/service.rs@@ -44,7 +44,9 @@ use crate::domain::journal::{CursorJournal, JournalEntry, JournalInput, ReplayMause crate::domain::model::{CursorAgentId, CursorModel, CursorRunId, McpServer, ModelChoice, RepoUrl, RunStatus,};-use crate::domain::ports::{CursorAgents, RepositoryChooser, RunStream, SessionNotifier};+use crate::domain::ports::{+ CursorAgents, RepositoryChooser, RunStream, SessionNotifier, StreamConnectError,+};use agent_client_protocol::schema::v1::{ContentBlock, SessionId, SessionUpdate, StopReason, TextContent,};@@ -73,6 +75,27 @@ const POLL_ERROR_TOLERANCE: usize = 5;/// interval.const STREAM_QUIET_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);+/// Backoff before each reconnect of an interrupted stream.+///+/// Observed twice in production: the SSE body fails mid-run during a long tool+/// call while the run itself carries on fine, and everything Cursor says+/// between the drop and the run's end — tool results, prose, the images it+/// writes into prose — never reaches the transcript. Resuming from the last+/// event id gets that back; the ramp keeps a provider that is actually down+/// from being hammered.+const STREAM_RECONNECT_DELAYS: [std::time::Duration; 5] = [+ std::time::Duration::from_millis(500),+ std::time::Duration::from_secs(1),+ std::time::Duration::from_secs(2),+ std::time::Duration::from_secs(4),+ std::time::Duration::from_secs(8),+];++/// Reconnects before a run gives up on its stream and polls instead. Counted+/// since the last reconnect that delivered content, so a long run that drops+/// repeatedly but recovers each time is never rationed.+const STREAM_RECONNECT_ATTEMPTS: usize = 5;+/// How long a prompt waits behind a run something else started (the same/// agent is drivable from cursor.com) before giving up, in poll intervals.const BUSY_ATTEMPTS: usize = 450;@@ -112,6 +135,30 @@ async fn sleep_unless_cancelled(}}+/// The run's content records already durable in this session's journal.+///+/// What a stream read from the beginning is reconciled against: every record+/// here must arrive again, in order, before anything new is appended. Recomputed+/// rather than captured once, because a reconnect that restarts the stream has+/// to account for what the interrupted connection already appended.+fn captured_content(+ session: &Session,+ run: &CursorRunId,+) -> Vec<crate::domain::journal::NativeRecord> {+ session+ .state+ .lock()+ .expect("session state poisoned")+ .journal_entries+ .iter()+ .filter(|entry| entry.run.as_ref() == Some(run))+ .filter_map(|entry| match &entry.input {+ JournalInput::Sse(record) if record.is_content() => Some(record.clone()),+ _ => None,+ })+ .collect()+}+/// Restate a repository rejection as something the person who prompted can/// act on, leaving every other failure exactly as it arrived.///@@ -1008,6 +1055,8 @@ wherecursor.ingest.emit = mode.emit,cursor.ingest.strict = mode.strict,cursor.ingest.attempt = mode.attempt,+ cursor.stream.reconnects = tracing::field::Empty,+ cursor.stream.fell_back_to_poll = tracing::field::Empty,),err,)]@@ -1062,134 +1111,280 @@ wherestatus => Err(rootcause::report!("cursor run {run} ended in {status:?}").into()),};}- let captured: Vec<_> = session- .state- .lock()- .expect("session state poisoned")- .journal_entries- .iter()- .filter(|e| e.run.as_ref() == Some(run))- .filter_map(|e| match &e.input {- JournalInput::Sse(r) if r.is_content() => Some(r.clone()),- _ => None,- })- .collect();+ let mut captured = captured_content(session, run);let mut matched = 0;let mut terminal = None;let mut saw_content = false;- let stream = match self.cursor.raw_stream(agent, run).await {- Ok(stream) => Some(stream),- Err(error) => {- self.capture(- session_id,- session,- Some(run),- JournalInput::TransportError(error.to_string()),- emit,- )- .await?;- None- }- };- if let Some(stream) = stream {- pin_mut!(stream);- loop {- let record = match tokio::time::timeout(STREAM_QUIET_TIMEOUT, stream.next()).await {- Ok(Some(Ok(record))) => record,- Ok(Some(Err(error))) => {+ // The provider's most recent event id. Opaque: it is handed back+ // verbatim as a resume position or not used at all.+ let mut last_event_id: Option<String> = None;+ // Reconnects since the last one that produced new records.+ let mut reconnects: u32 = 0;+ let mut fell_back_to_poll = false;+ let mut ever_connected = false;+ let mut resume_from: Option<String> = None;+ // Cursor rejecting a resume position buys exactly one connect without+ // one; a second rejection means something is wrong with the run, not+ // with the id we sent.+ let mut blind_reconnect_available = true;+ // The run's latest `status` frame, kept to recognize the sticky copy+ // Cursor re-sends at the top of every reconnect.+ let mut last_status: Option<crate::domain::journal::NativeRecord> = None;+ loop {+ // The connected stream borrows the position it resumed from, so+ // the position this iteration sends is its own owned copy.+ let resuming = resume_from.clone();+ let interruption = match self+ .cursor+ .raw_stream(agent, run, resuming.as_deref())+ .await+ {+ Err(StreamConnectError::InvalidResumePosition(detail))+ if blind_reconnect_available && resuming.is_some() =>+ {+ // Reading the run from the top still beats polling: the+ // prefix logic below matches what is already captured+ // instead of duplicating it. It has to be recomputed+ // first, because this ingestion has appended to it.+ blind_reconnect_available = false;+ tracing::warn!(+ cursor.run.id = %run,+ cursor.stream.last_event_id = resuming,+ cursor.stream.reason = %detail,+ "Cursor rejected the stream resume position; reconnecting from the start"+ );+ resume_from = None;+ captured = captured_content(session, run);+ matched = 0;+ continue;+ }+ // Past the retention window there is no stream left to resume,+ // so the run record is the only remaining account of the run.+ Err(StreamConnectError::Expired(detail)) => Some((detail, false)),+ Err(error) => Some((error.to_string(), true)),+ Ok(connected) => {+ let resumed = resuming.is_some();+ ever_connected = true;+ let retention_seconds = connected.retention_seconds;+ let records = connected.records;+ pin_mut!(records);+ let mut received = 0usize;+ let mut received_content = 0usize;+ let mut sticky_status_pending = resumed;+ let interruption = loop {+ let record = match tokio::time::timeout(+ STREAM_QUIET_TIMEOUT,+ records.next(),+ )+ .await+ {+ Ok(Some(Ok(record))) => record,+ Ok(Some(Err(error))) => break Some((error.to_string(), true)),+ // Cursor closes the stream after `done`, which+ // breaks below with an outcome. Reaching here+ // without one is a close mid-run.+ Ok(None) if terminal.is_some() => break None,+ Ok(None) => break Some(("stream closed".to_owned(), true)),+ Err(_) if strict => break None,+ Err(_) => {+ // Quiet streams are checked through the exact same raw+ // polling/capture path as disconnected streams.+ let status = self+ .poll_once(session_id, session, agent, run, cancel, emit)+ .await?;+ if status.is_terminal() {+ terminal = Some(status.status);+ break None;+ }+ if cancel.is_cancelled() {+ break None;+ }+ // Cursor heartbeats an open stream, so a+ // gap this long on a run that has not+ // ended is a connection that stopped+ // delivering rather than an agent thinking.+ break Some((+ format!(+ "no records for {} seconds",+ STREAM_QUIET_TIMEOUT.as_secs()+ ),+ true,+ ));+ }+ };+ if let Some(id) = &record.id {+ last_event_id = Some(id.clone());+ }+ // Cursor re-sends the run's `status` frame, without an+ // id, at the top of every reconnect. It is framing, not+ // a new fact: journaling it again would put the same+ // lifecycle event in the record twice.+ if std::mem::take(&mut sticky_status_pending)+ && record.id.is_none()+ && last_status.as_ref() == Some(&record)+ {+ continue;+ }+ if record.event == "status" {+ last_status = Some(record.clone());+ }+ received += 1;+ let content = record.is_content();+ if content && matched < captured.len() {+ if captured[matched] != record {+ return Err(rootcause::report!("Cursor stream prefix cannot be reconciled for {run}; refusing incomplete history").into());+ }+ matched += 1;+ if let CursorEvent::Result { status, .. } = record.decode() {+ terminal = Some(status);+ }+ continue;+ }self.capture(session_id,session,Some(run),- JournalInput::TransportError(error.to_string()),+ JournalInput::Sse(record.clone()),emit,).await?;- break;- }- Ok(None) => break,- Err(_) if strict => break,- Err(_) => {- // Quiet streams are checked through the exact same raw- // polling/capture path as disconnected streams.- let status = self- .poll_once(session_id, session, agent, run, cancel, emit)- .await?;- if status.is_terminal() {- terminal = Some(status.status);- break;+ saw_content |= content;+ received_content += usize::from(content);+ match record.decode() {+ CursorEvent::Error { code, .. }+ if code.as_deref() == Some("stream_unavailable")+ && !saw_content+ && attempt < 4 =>+ {+ if !sleep_unless_cancelled(+ cancel,+ std::time::Duration::from_millis(400),+ )+ .await+ {+ return Box::pin(self.ingest_run(+ session_id,+ session,+ agent,+ run,+ cancel,+ IngestMode {+ attempt: attempt + 1,+ ..mode+ },+ ))+ .await;+ }+ break None;+ }+ CursorEvent::Result { status, .. } => terminal = Some(status),+ // The stream's own account of the run's lifecycle, and a+ // terminal one is a terminal fact even with no `result`+ // frame behind it. Observed live: a run whose record+ // stayed `RUNNING` with a frozen `updatedAt` announced+ // `FINISHED` here and sent no `result` at all, so a turn+ // that accepted only `result` waited out its whole poll+ // budget against a record that was never going to move.+ //+ // Not a break: trailing content can still be in flight,+ // and `done`, the stream's end, or a quiet gap closes the+ // turn now that there is an outcome to close it with.+ CursorEvent::Status { status, .. } if status.is_terminal() => {+ terminal = Some(status);+ }+ // Cursor said its piece about this run; a fresh+ // connection would only be told the same thing.+ CursorEvent::Error { .. } => break None,+ CursorEvent::Done => break None,+ _ => {}}- if cancel.is_cancelled() {- break;+ };+ if resumed {+ tracing::info!(+ cursor.run.id = %run,+ cursor.stream.attempt = reconnects,+ cursor.stream.last_event_id = last_event_id,+ cursor.stream.retention_seconds = retention_seconds,+ cursor.stream.resumed_records = received,+ "Cursor stream resumed"+ );+ // A resume that delivered is a working stream again,+ // so the next drop gets the full budget rather than+ // whatever a much earlier one left over.+ if received_content > 0 {+ reconnects = 0;}- continue;- }- };- let content = record.is_content();- if content && matched < captured.len() {- if captured[matched] != record {- return Err(rootcause::report!("Cursor stream prefix cannot be reconciled for {run}; refusing incomplete history").into());}- matched += 1;- if let CursorEvent::Result { status, .. } = record.decode() {- terminal = Some(status);- }- continue;+ interruption}+ };+ let Some((reason, recoverable)) = interruption else {+ break;+ };+ fell_back_to_poll = true;+ if !ever_connected {+ // Nothing to resume: the stream never opened, and the client+ // has already retried an unavailable one on its own.self.capture(session_id,session,Some(run),- JournalInput::Sse(record.clone()),+ JournalInput::TransportError(reason.clone()),emit,).await?;- saw_content |= content;- match record.decode() {- CursorEvent::Error { code, .. }- if code.as_deref() == Some("stream_unavailable")- && !saw_content- && attempt < 4 =>- {- if !sleep_unless_cancelled(cancel, std::time::Duration::from_millis(400))- .await- {- return Box::pin(self.ingest_run(- session_id,- session,- agent,- run,- cancel,- IngestMode {- attempt: attempt + 1,- ..mode- },- ))- .await;- }- break;- }- CursorEvent::Result { status, .. } => terminal = Some(status),- // The stream's own account of the run's lifecycle, and a- // terminal one is a terminal fact even with no `result`- // frame behind it. Observed live: a run whose record- // stayed `RUNNING` with a frozen `updatedAt` announced- // `FINISHED` here and sent no `result` at all, so a turn- // that accepted only `result` waited out its whole poll- // budget against a record that was never going to move.- //- // Not a break: trailing content can still be in flight,- // and `done`, the stream's end, or a quiet gap closes the- // turn now that there is an outcome to close it with.- CursorEvent::Status { status, .. } if status.is_terminal() => {- terminal = Some(status);- }- CursorEvent::Error { .. } => break,- CursorEvent::Done => break,- _ => {}- }+ tracing::warn!(+ cursor.run.id = %run,+ cursor.stream.reason = %reason,+ "Cursor stream never connected; polling the run record instead"+ );+ break;+ }+ let attempt = reconnects + 1;+ self.capture(+ session_id,+ session,+ Some(run),+ JournalInput::StreamInterrupted {+ reason: reason.clone(),+ last_event_id: last_event_id.clone(),+ attempt,+ },+ emit,+ )+ .await?;+ // Hydration verifies the whole captured prefix against a stream it+ // reads from the beginning, which a resumed stream cannot offer.+ let resumable = recoverable && !strict && attempt <= STREAM_RECONNECT_ATTEMPTS as u32;+ if !resumable {+ tracing::warn!(+ cursor.run.id = %run,+ cursor.stream.last_event_id = last_event_id,+ cursor.stream.attempt = attempt,+ cursor.stream.reason = %reason,+ "Cursor stream is not resumable; polling the run record instead"+ );+ break;}+ tracing::warn!(+ cursor.run.id = %run,+ cursor.stream.last_event_id = last_event_id,+ cursor.stream.attempt = attempt,+ cursor.stream.reason = %reason,+ "Cursor stream interrupted; reconnecting"+ );+ let delay = STREAM_RECONNECT_DELAYS+ [(attempt as usize - 1).min(STREAM_RECONNECT_DELAYS.len() - 1)];+ if sleep_unless_cancelled(cancel, delay).await {+ break;+ }+ reconnects = attempt;+ resume_from = last_event_id.clone();+ fell_back_to_poll = false;}+ let span = tracing::Span::current();+ span.record("cursor.stream.reconnects", reconnects);+ span.record("cursor.stream.fell_back_to_poll", fell_back_to_poll);if matched < captured.len() && strict {return Err(rootcause::report!("Cursor no longer exposes the captured stream prefix for {run}"diff --git a/crates/cursor_cloud_agents/src/domain/service/test.rs b/crates/cursor_cloud_agents/src/domain/service/test.rsindex bab270ac0a..9f310c4571 100644--- a/crates/cursor_cloud_agents/src/domain/service/test.rs+++ b/crates/cursor_cloud_agents/src/domain/service/test.rs@@ -1,9 +1,11 @@use super::*;use crate::domain::error::SessionError;use crate::domain::event::{CursorEvent, InteractionUpdate, ToolCallEvent, Truncation};+use crate::domain::journal::NativeRecord;use crate::domain::model::{McpHeader, McpServer, McpTransport, RepoUrl, RunListing, RunOutcome, RunStatus,};+use crate::domain::ports::StreamConnectError;use crate::testing::{CursorCall, FakeCursor, FixedChooser, RecordingNotifier};use agent_client_protocol::schema::v1::{SessionUpdate, StopReason, ToolCallStatus};use std::path::Path;@@ -680,6 +682,9 @@ async fn an_unknown_terminal_status_fails_the_turn() {/// that never sees a terminal result "must treat the run's outcome as unknown/// rather than successful". A dropped connection ends the stream exactly like/// this, and the run may well still be going server-side.+///+/// Reconnects come first now — no stream is queued for them, so they are+/// exhausted — and the turn still fails when nothing can say how the run ended.#[tokio::test(start_paused = true)]async fn a_stream_that_ends_without_a_result_fails_the_turn_when_polling_cannot_answer() {let (service, cursor, notifier) = service(None);@@ -704,8 +709,10 @@ async fn a_stream_that_ends_without_a_result_fails_the_turn_when_polling_cannot_assert_eq!(notifier.updates().len(), 1);}-/// A stream that dies after delivering text is finished by the poll - and-/// the answer the client already saw is not delivered twice.+/// A stream that dies after delivering text is finished by the poll once the+/// reconnects are exhausted - and the answer the client already saw is not+/// delivered twice. No records carried an id here, so the reconnects have no+/// resume position to offer and none of them reaches a stream.#[tokio::test(start_paused = true)]async fn a_truncated_stream_is_finished_by_the_poll_without_repeating_text() {let (service, cursor, notifier) = service(None);@@ -2964,3 +2971,384 @@ async fn a_cancel_near_poll_exhaustion_still_reports_cancelled() {.expect("a cancelled turn reports a stop reason, not an error");assert_eq!(stop, StopReason::Cancelled);}++/// A stream that dies mid-run is resumed, not abandoned.+///+/// Observed twice in production: the SSE body fails during a long tool call+/// while the run carries on server-side, and everything Cursor says between+/// the drop and the run's end is lost from the transcript. The last event id+/// is the only thing Cursor accepts as a resume position, so it is what goes+/// back.+#[tokio::test(start_paused = true)]+async fn a_dropped_stream_is_resumed_from_the_last_event_id() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "before the drop".to_owned(),+ },+ "1713033006000-0",+ )+ .expect("stream open");+ drop(dropped);+ let resumed = cursor.script_stream();+ resumed+ .send(CursorEvent::Assistant {+ text: " and after it".to_owned(),+ })+ .expect("stream open");+ resumed.send(finished("run-fake-1")).expect("stream open");+ resumed.send(CursorEvent::Done).expect("stream open");+ drop(resumed);++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the resumed stream finishes the turn");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(+ cursor.resume_positions(),+ vec![None, Some("1713033006000-0".to_owned())],+ "the reconnect carries the last id seen, verbatim"+ );+ assert_eq!(+ agent_texts(¬ifier.updates()),+ vec!["before the drop", " and after it"],+ "what Cursor said after the drop still reaches the client"+ );+ let entries = service.journal.read(&session).await.expect("journal");+ assert!(+ entries.iter().any(|entry| matches!(+ &entry.input,+ JournalInput::StreamInterrupted { last_event_id, attempt, reason }+ if last_event_id.as_deref() == Some("1713033006000-0")+ && *attempt == 1+ && reason.contains("error decoding response body")+ )),+ "the break is durable, so the transcript's gap stays explicable"+ );+}++/// Cursor re-sends the run's `status` frame at the top of every reconnect,+/// without an id. It is framing, not a new fact, and journaling it twice would+/// put the same lifecycle event in the durable record twice.+#[tokio::test(start_paused = true)]+async fn a_resumed_stream_does_not_repeat_the_sticky_status_frame() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());+ let running = CursorEvent::Status {+ run_id: CursorRunId::new("run-fake-1"),+ status: RunStatus::Running,+ };++ let dropped = cursor.script_stream_failing_with("connection reset");+ dropped.send(running.clone()).expect("stream open");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "half".to_owned(),+ },+ "evt-1",+ )+ .expect("stream open");+ drop(dropped);+ let resumed = cursor.script_stream();+ resumed.send(running).expect("stream open");+ resumed+ .send(CursorEvent::Assistant {+ text: " and half".to_owned(),+ })+ .expect("stream open");+ resumed.send(finished("run-fake-1")).expect("stream open");+ resumed.send(CursorEvent::Done).expect("stream open");+ drop(resumed);++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("a resumed stream never trips prefix reconciliation");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(agent_texts(¬ifier.updates()), vec!["half", " and half"]);+ let statuses = service+ .journal+ .read(&session)+ .await+ .expect("journal")+ .iter()+ .filter(+ |entry| matches!(&entry.input, JournalInput::Sse(record) if record.event == "status"),+ )+ .count();+ assert_eq!(statuses, 1, "the sticky frame is recognized, not recorded");+}++/// A stream that will not come back stops being retried, and the run's record+/// closes the turn — a turn may lose its liveness, never its answer.+#[tokio::test(start_paused = true)]+async fn exhausted_reconnects_fall_back_to_polling_the_run_record() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "the whole answer".to_owned(),+ },+ "evt-1",+ )+ .expect("stream open");+ drop(dropped);+ // No further streams are queued, so every reconnect fails to connect.+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Finished,+ text: Some("the whole answer".to_owned()),+ });++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the poll still ends the turn");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(+ cursor.resume_positions().len(),+ 1 + STREAM_RECONNECT_ATTEMPTS,+ "the first connect plus a bounded number of reconnects"+ );+ assert_eq!(+ agent_texts(¬ifier.updates()),+ vec!["the whole answer"],+ "the poll does not repeat what the stream already delivered"+ );+}++/// Past Cursor's retention window there is no stream left to resume, so+/// retrying one only delays the run record that can still answer.+#[tokio::test(start_paused = true)]+async fn an_expired_stream_polls_instead_of_reconnecting_again() {+ let (service, cursor, _notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "the whole answer".to_owned(),+ },+ "evt-1",+ )+ .expect("stream open");+ drop(dropped);+ cursor.script_stream_connect_error(StreamConnectError::Expired(+ "410 Gone: stream_expired".to_owned(),+ ));+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Finished,+ text: Some("the whole answer".to_owned()),+ });++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the poll ends the turn");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(+ cursor.resume_positions(),+ vec![None, Some("evt-1".to_owned())],+ "an expired stream is not connected a third time"+ );+}++/// A resume position Cursor will not accept is worth one connect without one:+/// reading the run from the top still beats polling, and the prefix already+/// captured is matched rather than delivered twice.+#[tokio::test(start_paused = true)]+async fn a_rejected_resume_position_reconnects_once_without_one() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "half".to_owned(),+ },+ "evt-1",+ )+ .expect("stream open");+ drop(dropped);+ cursor.script_stream_connect_error(StreamConnectError::InvalidResumePosition(+ "400 Bad Request: invalid_last_event_id".to_owned(),+ ));+ let restarted = cursor.script_stream();+ restarted+ .send_with_id(+ CursorEvent::Assistant {+ text: "half".to_owned(),+ },+ "evt-1",+ )+ .expect("stream open");+ restarted+ .send(CursorEvent::Assistant {+ text: " and half".to_owned(),+ })+ .expect("stream open");+ restarted.send(finished("run-fake-1")).expect("stream open");+ restarted.send(CursorEvent::Done).expect("stream open");+ drop(restarted);++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the restarted stream finishes the turn");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(+ cursor.resume_positions(),+ vec![None, Some("evt-1".to_owned()), None],+ "exactly one blind reconnect follows the rejection"+ );+ assert_eq!(+ agent_texts(¬ifier.updates()),+ vec!["half", " and half"],+ "the replayed prefix is matched, not delivered again"+ );+}++/// Cursor heartbeats an open stream, so a long gap on a run that has not+/// ended is a connection that stopped delivering. The run record still gets+/// its check — that is what closes a turn whose stream hangs after the answer+/// — but a still-running run gets a fresh connection rather than a stream that+/// has gone silent.+#[tokio::test(start_paused = true)]+async fn a_quiet_stream_reconnects_when_the_run_is_still_going() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let quiet = cursor.script_stream();+ quiet+ .send_with_id(+ CursorEvent::Assistant {+ text: "thinking".to_owned(),+ },+ "evt-1",+ )+ .expect("stream open");+ // The sender stays alive: the connection is open and says nothing.+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Running,+ text: None,+ });+ let resumed = cursor.script_stream();+ resumed+ .send(CursorEvent::Assistant {+ text: " done".to_owned(),+ })+ .expect("stream open");+ resumed.send(finished("run-fake-1")).expect("stream open");+ resumed.send(CursorEvent::Done).expect("stream open");+ drop(resumed);++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the reconnect finishes the turn");+ assert_eq!(stop, StopReason::EndTurn);+ drop(quiet);+ assert_eq!(+ cursor.resume_positions(),+ vec![None, Some("evt-1".to_owned())],+ "the quiet connection is replaced, not merely polled around"+ );+ assert_eq!(agent_texts(¬ifier.updates()), vec!["thinking", " done"]);+}++/// A resume never trips prefix reconciliation on a partially captured run.+///+/// The resume position is the last record this connection actually received,+/// so what comes back starts exactly where the prefix check had got to: the+/// rest of the captured prefix is matched as usual and only genuinely new+/// records are appended.+#[tokio::test(start_paused = true)]+async fn a_resumed_stream_continues_the_captured_prefix_without_duplicating_it() {+ let (service, cursor, notifier) = service(None);+ let id = service.new_session(Path::new(""), vec![]);+ let session = service.session(&id).expect("session exists");+ let run = CursorRunId::new("foreign");+ let asked = NativeRecord {+ id: Some("evt-1".to_owned()),+ ..crate::testing::raw_record(CursorEvent::Interaction(InteractionUpdate::UserMessage {+ text: "asked elsewhere".into(),+ }))+ };+ let half = NativeRecord {+ id: Some("evt-2".to_owned()),+ ..crate::testing::raw_record(CursorEvent::Assistant { text: "hel".into() })+ };+ {+ let _gate = session.turn_gate.lock().await;+ service+ .ensure_journal(&id, &session)+ .await+ .expect("journal");+ for record in [&asked, &half] {+ service+ .capture(+ &id,+ &session,+ Some(&run),+ JournalInput::Sse(record.clone()),+ false,+ )+ .await+ .expect("capture");+ }+ // The first connection re-reads only the head of the captured prefix+ // before the transport breaks.+ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped.send_record(asked).expect("stream open");+ drop(dropped);+ // The resume picks up at the rest of the prefix and runs past it.+ let resumed = cursor.script_stream();+ resumed.send_record(half).expect("stream open");+ resumed+ .send(CursorEvent::Assistant { text: "lo".into() })+ .expect("stream open");+ resumed.send(finished("foreign")).expect("stream open");+ resumed.send(CursorEvent::Done).expect("stream open");+ drop(resumed);+ service+ .ingest_run(+ &id,+ &session,+ &CursorAgentId::new("agent"),+ &run,+ &tokio_util::sync::CancellationToken::new(),+ IngestMode::LIVE,+ )+ .await+ .expect("a resumed stream reconciles");+ }+ assert_eq!(+ cursor.resume_positions(),+ vec![None, Some("evt-1".to_owned())]+ );+ assert_eq!(+ agent_texts(¬ifier.updates()),+ vec!["lo"],+ "only the records past the captured prefix are delivered"+ );+ let entries = service.journal.read(&id).await.expect("journal");+ assert_eq!(+ entries+ .iter()+ .filter(|entry| matches!(&entry.input, JournalInput::Sse(record)+ if matches!(record.decode(), CursorEvent::Assistant { text } if text == "hel")))+ .count(),+ 1,+ "the matched prefix is not captured a second time"+ );+}diff --git a/crates/cursor_cloud_agents/src/testing.rs b/crates/cursor_cloud_agents/src/testing.rsindex 5428d9b699..f349dc8214 100644--- a/crates/cursor_cloud_agents/src/testing.rs+++ b/crates/cursor_cloud_agents/src/testing.rs@@ -7,7 +7,8 @@ use crate::domain::model::{RunOutcome,};use crate::domain::ports::{- CursorAgents, RepositoryChooser, RunStream, SessionIntent, SessionNotifier,+ ConnectedStream, CursorAgents, RepositoryChooser, RunStream, SessionIntent, SessionNotifier,+ StreamConnectError,};use agent_client_protocol::schema::v1::{SessionId, SessionUpdate};use futures::Stream;@@ -115,6 +116,27 @@ impl ScriptSender {pub fn send(&self, event: CursorEvent) -> Result<(), mpsc::error::SendError<NativeRecord>> {self.0.send(raw_record(event))}++ /// Enqueue an already-built native record, ids and all.+ pub fn send_record(+ &self,+ record: NativeRecord,+ ) -> Result<(), mpsc::error::SendError<NativeRecord>> {+ self.0.send(record)+ }++ /// Enqueue an event carrying a provider event id, the way Cursor stamps+ /// the records a resume position can point at.+ pub fn send_with_id(+ &self,+ event: CursorEvent,+ id: &str,+ ) -> Result<(), mpsc::error::SendError<NativeRecord>> {+ self.0.send(NativeRecord {+ id: Some(id.to_owned()),+ ..raw_record(event)+ })+ }}/// What a [`FakeCursor`] was asked to do.@@ -148,8 +170,12 @@ pub struct FakeCursor {struct FakeCursorState {calls: Vec<CursorCall>,next_run: u64,- /// The receiver the next `raw_stream()` call will drain.- streams: Vec<mpsc::UnboundedReceiver<NativeRecord>>,+ /// What the next `raw_stream()` calls answer with, consumed in order.+ streams: Vec<ScriptedStream>,+ /// The resume position every `raw_stream()` call received, in order.+ resume_positions: Vec<Option<String>>,+ /// The retention window every connected stream reports.+ retention_seconds: Option<u64>,/// Answers for `run_result`, consumed in order.run_results: Vec<RunOutcome>,/// The answer every `list_runs` call gets.@@ -182,12 +208,49 @@ impl FakeCursor {/// Queue original native records, including recorded wire fixtures.pub fn script_raw_stream(&self) -> mpsc::UnboundedSender<NativeRecord> {+ self.queue_stream(None)+ }++ /// Queue a stream that dies with `failure` once its sender is dropped —+ /// the mid-run transport break, after whatever records were sent first.+ pub fn script_stream_failing_with(&self, failure: &str) -> ScriptSender {+ ScriptSender(self.queue_stream(Some(failure.to_owned())))+ }++ /// Queue a connect that fails instead of producing a stream.+ pub fn script_stream_connect_error(&self, error: StreamConnectError) {+ self.inner+ .lock()+ .expect("fake cursor poisoned")+ .streams+ .push(ScriptedStream::ConnectError(error));+ }++ /// Report `seconds` as the retention window on every connected stream.+ pub fn script_stream_retention(&self, seconds: u64) {+ self.inner+ .lock()+ .expect("fake cursor poisoned")+ .retention_seconds = Some(seconds);+ }++ /// The resume position each `raw_stream()` call was given, in order.+ #[must_use]+ pub fn resume_positions(&self) -> Vec<Option<String>> {+ self.inner+ .lock()+ .expect("fake cursor poisoned")+ .resume_positions+ .clone()+ }++ fn queue_stream(&self, failure: Option<String>) -> mpsc::UnboundedSender<NativeRecord> {let (sender, receiver) = mpsc::unbounded_channel();self.inner.lock().expect("fake cursor poisoned").streams- .push(receiver);+ .push(ScriptedStream::Records { receiver, failure });sender}@@ -429,23 +492,59 @@ impl CursorAgents for FakeCursor {}}+/// What one scripted `raw_stream()` call answers with.+#[derive(Debug)]+enum ScriptedStream {+ /// Records the test pushes, optionally ending in a transport failure once+ /// the sender is dropped.+ Records {+ receiver: mpsc::UnboundedReceiver<NativeRecord>,+ failure: Option<String>,+ },+ /// A connect that never produces a stream.+ ConnectError(StreamConnectError),+}+impl RunStream for FakeCursor {async fn raw_stream(&self,_agent: &CursorAgentId,_run: &CursorRunId,- ) -> Result<impl Stream<Item = Result<NativeRecord, rootcause::Report>> + Send, rootcause::Report>- {- let receiver = {+ resume_from: Option<&str>,+ ) -> Result<+ ConnectedStream<impl Stream<Item = Result<NativeRecord, rootcause::Report>> + Send>,+ StreamConnectError,+ > {+ let (receiver, failure, retention_seconds) = {let mut state = self.inner.lock().expect("fake cursor poisoned");+ state.resume_positions.push(resume_from.map(str::to_owned));if state.streams.is_empty() {- return Err(rootcause::report!("no scripted stream queued"));+ return Err(StreamConnectError::Other(rootcause::report!(+ "no scripted stream queued"+ )));+ }+ let retention_seconds = state.retention_seconds;+ match state.streams.remove(0) {+ ScriptedStream::ConnectError(error) => return Err(error),+ ScriptedStream::Records { receiver, failure } => {+ (receiver, failure, retention_seconds)+ }}- state.streams.remove(0)};- Ok(futures::stream::unfold(receiver, |mut receiver| async {- receiver.recv().await.map(|event| (Ok(event), receiver))- }))+ let records =+ futures::stream::unfold((receiver, failure), |(mut receiver, failure)| async move {+ match receiver.recv().await {+ Some(event) => Some((Ok(event), (receiver, failure))),+ // The channel closing is the connection closing: a+ // scripted failure is what the transport says as it goes.+ None => failure+ .map(|failure| (Err(rootcause::report!("{failure}")), (receiver, None))),+ }+ });+ Ok(ConnectedStream {+ records,+ retention_seconds,+ })}}