Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(cursor): resume a dropped run stream instead of polling out the run (#6459)

Wolf Mermelstein39591d759b54
diff --git a/crates/agent_harness/src/outbound/cursor/manager.rs b/crates/agent_harness/src/outbound/cursor/manager.rs
index 4eb3b3a54b..2de5a33844 100644
--- a/crates/agent_harness/src/outbound/cursor/manager.rs
+++ b/crates/agent_harness/src/outbound/cursor/manager.rs
@@ -702,15 +702,18 @@ where
&self,
agent: &CursorAgentId,
run: &CursorRunId,
+ resume_from: Option<&str>,
) -> std::result::Result<
- impl Stream<
- Item = std::result::Result<
- cursor_cloud_agents::domain::journal::NativeRecord,
- rootcause::Report,
- >,
- > + Send,
- rootcause::Report,
+ cursor_cloud_agents::domain::ports::ConnectedStream<
+ impl Stream<
+ Item = std::result::Result<
+ cursor_cloud_agents::domain::journal::NativeRecord,
+ rootcause::Report,
+ >,
+ > + Send,
+ >,
+ cursor_cloud_agents::domain::ports::StreamConnectError,
> {
- self.client.raw_stream(agent, run).await
+ self.client.raw_stream(agent, run, resume_from).await
}
}
diff --git a/crates/cursor_cloud_agents/src/api.rs b/crates/cursor_cloud_agents/src/api.rs
index 3b0f75e189..0ff440d3c3 100644
--- a/crates/cursor_cloud_agents/src/api.rs
+++ b/crates/cursor_cloud_agents/src/api.rs
@@ -30,7 +30,7 @@ use crate::domain::model::{
CursorAgentId, CursorModel, CursorRunId, McpServer, ModelChoice, ModelParam, ModelVariant,
RepoUrl, RunListing,
};
-use crate::domain::ports::{CursorAgents, RunStream};
+use crate::domain::ports::{ConnectedStream, CursorAgents, RunStream, StreamConnectError};
use futures::{Stream, StreamExt as _};
use sse_core::SseEvent;
use std::collections::VecDeque;
@@ -549,73 +549,93 @@ const STREAM_CONNECT_ATTEMPTS: usize = 5;
/// the second or so between a run's creation and its stream existing.
const STREAM_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(400);
+/// Cursor's header naming how long a dropped stream stays resumable.
+const RETENTION_HEADER: &str = "x-cursor-stream-retention-seconds";
+
+/// The SSE resume header. Not in `http`'s constant set, so it is spelled here.
+const LAST_EVENT_ID_HEADER: &str = "last-event-id";
+
impl RunStream for CursorClient {
// Covers connecting the stream, retries included; reading it belongs to
// the caller's `cursor.run.ingest` span.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self), fields(cursor.stream.resumed = resume_from.is_some()))]
async fn raw_stream(
&self,
agent: &CursorAgentId,
run: &CursorRunId,
+ resume_from: Option<&str>,
) -> Result<
- impl Stream<Item = Result<crate::domain::journal::NativeRecord, rootcause::Report>> + Send,
- rootcause::Report,
+ ConnectedStream<
+ impl Stream<Item = Result<crate::domain::journal::NativeRecord, rootcause::Report>> + Send,
+ >,
+ StreamConnectError,
> {
- // Only retry failed HTTP connects. Successful SSE records, including
- // stream_unavailable, must reach the domain journal before inspection.
+ // Only retry an unavailable stream. Successful SSE records, including
+ // a `stream_unavailable` record, must reach the domain journal before
+ // inspection, and a rejected or expired resume position is the
+ // domain's decision to make, not a thing to sit and retry.
for attempt in 1..=STREAM_CONNECT_ATTEMPTS {
- match self.connect_stream(agent, run).await {
+ match self.connect_stream(agent, run, resume_from).await {
Ok(stream) => return Ok(stream),
Err(StreamConnectError::Unavailable(_)) if attempt < STREAM_CONNECT_ATTEMPTS => {
tokio::time::sleep(STREAM_RETRY_DELAY).await
}
- Err(StreamConnectError::Unavailable(message)) => {
- return Err(rootcause::report!("Cursor stream unavailable: {message}"));
- }
- Err(StreamConnectError::Other(error)) => return Err(error),
+ Err(error) => return Err(error),
}
}
unreachable!("connect attempts are nonzero")
}
}
-/// Why one stream connect did not produce a stream: the endpoint saying the
-/// stream is not there (retryable — it appears seconds after run creation),
-/// or anything else (not).
-enum StreamConnectError {
- /// `stream_unavailable`, as an HTTP status. Carries the server's message.
- Unavailable(String),
- /// Every other failure.
- Other(rootcause::Report),
-}
-
impl CursorClient {
async fn connect_stream(
&self,
agent: &CursorAgentId,
run: &CursorRunId,
+ resume_from: Option<&str>,
) -> Result<
- impl Stream<Item = Result<crate::domain::journal::NativeRecord, rootcause::Report>>
- + Send
- + use<>,
+ ConnectedStream<
+ impl Stream<Item = Result<crate::domain::journal::NativeRecord, rootcause::Report>>
+ + Send
+ + use<>,
+ >,
StreamConnectError,
> {
- let response = self
+ let mut request = self
.http
.get(self.url(&format!("/v1/agents/{agent}/runs/{run}/stream")))
.basic_auth(self.config.api_key.expose(), Some(""))
- .header(reqwest::header::ACCEPT, "text/event-stream")
+ .header(reqwest::header::ACCEPT, "text/event-stream");
+ if let Some(resume_from) = resume_from {
+ request = request.header(LAST_EVENT_ID_HEADER, resume_from);
+ }
+ let response = request
.send()
.await
.map_err(|error| StreamConnectError::Other(rootcause::report!(error).into()))?;
let status = response.status();
+ // Read before the body is consumed; a resumed stream's window is the
+ // only thing that says whether the next drop is recoverable at all.
+ let retention_seconds = response
+ .headers()
+ .get(RETENTION_HEADER)
+ .and_then(|value| value.to_str().ok())
+ .and_then(|value| value.parse().ok());
if !status.is_success() {
let text = response.text().await.unwrap_or_default();
+ let detail = format!("{status}: {text}");
if text.contains("stream_unavailable") {
- return Err(StreamConnectError::Unavailable(format!("{status}: {text}")));
+ return Err(StreamConnectError::Unavailable(detail));
+ }
+ if status == reqwest::StatusCode::BAD_REQUEST && text.contains("invalid_last_event_id")
+ {
+ return Err(StreamConnectError::InvalidResumePosition(detail));
+ }
+ if status == reqwest::StatusCode::GONE || text.contains("stream_expired") {
+ return Err(StreamConnectError::Expired(detail));
}
return Err(StreamConnectError::Other(rootcause::report!(
- "cursor stream -> {status}: {text}"
+ "cursor stream -> {detail}"
)));
}
@@ -636,7 +656,7 @@ impl CursorClient {
VecDeque::new(),
recording,
);
- Ok(futures::stream::try_unfold(
+ let records = futures::stream::try_unfold(
state,
|(mut bytes, mut decoder, mut pending, mut recording)| async move {
loop {
@@ -665,7 +685,7 @@ impl CursorClient {
}
};
let SseEvent::Message(message) = record else {
- continue; // `retry:`; nothing reconnects yet
+ continue; // `retry:`; the domain drives reconnects
};
pending.push_back(Ok(crate::domain::journal::NativeRecord {
event: message.event.into_owned(),
@@ -681,6 +701,10 @@ impl CursorClient {
}
}
},
- ))
+ );
+ Ok(ConnectedStream {
+ records,
+ retention_seconds,
+ })
}
}
diff --git a/crates/cursor_cloud_agents/src/domain/journal.rs b/crates/cursor_cloud_agents/src/domain/journal.rs
index 5c3133ce47..6e5433aa2d 100644
--- a/crates/cursor_cloud_agents/src/domain/journal.rs
+++ b/crates/cursor_cloud_agents/src/domain/journal.rs
@@ -48,6 +48,24 @@ pub enum JournalInput {
PromptAborted(i64),
/// Transport failure, retained without prematurely closing running tools.
TransportError(String),
+ /// The run's stream broke or went silent and a reconnect was attempted.
+ ///
+ /// Journaled rather than merely logged because the journal is the durable
+ /// record of what the transport did: a transcript with a gap in it is only
+ /// explicable if the gap's cause is in the same ordered record as the
+ /// content around it, and a later replay of the session has no other way
+ /// to know a reconnect happened here. Like
+ /// [`JournalInput::TransportError`] it projects to nothing — it is a fact
+ /// about the connection, never about the conversation.
+ StreamInterrupted {
+ /// What broke, in the transport's own words.
+ reason: String,
+ /// The last provider event id captured before the break, if any; what
+ /// the reconnect resumed from.
+ last_event_id: Option<String>,
+ /// Which reconnect attempt this interruption started, from one.
+ attempt: u32,
+ },
/// Raw complete provider message, including unknown payloads.
Sse(NativeRecord),
/// Original successful polling response body.
@@ -201,7 +219,8 @@ impl ReplayMachine {
| JournalInput::Reconciled
| JournalInput::PromptAccepted(_)
| JournalInput::PromptAborted(_)
- | JournalInput::TransportError(_) => Ok(Vec::new()),
+ | JournalInput::TransportError(_)
+ | JournalInput::StreamInterrupted { .. } => Ok(Vec::new()),
}
}
fn event(
diff --git a/crates/cursor_cloud_agents/src/domain/journal/test.rs b/crates/cursor_cloud_agents/src/domain/journal/test.rs
index c30810b9ee..9f3cb40c93 100644
--- a/crates/cursor_cloud_agents/src/domain/journal/test.rs
+++ b/crates/cursor_cloud_agents/src/domain/journal/test.rs
@@ -363,3 +363,39 @@ fn divergent_terminal_text_keeps_the_captured_answer_and_the_terminal_facts() {
);
}
}
+
+/// A stream interruption is a fact about the connection, not the conversation:
+/// it is durable so a gap in a transcript is explicable, and it must add
+/// nothing to the transcript it explains — on capture or on any later replay.
+#[test]
+fn a_stream_interruption_projects_to_nothing() {
+ let run = CursorRunId::new("run");
+ let interruption = JournalInput::StreamInterrupted {
+ reason: "error decoding response body".into(),
+ last_event_id: Some("1713033006000-0".into()),
+ attempt: 2,
+ };
+ let mut machine = ReplayMachine::default();
+ assert!(
+ machine
+ .push(
+ Some(&run),
+ &JournalInput::Sse(crate::testing::raw_record(CursorEvent::Assistant {
+ text: "before".into()
+ }))
+ )
+ .unwrap()
+ .len()
+ == 1
+ );
+ assert!(machine.push(Some(&run), &interruption).unwrap().is_empty());
+ assert!(machine.push(None, &interruption).unwrap().is_empty());
+ assert!(!machine.complete(&run), "no outcome was invented either");
+ // Durability is the point, so the variant has to survive the journal's
+ // own encoding unchanged.
+ let encoded = serde_json::to_string(&interruption).unwrap();
+ assert_eq!(
+ serde_json::from_str::<JournalInput>(&encoded).unwrap(),
+ interruption
+ );
+}
diff --git a/crates/cursor_cloud_agents/src/domain/ports.rs b/crates/cursor_cloud_agents/src/domain/ports.rs
index a6b2192a62..e51033933b 100644
--- a/crates/cursor_cloud_agents/src/domain/ports.rs
+++ b/crates/cursor_cloud_agents/src/domain/ports.rs
@@ -92,6 +92,51 @@ pub trait CursorAgents: Sync {
) -> impl Future<Output = Result<Vec<RunListing>, rootcause::Report>> + Send;
}
+/// One connected run stream, with what the provider said about resuming it.
+pub struct ConnectedStream<Records> {
+ /// Complete native records, captured before decoding or translation.
+ pub records: Records,
+ /// The provider's `X-Cursor-Stream-Retention-Seconds`, when it sent one.
+ ///
+ /// How long a dropped stream stays resumable. Not enforced here — the
+ /// provider answers an expired resume with
+ /// [`StreamConnectError::Expired`] — but worth logging, because a run
+ /// whose tool call outlives the window can never be resumed and the
+ /// number is the only warning of that.
+ pub retention_seconds: Option<u64>,
+}
+
+/// Why one connect did not produce a stream.
+///
+/// The domain acts differently on each: an unavailable stream is retried, an
+/// invalid resume position is retried without one, and an expired stream is
+/// gone for good and leaves polling as the only way to learn the outcome.
+#[derive(Debug)]
+pub enum StreamConnectError {
+ /// `stream_unavailable`: the stream is not there (yet). Carries the
+ /// provider's message.
+ Unavailable(String),
+ /// `invalid_last_event_id`: the resume position is not this run's.
+ InvalidResumePosition(String),
+ /// `stream_expired`: the retention window closed behind us.
+ Expired(String),
+ /// Every other failure.
+ Other(rootcause::Report),
+}
+
+impl std::fmt::Display for StreamConnectError {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::Unavailable(message) => write!(formatter, "Cursor stream unavailable: {message}"),
+ Self::InvalidResumePosition(message) => {
+ write!(formatter, "Cursor rejected the resume position: {message}")
+ }
+ Self::Expired(message) => write!(formatter, "Cursor stream expired: {message}"),
+ Self::Other(report) => write!(formatter, "{report}"),
+ }
+ }
+}
+
/// Observe a run as a stream of native SSE records.
///
/// The stream ends when the server closes it — normally just after a
@@ -99,15 +144,23 @@ pub trait CursorAgents: Sync {
/// [`CursorEvent::Result`](super::event::CursorEvent::Result) must treat the run's outcome as unknown rather
/// than successful.
pub trait RunStream: Sync {
- /// Complete native records, captured before decoding or translation.
+ /// Connect a run's stream, optionally resuming after an event id.
+ ///
+ /// `resume_from` is an id observed on an earlier record of this same run,
+ /// passed back verbatim: the provider's ids are opaque and a consumer that
+ /// parses or invents one gets [`StreamConnectError::InvalidResumePosition`].
+ /// `None` connects from the beginning of what the provider still retains.
fn raw_stream(
&self,
agent: &CursorAgentId,
run: &CursorRunId,
+ resume_from: Option<&str>,
) -> impl Future<
Output = Result<
- impl Stream<Item = Result<super::journal::NativeRecord, rootcause::Report>> + Send,
- rootcause::Report,
+ ConnectedStream<
+ impl Stream<Item = Result<super::journal::NativeRecord, rootcause::Report>> + Send,
+ >,
+ StreamConnectError,
>,
> + Send;
}
diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rs
index ce239e3377..475ef99134 100644
--- a/crates/cursor_cloud_agents/src/domain/service.rs
+++ b/crates/cursor_cloud_agents/src/domain/service.rs
@@ -44,7 +44,9 @@ use crate::domain::journal::{CursorJournal, JournalEntry, JournalInput, ReplayMa
use crate::domain::model::{
CursorAgentId, CursorModel, CursorRunId, McpServer, ModelChoice, RepoUrl, RunStatus,
};
-use crate::domain::ports::{CursorAgents, RepositoryChooser, RunStream, SessionNotifier};
+use crate::domain::ports::{
+ CursorAgents, RepositoryChooser, RunStream, SessionNotifier, StreamConnectError,
+};
use agent_client_protocol::schema::v1::{
ContentBlock, SessionId, SessionUpdate, StopReason, TextContent,
};
@@ -73,6 +75,27 @@ const POLL_ERROR_TOLERANCE: usize = 5;
/// interval.
const STREAM_QUIET_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
+/// Backoff before each reconnect of an interrupted stream.
+///
+/// Observed twice in production: the SSE body fails mid-run during a long tool
+/// call while the run itself carries on fine, and everything Cursor says
+/// between the drop and the run's end — tool results, prose, the images it
+/// writes into prose — never reaches the transcript. Resuming from the last
+/// event id gets that back; the ramp keeps a provider that is actually down
+/// from being hammered.
+const STREAM_RECONNECT_DELAYS: [std::time::Duration; 5] = [
+ std::time::Duration::from_millis(500),
+ std::time::Duration::from_secs(1),
+ std::time::Duration::from_secs(2),
+ std::time::Duration::from_secs(4),
+ std::time::Duration::from_secs(8),
+];
+
+/// Reconnects before a run gives up on its stream and polls instead. Counted
+/// since the last reconnect that delivered content, so a long run that drops
+/// repeatedly but recovers each time is never rationed.
+const STREAM_RECONNECT_ATTEMPTS: usize = 5;
+
/// How long a prompt waits behind a run something else started (the same
/// agent is drivable from cursor.com) before giving up, in poll intervals.
const BUSY_ATTEMPTS: usize = 450;
@@ -112,6 +135,30 @@ async fn sleep_unless_cancelled(
}
}
+/// The run's content records already durable in this session's journal.
+///
+/// What a stream read from the beginning is reconciled against: every record
+/// here must arrive again, in order, before anything new is appended. Recomputed
+/// rather than captured once, because a reconnect that restarts the stream has
+/// to account for what the interrupted connection already appended.
+fn captured_content(
+ session: &Session,
+ run: &CursorRunId,
+) -> Vec<crate::domain::journal::NativeRecord> {
+ session
+ .state
+ .lock()
+ .expect("session state poisoned")
+ .journal_entries
+ .iter()
+ .filter(|entry| entry.run.as_ref() == Some(run))
+ .filter_map(|entry| match &entry.input {
+ JournalInput::Sse(record) if record.is_content() => Some(record.clone()),
+ _ => None,
+ })
+ .collect()
+}
+
/// Restate a repository rejection as something the person who prompted can
/// act on, leaving every other failure exactly as it arrived.
///
@@ -1008,6 +1055,8 @@ where
cursor.ingest.emit = mode.emit,
cursor.ingest.strict = mode.strict,
cursor.ingest.attempt = mode.attempt,
+ cursor.stream.reconnects = tracing::field::Empty,
+ cursor.stream.fell_back_to_poll = tracing::field::Empty,
),
err,
)]
@@ -1062,134 +1111,280 @@ where
status => Err(rootcause::report!("cursor run {run} ended in {status:?}").into()),
};
}
- let captured: Vec<_> = session
- .state
- .lock()
- .expect("session state poisoned")
- .journal_entries
- .iter()
- .filter(|e| e.run.as_ref() == Some(run))
- .filter_map(|e| match &e.input {
- JournalInput::Sse(r) if r.is_content() => Some(r.clone()),
- _ => None,
- })
- .collect();
+ let mut captured = captured_content(session, run);
let mut matched = 0;
let mut terminal = None;
let mut saw_content = false;
- let stream = match self.cursor.raw_stream(agent, run).await {
- Ok(stream) => Some(stream),
- Err(error) => {
- self.capture(
- session_id,
- session,
- Some(run),
- JournalInput::TransportError(error.to_string()),
- emit,
- )
- .await?;
- None
- }
- };
- if let Some(stream) = stream {
- pin_mut!(stream);
- loop {
- let record = match tokio::time::timeout(STREAM_QUIET_TIMEOUT, stream.next()).await {
- Ok(Some(Ok(record))) => record,
- Ok(Some(Err(error))) => {
+ // The provider's most recent event id. Opaque: it is handed back
+ // verbatim as a resume position or not used at all.
+ let mut last_event_id: Option<String> = None;
+ // Reconnects since the last one that produced new records.
+ let mut reconnects: u32 = 0;
+ let mut fell_back_to_poll = false;
+ let mut ever_connected = false;
+ let mut resume_from: Option<String> = None;
+ // Cursor rejecting a resume position buys exactly one connect without
+ // one; a second rejection means something is wrong with the run, not
+ // with the id we sent.
+ let mut blind_reconnect_available = true;
+ // The run's latest `status` frame, kept to recognize the sticky copy
+ // Cursor re-sends at the top of every reconnect.
+ let mut last_status: Option<crate::domain::journal::NativeRecord> = None;
+ loop {
+ // The connected stream borrows the position it resumed from, so
+ // the position this iteration sends is its own owned copy.
+ let resuming = resume_from.clone();
+ let interruption = match self
+ .cursor
+ .raw_stream(agent, run, resuming.as_deref())
+ .await
+ {
+ Err(StreamConnectError::InvalidResumePosition(detail))
+ if blind_reconnect_available && resuming.is_some() =>
+ {
+ // Reading the run from the top still beats polling: the
+ // prefix logic below matches what is already captured
+ // instead of duplicating it. It has to be recomputed
+ // first, because this ingestion has appended to it.
+ blind_reconnect_available = false;
+ tracing::warn!(
+ cursor.run.id = %run,
+ cursor.stream.last_event_id = resuming,
+ cursor.stream.reason = %detail,
+ "Cursor rejected the stream resume position; reconnecting from the start"
+ );
+ resume_from = None;
+ captured = captured_content(session, run);
+ matched = 0;
+ continue;
+ }
+ // Past the retention window there is no stream left to resume,
+ // so the run record is the only remaining account of the run.
+ Err(StreamConnectError::Expired(detail)) => Some((detail, false)),
+ Err(error) => Some((error.to_string(), true)),
+ Ok(connected) => {
+ let resumed = resuming.is_some();
+ ever_connected = true;
+ let retention_seconds = connected.retention_seconds;
+ let records = connected.records;
+ pin_mut!(records);
+ let mut received = 0usize;
+ let mut received_content = 0usize;
+ let mut sticky_status_pending = resumed;
+ let interruption = loop {
+ let record = match tokio::time::timeout(
+ STREAM_QUIET_TIMEOUT,
+ records.next(),
+ )
+ .await
+ {
+ Ok(Some(Ok(record))) => record,
+ Ok(Some(Err(error))) => break Some((error.to_string(), true)),
+ // Cursor closes the stream after `done`, which
+ // breaks below with an outcome. Reaching here
+ // without one is a close mid-run.
+ Ok(None) if terminal.is_some() => break None,
+ Ok(None) => break Some(("stream closed".to_owned(), true)),
+ Err(_) if strict => break None,
+ Err(_) => {
+ // Quiet streams are checked through the exact same raw
+ // polling/capture path as disconnected streams.
+ let status = self
+ .poll_once(session_id, session, agent, run, cancel, emit)
+ .await?;
+ if status.is_terminal() {
+ terminal = Some(status.status);
+ break None;
+ }
+ if cancel.is_cancelled() {
+ break None;
+ }
+ // Cursor heartbeats an open stream, so a
+ // gap this long on a run that has not
+ // ended is a connection that stopped
+ // delivering rather than an agent thinking.
+ break Some((
+ format!(
+ "no records for {} seconds",
+ STREAM_QUIET_TIMEOUT.as_secs()
+ ),
+ true,
+ ));
+ }
+ };
+ if let Some(id) = &record.id {
+ last_event_id = Some(id.clone());
+ }
+ // Cursor re-sends the run's `status` frame, without an
+ // id, at the top of every reconnect. It is framing, not
+ // a new fact: journaling it again would put the same
+ // lifecycle event in the record twice.
+ if std::mem::take(&mut sticky_status_pending)
+ && record.id.is_none()
+ && last_status.as_ref() == Some(&record)
+ {
+ continue;
+ }
+ if record.event == "status" {
+ last_status = Some(record.clone());
+ }
+ received += 1;
+ let content = record.is_content();
+ if content && matched < captured.len() {
+ if captured[matched] != record {
+ return Err(rootcause::report!("Cursor stream prefix cannot be reconciled for {run}; refusing incomplete history").into());
+ }
+ matched += 1;
+ if let CursorEvent::Result { status, .. } = record.decode() {
+ terminal = Some(status);
+ }
+ continue;
+ }
self.capture(
session_id,
session,
Some(run),
- JournalInput::TransportError(error.to_string()),
+ JournalInput::Sse(record.clone()),
emit,
)
.await?;
- break;
- }
- Ok(None) => break,
- Err(_) if strict => break,
- Err(_) => {
- // Quiet streams are checked through the exact same raw
- // polling/capture path as disconnected streams.
- let status = self
- .poll_once(session_id, session, agent, run, cancel, emit)
- .await?;
- if status.is_terminal() {
- terminal = Some(status.status);
- break;
+ saw_content |= content;
+ received_content += usize::from(content);
+ match record.decode() {
+ CursorEvent::Error { code, .. }
+ if code.as_deref() == Some("stream_unavailable")
+ && !saw_content
+ && attempt < 4 =>
+ {
+ if !sleep_unless_cancelled(
+ cancel,
+ std::time::Duration::from_millis(400),
+ )
+ .await
+ {
+ return Box::pin(self.ingest_run(
+ session_id,
+ session,
+ agent,
+ run,
+ cancel,
+ IngestMode {
+ attempt: attempt + 1,
+ ..mode
+ },
+ ))
+ .await;
+ }
+ break None;
+ }
+ CursorEvent::Result { status, .. } => terminal = Some(status),
+ // The stream's own account of the run's lifecycle, and a
+ // terminal one is a terminal fact even with no `result`
+ // frame behind it. Observed live: a run whose record
+ // stayed `RUNNING` with a frozen `updatedAt` announced
+ // `FINISHED` here and sent no `result` at all, so a turn
+ // that accepted only `result` waited out its whole poll
+ // budget against a record that was never going to move.
+ //
+ // Not a break: trailing content can still be in flight,
+ // and `done`, the stream's end, or a quiet gap closes the
+ // turn now that there is an outcome to close it with.
+ CursorEvent::Status { status, .. } if status.is_terminal() => {
+ terminal = Some(status);
+ }
+ // Cursor said its piece about this run; a fresh
+ // connection would only be told the same thing.
+ CursorEvent::Error { .. } => break None,
+ CursorEvent::Done => break None,
+ _ => {}
}
- if cancel.is_cancelled() {
- break;
+ };
+ if resumed {
+ tracing::info!(
+ cursor.run.id = %run,
+ cursor.stream.attempt = reconnects,
+ cursor.stream.last_event_id = last_event_id,
+ cursor.stream.retention_seconds = retention_seconds,
+ cursor.stream.resumed_records = received,
+ "Cursor stream resumed"
+ );
+ // A resume that delivered is a working stream again,
+ // so the next drop gets the full budget rather than
+ // whatever a much earlier one left over.
+ if received_content > 0 {
+ reconnects = 0;
}
- continue;
- }
- };
- let content = record.is_content();
- if content && matched < captured.len() {
- if captured[matched] != record {
- return Err(rootcause::report!("Cursor stream prefix cannot be reconciled for {run}; refusing incomplete history").into());
}
- matched += 1;
- if let CursorEvent::Result { status, .. } = record.decode() {
- terminal = Some(status);
- }
- continue;
+ interruption
}
+ };
+ let Some((reason, recoverable)) = interruption else {
+ break;
+ };
+ fell_back_to_poll = true;
+ if !ever_connected {
+ // Nothing to resume: the stream never opened, and the client
+ // has already retried an unavailable one on its own.
self.capture(
session_id,
session,
Some(run),
- JournalInput::Sse(record.clone()),
+ JournalInput::TransportError(reason.clone()),
emit,
)
.await?;
- saw_content |= content;
- match record.decode() {
- CursorEvent::Error { code, .. }
- if code.as_deref() == Some("stream_unavailable")
- && !saw_content
- && attempt < 4 =>
- {
- if !sleep_unless_cancelled(cancel, std::time::Duration::from_millis(400))
- .await
- {
- return Box::pin(self.ingest_run(
- session_id,
- session,
- agent,
- run,
- cancel,
- IngestMode {
- attempt: attempt + 1,
- ..mode
- },
- ))
- .await;
- }
- break;
- }
- CursorEvent::Result { status, .. } => terminal = Some(status),
- // The stream's own account of the run's lifecycle, and a
- // terminal one is a terminal fact even with no `result`
- // frame behind it. Observed live: a run whose record
- // stayed `RUNNING` with a frozen `updatedAt` announced
- // `FINISHED` here and sent no `result` at all, so a turn
- // that accepted only `result` waited out its whole poll
- // budget against a record that was never going to move.
- //
- // Not a break: trailing content can still be in flight,
- // and `done`, the stream's end, or a quiet gap closes the
- // turn now that there is an outcome to close it with.
- CursorEvent::Status { status, .. } if status.is_terminal() => {
- terminal = Some(status);
- }
- CursorEvent::Error { .. } => break,
- CursorEvent::Done => break,
- _ => {}
- }
+ tracing::warn!(
+ cursor.run.id = %run,
+ cursor.stream.reason = %reason,
+ "Cursor stream never connected; polling the run record instead"
+ );
+ break;
+ }
+ let attempt = reconnects + 1;
+ self.capture(
+ session_id,
+ session,
+ Some(run),
+ JournalInput::StreamInterrupted {
+ reason: reason.clone(),
+ last_event_id: last_event_id.clone(),
+ attempt,
+ },
+ emit,
+ )
+ .await?;
+ // Hydration verifies the whole captured prefix against a stream it
+ // reads from the beginning, which a resumed stream cannot offer.
+ let resumable = recoverable && !strict && attempt <= STREAM_RECONNECT_ATTEMPTS as u32;
+ if !resumable {
+ tracing::warn!(
+ cursor.run.id = %run,
+ cursor.stream.last_event_id = last_event_id,
+ cursor.stream.attempt = attempt,
+ cursor.stream.reason = %reason,
+ "Cursor stream is not resumable; polling the run record instead"
+ );
+ break;
}
+ tracing::warn!(
+ cursor.run.id = %run,
+ cursor.stream.last_event_id = last_event_id,
+ cursor.stream.attempt = attempt,
+ cursor.stream.reason = %reason,
+ "Cursor stream interrupted; reconnecting"
+ );
+ let delay = STREAM_RECONNECT_DELAYS
+ [(attempt as usize - 1).min(STREAM_RECONNECT_DELAYS.len() - 1)];
+ if sleep_unless_cancelled(cancel, delay).await {
+ break;
+ }
+ reconnects = attempt;
+ resume_from = last_event_id.clone();
+ fell_back_to_poll = false;
}
+ let span = tracing::Span::current();
+ span.record("cursor.stream.reconnects", reconnects);
+ span.record("cursor.stream.fell_back_to_poll", fell_back_to_poll);
if matched < captured.len() && strict {
return Err(rootcause::report!(
"Cursor no longer exposes the captured stream prefix for {run}"
diff --git a/crates/cursor_cloud_agents/src/domain/service/test.rs b/crates/cursor_cloud_agents/src/domain/service/test.rs
index bab270ac0a..9f310c4571 100644
--- a/crates/cursor_cloud_agents/src/domain/service/test.rs
+++ b/crates/cursor_cloud_agents/src/domain/service/test.rs
@@ -1,9 +1,11 @@
use super::*;
use crate::domain::error::SessionError;
use crate::domain::event::{CursorEvent, InteractionUpdate, ToolCallEvent, Truncation};
+use crate::domain::journal::NativeRecord;
use crate::domain::model::{
McpHeader, McpServer, McpTransport, RepoUrl, RunListing, RunOutcome, RunStatus,
};
+use crate::domain::ports::StreamConnectError;
use crate::testing::{CursorCall, FakeCursor, FixedChooser, RecordingNotifier};
use agent_client_protocol::schema::v1::{SessionUpdate, StopReason, ToolCallStatus};
use std::path::Path;
@@ -680,6 +682,9 @@ async fn an_unknown_terminal_status_fails_the_turn() {
/// that never sees a terminal result "must treat the run's outcome as unknown
/// rather than successful". A dropped connection ends the stream exactly like
/// this, and the run may well still be going server-side.
+///
+/// Reconnects come first now — no stream is queued for them, so they are
+/// exhausted — and the turn still fails when nothing can say how the run ended.
#[tokio::test(start_paused = true)]
async fn a_stream_that_ends_without_a_result_fails_the_turn_when_polling_cannot_answer() {
let (service, cursor, notifier) = service(None);
@@ -704,8 +709,10 @@ async fn a_stream_that_ends_without_a_result_fails_the_turn_when_polling_cannot_
assert_eq!(notifier.updates().len(), 1);
}
-/// A stream that dies after delivering text is finished by the poll - and
-/// the answer the client already saw is not delivered twice.
+/// A stream that dies after delivering text is finished by the poll once the
+/// reconnects are exhausted - and the answer the client already saw is not
+/// delivered twice. No records carried an id here, so the reconnects have no
+/// resume position to offer and none of them reaches a stream.
#[tokio::test(start_paused = true)]
async fn a_truncated_stream_is_finished_by_the_poll_without_repeating_text() {
let (service, cursor, notifier) = service(None);
@@ -2964,3 +2971,384 @@ async fn a_cancel_near_poll_exhaustion_still_reports_cancelled() {
.expect("a cancelled turn reports a stop reason, not an error");
assert_eq!(stop, StopReason::Cancelled);
}
+
+/// A stream that dies mid-run is resumed, not abandoned.
+///
+/// Observed twice in production: the SSE body fails during a long tool call
+/// while the run carries on server-side, and everything Cursor says between
+/// the drop and the run's end is lost from the transcript. The last event id
+/// is the only thing Cursor accepts as a resume position, so it is what goes
+/// back.
+#[tokio::test(start_paused = true)]
+async fn a_dropped_stream_is_resumed_from_the_last_event_id() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "before the drop".to_owned(),
+ },
+ "1713033006000-0",
+ )
+ .expect("stream open");
+ drop(dropped);
+ let resumed = cursor.script_stream();
+ resumed
+ .send(CursorEvent::Assistant {
+ text: " and after it".to_owned(),
+ })
+ .expect("stream open");
+ resumed.send(finished("run-fake-1")).expect("stream open");
+ resumed.send(CursorEvent::Done).expect("stream open");
+ drop(resumed);
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the resumed stream finishes the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(
+ cursor.resume_positions(),
+ vec![None, Some("1713033006000-0".to_owned())],
+ "the reconnect carries the last id seen, verbatim"
+ );
+ assert_eq!(
+ agent_texts(&notifier.updates()),
+ vec!["before the drop", " and after it"],
+ "what Cursor said after the drop still reaches the client"
+ );
+ let entries = service.journal.read(&session).await.expect("journal");
+ assert!(
+ entries.iter().any(|entry| matches!(
+ &entry.input,
+ JournalInput::StreamInterrupted { last_event_id, attempt, reason }
+ if last_event_id.as_deref() == Some("1713033006000-0")
+ && *attempt == 1
+ && reason.contains("error decoding response body")
+ )),
+ "the break is durable, so the transcript's gap stays explicable"
+ );
+}
+
+/// Cursor re-sends the run's `status` frame at the top of every reconnect,
+/// without an id. It is framing, not a new fact, and journaling it twice would
+/// put the same lifecycle event in the durable record twice.
+#[tokio::test(start_paused = true)]
+async fn a_resumed_stream_does_not_repeat_the_sticky_status_frame() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+ let running = CursorEvent::Status {
+ run_id: CursorRunId::new("run-fake-1"),
+ status: RunStatus::Running,
+ };
+
+ let dropped = cursor.script_stream_failing_with("connection reset");
+ dropped.send(running.clone()).expect("stream open");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "half".to_owned(),
+ },
+ "evt-1",
+ )
+ .expect("stream open");
+ drop(dropped);
+ let resumed = cursor.script_stream();
+ resumed.send(running).expect("stream open");
+ resumed
+ .send(CursorEvent::Assistant {
+ text: " and half".to_owned(),
+ })
+ .expect("stream open");
+ resumed.send(finished("run-fake-1")).expect("stream open");
+ resumed.send(CursorEvent::Done).expect("stream open");
+ drop(resumed);
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("a resumed stream never trips prefix reconciliation");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(agent_texts(&notifier.updates()), vec!["half", " and half"]);
+ let statuses = service
+ .journal
+ .read(&session)
+ .await
+ .expect("journal")
+ .iter()
+ .filter(
+ |entry| matches!(&entry.input, JournalInput::Sse(record) if record.event == "status"),
+ )
+ .count();
+ assert_eq!(statuses, 1, "the sticky frame is recognized, not recorded");
+}
+
+/// A stream that will not come back stops being retried, and the run's record
+/// closes the turn — a turn may lose its liveness, never its answer.
+#[tokio::test(start_paused = true)]
+async fn exhausted_reconnects_fall_back_to_polling_the_run_record() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "the whole answer".to_owned(),
+ },
+ "evt-1",
+ )
+ .expect("stream open");
+ drop(dropped);
+ // No further streams are queued, so every reconnect fails to connect.
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Finished,
+ text: Some("the whole answer".to_owned()),
+ });
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the poll still ends the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(
+ cursor.resume_positions().len(),
+ 1 + STREAM_RECONNECT_ATTEMPTS,
+ "the first connect plus a bounded number of reconnects"
+ );
+ assert_eq!(
+ agent_texts(&notifier.updates()),
+ vec!["the whole answer"],
+ "the poll does not repeat what the stream already delivered"
+ );
+}
+
+/// Past Cursor's retention window there is no stream left to resume, so
+/// retrying one only delays the run record that can still answer.
+#[tokio::test(start_paused = true)]
+async fn an_expired_stream_polls_instead_of_reconnecting_again() {
+ let (service, cursor, _notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "the whole answer".to_owned(),
+ },
+ "evt-1",
+ )
+ .expect("stream open");
+ drop(dropped);
+ cursor.script_stream_connect_error(StreamConnectError::Expired(
+ "410 Gone: stream_expired".to_owned(),
+ ));
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Finished,
+ text: Some("the whole answer".to_owned()),
+ });
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the poll ends the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(
+ cursor.resume_positions(),
+ vec![None, Some("evt-1".to_owned())],
+ "an expired stream is not connected a third time"
+ );
+}
+
+/// A resume position Cursor will not accept is worth one connect without one:
+/// reading the run from the top still beats polling, and the prefix already
+/// captured is matched rather than delivered twice.
+#[tokio::test(start_paused = true)]
+async fn a_rejected_resume_position_reconnects_once_without_one() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "half".to_owned(),
+ },
+ "evt-1",
+ )
+ .expect("stream open");
+ drop(dropped);
+ cursor.script_stream_connect_error(StreamConnectError::InvalidResumePosition(
+ "400 Bad Request: invalid_last_event_id".to_owned(),
+ ));
+ let restarted = cursor.script_stream();
+ restarted
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "half".to_owned(),
+ },
+ "evt-1",
+ )
+ .expect("stream open");
+ restarted
+ .send(CursorEvent::Assistant {
+ text: " and half".to_owned(),
+ })
+ .expect("stream open");
+ restarted.send(finished("run-fake-1")).expect("stream open");
+ restarted.send(CursorEvent::Done).expect("stream open");
+ drop(restarted);
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the restarted stream finishes the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(
+ cursor.resume_positions(),
+ vec![None, Some("evt-1".to_owned()), None],
+ "exactly one blind reconnect follows the rejection"
+ );
+ assert_eq!(
+ agent_texts(&notifier.updates()),
+ vec!["half", " and half"],
+ "the replayed prefix is matched, not delivered again"
+ );
+}
+
+/// Cursor heartbeats an open stream, so a long gap on a run that has not
+/// ended is a connection that stopped delivering. The run record still gets
+/// its check — that is what closes a turn whose stream hangs after the answer
+/// — but a still-running run gets a fresh connection rather than a stream that
+/// has gone silent.
+#[tokio::test(start_paused = true)]
+async fn a_quiet_stream_reconnects_when_the_run_is_still_going() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let quiet = cursor.script_stream();
+ quiet
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "thinking".to_owned(),
+ },
+ "evt-1",
+ )
+ .expect("stream open");
+ // The sender stays alive: the connection is open and says nothing.
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Running,
+ text: None,
+ });
+ let resumed = cursor.script_stream();
+ resumed
+ .send(CursorEvent::Assistant {
+ text: " done".to_owned(),
+ })
+ .expect("stream open");
+ resumed.send(finished("run-fake-1")).expect("stream open");
+ resumed.send(CursorEvent::Done).expect("stream open");
+ drop(resumed);
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the reconnect finishes the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ drop(quiet);
+ assert_eq!(
+ cursor.resume_positions(),
+ vec![None, Some("evt-1".to_owned())],
+ "the quiet connection is replaced, not merely polled around"
+ );
+ assert_eq!(agent_texts(&notifier.updates()), vec!["thinking", " done"]);
+}
+
+/// A resume never trips prefix reconciliation on a partially captured run.
+///
+/// The resume position is the last record this connection actually received,
+/// so what comes back starts exactly where the prefix check had got to: the
+/// rest of the captured prefix is matched as usual and only genuinely new
+/// records are appended.
+#[tokio::test(start_paused = true)]
+async fn a_resumed_stream_continues_the_captured_prefix_without_duplicating_it() {
+ let (service, cursor, notifier) = service(None);
+ let id = service.new_session(Path::new(""), vec![]);
+ let session = service.session(&id).expect("session exists");
+ let run = CursorRunId::new("foreign");
+ let asked = NativeRecord {
+ id: Some("evt-1".to_owned()),
+ ..crate::testing::raw_record(CursorEvent::Interaction(InteractionUpdate::UserMessage {
+ text: "asked elsewhere".into(),
+ }))
+ };
+ let half = NativeRecord {
+ id: Some("evt-2".to_owned()),
+ ..crate::testing::raw_record(CursorEvent::Assistant { text: "hel".into() })
+ };
+ {
+ let _gate = session.turn_gate.lock().await;
+ service
+ .ensure_journal(&id, &session)
+ .await
+ .expect("journal");
+ for record in [&asked, &half] {
+ service
+ .capture(
+ &id,
+ &session,
+ Some(&run),
+ JournalInput::Sse(record.clone()),
+ false,
+ )
+ .await
+ .expect("capture");
+ }
+ // The first connection re-reads only the head of the captured prefix
+ // before the transport breaks.
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped.send_record(asked).expect("stream open");
+ drop(dropped);
+ // The resume picks up at the rest of the prefix and runs past it.
+ let resumed = cursor.script_stream();
+ resumed.send_record(half).expect("stream open");
+ resumed
+ .send(CursorEvent::Assistant { text: "lo".into() })
+ .expect("stream open");
+ resumed.send(finished("foreign")).expect("stream open");
+ resumed.send(CursorEvent::Done).expect("stream open");
+ drop(resumed);
+ service
+ .ingest_run(
+ &id,
+ &session,
+ &CursorAgentId::new("agent"),
+ &run,
+ &tokio_util::sync::CancellationToken::new(),
+ IngestMode::LIVE,
+ )
+ .await
+ .expect("a resumed stream reconciles");
+ }
+ assert_eq!(
+ cursor.resume_positions(),
+ vec![None, Some("evt-1".to_owned())]
+ );
+ assert_eq!(
+ agent_texts(&notifier.updates()),
+ vec!["lo"],
+ "only the records past the captured prefix are delivered"
+ );
+ let entries = service.journal.read(&id).await.expect("journal");
+ assert_eq!(
+ entries
+ .iter()
+ .filter(|entry| matches!(&entry.input, JournalInput::Sse(record)
+ if matches!(record.decode(), CursorEvent::Assistant { text } if text == "hel")))
+ .count(),
+ 1,
+ "the matched prefix is not captured a second time"
+ );
+}
diff --git a/crates/cursor_cloud_agents/src/testing.rs b/crates/cursor_cloud_agents/src/testing.rs
index 5428d9b699..f349dc8214 100644
--- a/crates/cursor_cloud_agents/src/testing.rs
+++ b/crates/cursor_cloud_agents/src/testing.rs
@@ -7,7 +7,8 @@ use crate::domain::model::{
RunOutcome,
};
use crate::domain::ports::{
- CursorAgents, RepositoryChooser, RunStream, SessionIntent, SessionNotifier,
+ ConnectedStream, CursorAgents, RepositoryChooser, RunStream, SessionIntent, SessionNotifier,
+ StreamConnectError,
};
use agent_client_protocol::schema::v1::{SessionId, SessionUpdate};
use futures::Stream;
@@ -115,6 +116,27 @@ impl ScriptSender {
pub fn send(&self, event: CursorEvent) -> Result<(), mpsc::error::SendError<NativeRecord>> {
self.0.send(raw_record(event))
}
+
+ /// Enqueue an already-built native record, ids and all.
+ pub fn send_record(
+ &self,
+ record: NativeRecord,
+ ) -> Result<(), mpsc::error::SendError<NativeRecord>> {
+ self.0.send(record)
+ }
+
+ /// Enqueue an event carrying a provider event id, the way Cursor stamps
+ /// the records a resume position can point at.
+ pub fn send_with_id(
+ &self,
+ event: CursorEvent,
+ id: &str,
+ ) -> Result<(), mpsc::error::SendError<NativeRecord>> {
+ self.0.send(NativeRecord {
+ id: Some(id.to_owned()),
+ ..raw_record(event)
+ })
+ }
}
/// What a [`FakeCursor`] was asked to do.
@@ -148,8 +170,12 @@ pub struct FakeCursor {
struct FakeCursorState {
calls: Vec<CursorCall>,
next_run: u64,
- /// The receiver the next `raw_stream()` call will drain.
- streams: Vec<mpsc::UnboundedReceiver<NativeRecord>>,
+ /// What the next `raw_stream()` calls answer with, consumed in order.
+ streams: Vec<ScriptedStream>,
+ /// The resume position every `raw_stream()` call received, in order.
+ resume_positions: Vec<Option<String>>,
+ /// The retention window every connected stream reports.
+ retention_seconds: Option<u64>,
/// Answers for `run_result`, consumed in order.
run_results: Vec<RunOutcome>,
/// The answer every `list_runs` call gets.
@@ -182,12 +208,49 @@ impl FakeCursor {
/// Queue original native records, including recorded wire fixtures.
pub fn script_raw_stream(&self) -> mpsc::UnboundedSender<NativeRecord> {
+ self.queue_stream(None)
+ }
+
+ /// Queue a stream that dies with `failure` once its sender is dropped —
+ /// the mid-run transport break, after whatever records were sent first.
+ pub fn script_stream_failing_with(&self, failure: &str) -> ScriptSender {
+ ScriptSender(self.queue_stream(Some(failure.to_owned())))
+ }
+
+ /// Queue a connect that fails instead of producing a stream.
+ pub fn script_stream_connect_error(&self, error: StreamConnectError) {
+ self.inner
+ .lock()
+ .expect("fake cursor poisoned")
+ .streams
+ .push(ScriptedStream::ConnectError(error));
+ }
+
+ /// Report `seconds` as the retention window on every connected stream.
+ pub fn script_stream_retention(&self, seconds: u64) {
+ self.inner
+ .lock()
+ .expect("fake cursor poisoned")
+ .retention_seconds = Some(seconds);
+ }
+
+ /// The resume position each `raw_stream()` call was given, in order.
+ #[must_use]
+ pub fn resume_positions(&self) -> Vec<Option<String>> {
+ self.inner
+ .lock()
+ .expect("fake cursor poisoned")
+ .resume_positions
+ .clone()
+ }
+
+ fn queue_stream(&self, failure: Option<String>) -> mpsc::UnboundedSender<NativeRecord> {
let (sender, receiver) = mpsc::unbounded_channel();
self.inner
.lock()
.expect("fake cursor poisoned")
.streams
- .push(receiver);
+ .push(ScriptedStream::Records { receiver, failure });
sender
}
@@ -429,23 +492,59 @@ impl CursorAgents for FakeCursor {
}
}
+/// What one scripted `raw_stream()` call answers with.
+#[derive(Debug)]
+enum ScriptedStream {
+ /// Records the test pushes, optionally ending in a transport failure once
+ /// the sender is dropped.
+ Records {
+ receiver: mpsc::UnboundedReceiver<NativeRecord>,
+ failure: Option<String>,
+ },
+ /// A connect that never produces a stream.
+ ConnectError(StreamConnectError),
+}
+
impl RunStream for FakeCursor {
async fn raw_stream(
&self,
_agent: &CursorAgentId,
_run: &CursorRunId,
- ) -> Result<impl Stream<Item = Result<NativeRecord, rootcause::Report>> + Send, rootcause::Report>
- {
- let receiver = {
+ resume_from: Option<&str>,
+ ) -> Result<
+ ConnectedStream<impl Stream<Item = Result<NativeRecord, rootcause::Report>> + Send>,
+ StreamConnectError,
+ > {
+ let (receiver, failure, retention_seconds) = {
let mut state = self.inner.lock().expect("fake cursor poisoned");
+ state.resume_positions.push(resume_from.map(str::to_owned));
if state.streams.is_empty() {
- return Err(rootcause::report!("no scripted stream queued"));
+ return Err(StreamConnectError::Other(rootcause::report!(
+ "no scripted stream queued"
+ )));
+ }
+ let retention_seconds = state.retention_seconds;
+ match state.streams.remove(0) {
+ ScriptedStream::ConnectError(error) => return Err(error),
+ ScriptedStream::Records { receiver, failure } => {
+ (receiver, failure, retention_seconds)
+ }
}
- state.streams.remove(0)
};
- Ok(futures::stream::unfold(receiver, |mut receiver| async {
- receiver.recv().await.map(|event| (Ok(event), receiver))
- }))
+ let records =
+ futures::stream::unfold((receiver, failure), |(mut receiver, failure)| async move {
+ match receiver.recv().await {
+ Some(event) => Some((Ok(event), (receiver, failure))),
+ // The channel closing is the connection closing: a
+ // scripted failure is what the transport says as it goes.
+ None => failure
+ .map(|failure| (Err(rootcause::report!("{failure}")), (receiver, None))),
+ }
+ });
+ Ok(ConnectedStream {
+ records,
+ retention_seconds,
+ })
}
}