Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
chore(agents): cursor SSE ingest: diagnose body failures, survive stalled resumes, log MCP stream ends (#7067)
Wolf Mermelstein4e392cea0c7a
diff --git a/Cargo.lock b/Cargo.lockindex a2c137e7da..ef327a587e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -265,8 +265,10 @@ dependencies = ["base64 0.22.1","bytes","chrono",+ "futures","github","http 1.4.0",+ "http-body 1.0.1","http-body-util","lru 0.16.4","macro_auth",@@ -286,6 +288,7 @@ dependencies = ["tower 0.5.3","tracing","url",+ "uuid","workspace-hack",]@@ -5186,6 +5189,7 @@ dependencies = ["agent_fold","agent_runtime_protocol","agent_session",+ "base64 0.22.1","bytes","futures","insta",diff --git a/Cargo.toml b/Cargo.tomlindex 564b8bb4a6..89e11d7f59 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -242,6 +242,7 @@ hex = "0.4.3"hmac = "0.12.1"encoding_rs = "0.8"http = "1"+http-body = "1.0.1"http-body-util = "0.1.2"httpdate = "1"image = "0.25"diff --git a/crates/agent_egress/Cargo.toml b/crates/agent_egress/Cargo.tomlindex ae9d860484..778f71f9bb 100644--- a/crates/agent_egress/Cargo.toml+++ b/crates/agent_egress/Cargo.toml@@ -25,6 +25,8 @@ github = { path = "../github", default-features = false, features = ["sync",] }http = { workspace = true }+# Wrapping upstream event-stream bodies so their lifetime and volume get logged.+http-body = { workspace = true }http-body-util = { workspace = true }lru = { workspace = true }# Signing Macro API tokens, with the same shared vocabulary@@ -51,5 +53,7 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]anyhow = { workspace = true }+futures = { workspace = true }tokio = { workspace = true, features = ["macros", "rt"] }+uuid = { workspace = true }tower = { workspace = true }diff --git a/crates/agent_egress/src/domain/mod.rs b/crates/agent_egress/src/domain/mod.rsindex 024cc230b7..bb31ddf103 100644--- a/crates/agent_egress/src/domain/mod.rs+++ b/crates/agent_egress/src/domain/mod.rs@@ -5,6 +5,8 @@ pub mod error;/// Vocabulary: grants, slugs, targets, and the transport-neutral request and/// response the service passes through.pub mod model;+/// A response body that logs how long it lived and how it ended.+pub mod observed_body;/// The capabilities the service needs from the outside.pub mod ports;/// The service itself.diff --git a/crates/agent_egress/src/domain/observed_body.rs b/crates/agent_egress/src/domain/observed_body.rsnew file mode 100644index 0000000000..2c5b873d17--- /dev/null+++ b/crates/agent_egress/src/domain/observed_body.rs@@ -0,0 +1,123 @@+//! A response body that writes down how it ended.+//!+//! The proxy answers a GET for an MCP event stream with the upstream's body+//! and hears nothing more about it: how long it stayed open, how much it+//! carried, and whether the upstream closed it or the sandbox went away all+//! happen after the handler has returned. When one session reopened a+//! stream 380 times in a row (prod, 2026-09-26) the logs could say only+//! that each open was proxied; not what the upstream answered, nor how+//! quickly each stream ended. This wrapper is what says so.++#[cfg(test)]+mod test;++use std::pin::Pin;+use std::task::{Context, Poll};+use std::time::Instant;++use bytes::Bytes;+use http_body::{Body, Frame};++use crate::domain::model::{AgentSessionId, BoxError, ProxyBody};++/// Facts about the stream that are known when it opens, for the line+/// written when it ends.+#[derive(Debug, Clone)]+pub struct StreamIdentity {+ /// The session that opened it.+ pub session: AgentSessionId,+ /// The upstream it was opened on.+ pub upstream: String,+ /// The status the upstream answered with.+ pub status: u16,+ /// The upstream's `content-type`, `text/event-stream` for a live stream.+ pub content_type: Option<String>,+}++/// A body that logs its lifetime when it ends or is dropped.+pub struct ObservedBody {+ inner: ProxyBody,+ identity: StreamIdentity,+ opened_at: Instant,+ bytes: u64,+ frames: u64,+ /// Whether the upstream ended the body, as opposed to the sandbox+ /// dropping it - or the body failing - before that.+ ended: bool,+ failed: bool,+}++impl ObservedBody {+ /// Wrap `inner`, to be logged as `identity` when it ends.+ pub fn new(inner: ProxyBody, identity: StreamIdentity) -> Self {+ Self {+ inner,+ identity,+ opened_at: Instant::now(),+ bytes: 0,+ frames: 0,+ ended: false,+ failed: false,+ }+ }++ /// How the stream ended, in one word.+ fn outcome(&self) -> &'static str {+ if self.failed {+ "failed"+ } else if self.ended {+ "closed by upstream"+ } else {+ "dropped by sandbox"+ }+ }+}++impl Body for ObservedBody {+ type Data = Bytes;+ type Error = BoxError;++ fn poll_frame(+ mut self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {+ let polled = Pin::new(&mut self.inner).poll_frame(cx);+ match &polled {+ Poll::Ready(Some(Ok(frame))) => {+ self.frames += 1;+ if let Some(data) = frame.data_ref() {+ self.bytes += data.len() as u64;+ }+ }+ Poll::Ready(Some(Err(_))) => self.failed = true,+ Poll::Ready(None) => self.ended = true,+ Poll::Pending => {}+ }+ polled+ }++ fn is_end_stream(&self) -> bool {+ self.inner.is_end_stream()+ }++ fn size_hint(&self) -> http_body::SizeHint {+ self.inner.size_hint()+ }+}++impl Drop for ObservedBody {+ fn drop(&mut self) {+ let open_ms = self.opened_at.elapsed().as_millis() as u64;+ tracing::info!(+ session = %self.identity.session,+ upstream = %self.identity.upstream,+ status = self.identity.status,+ content_type = self.identity.content_type.as_deref(),+ outcome = self.outcome(),+ open_ms,+ bytes = self.bytes,+ frames = self.frames,+ "MCP event stream ended"+ );+ }+}diff --git a/crates/agent_egress/src/domain/observed_body/test.rs b/crates/agent_egress/src/domain/observed_body/test.rsnew file mode 100644index 0000000000..e8ee9d7413--- /dev/null+++ b/crates/agent_egress/src/domain/observed_body/test.rs@@ -0,0 +1,55 @@+use super::*;+use http_body_util::{BodyExt, Full, StreamBody};++fn identity() -> StreamIdentity {+ StreamIdentity {+ session: AgentSessionId::new(),+ upstream: "datadog".to_owned(),+ status: 200,+ content_type: Some("text/event-stream".to_owned()),+ }+}++#[tokio::test]+async fn a_body_read_to_its_end_counts_what_passed_and_knows_it_ended() {+ let inner: ProxyBody = Full::new(Bytes::from_static(b"data: hello\n\n"))+ .map_err(|never| match never {})+ .boxed_unsync();+ let mut observed = ObservedBody::new(inner, identity());+ let mut collected = Vec::new();+ while let Some(frame) = observed.frame().await {+ if let Ok(data) = frame.expect("a full body does not fail").into_data() {+ collected.extend_from_slice(&data);+ }+ }+ assert_eq!(collected, b"data: hello\n\n");+ assert_eq!(observed.bytes, 13);+ assert_eq!(observed.frames, 1);+ assert!(observed.ended);+ assert_eq!(observed.outcome(), "closed by upstream");+}++#[tokio::test]+async fn a_body_dropped_early_is_known_to_have_been_dropped() {+ let inner: ProxyBody = Full::new(Bytes::from_static(b"data: hello\n\n"))+ .map_err(|never| match never {})+ .boxed_unsync();+ let observed = ObservedBody::new(inner, identity());+ assert!(!observed.ended);+ assert_eq!(observed.outcome(), "dropped by sandbox");+}++#[tokio::test]+async fn a_body_that_fails_is_known_to_have_failed() {+ let failing = futures::stream::iter(vec![+ Ok::<_, BoxError>(Frame::data(Bytes::from_static(b"data: a\n\n"))),+ Err("connection reset".into()),+ ]);+ let inner: ProxyBody = StreamBody::new(failing).boxed_unsync();+ let mut observed = ObservedBody::new(inner, identity());+ assert!(observed.frame().await.expect("a frame").is_ok());+ assert!(observed.frame().await.expect("an error").is_err());+ assert!(observed.failed);+ assert_eq!(observed.outcome(), "failed");+ assert_eq!(observed.bytes, 9);+}diff --git a/crates/agent_egress/src/domain/service.rs b/crates/agent_egress/src/domain/service.rsindex 5b3a03ddd3..68df7667f7 100644--- a/crates/agent_egress/src/domain/service.rs+++ b/crates/agent_egress/src/domain/service.rs@@ -11,6 +11,7 @@ use crate::domain::model::{is_macro_staff, not_connected_tool_result, peek_json_rpc, sanitize_request_headers,sanitize_response_headers,};+use crate::domain::observed_body::{ObservedBody, StreamIdentity};use crate::domain::ports::{Forwarder, GithubTokens, McpCredentials, SessionAuthority};#[cfg(test)]@@ -217,6 +218,13 @@ whererequest.headers_mut().insert(name.clone(), value.clone());}+ // A GET on an MCP target opens the server's event stream. Those are+ // the requests a client reopens in a loop when the upstream keeps+ // closing them, so its body is watched to its end - the one place the+ // upstream's side of the loop can be seen.+ let event_stream =+ matches!(target, EgressTarget::McpServer(_)) && *request.method() == Method::GET;+let mut response = self.forward.forward(request).await?;sanitize_response_headers(response.headers_mut());@@ -233,6 +241,27 @@ where);}+ if event_stream {+ let identity = StreamIdentity {+ session: grant.session,+ upstream: target.name(),+ status: response.status().as_u16(),+ content_type: response+ .headers()+ .get(http::header::CONTENT_TYPE)+ .and_then(|value| value.to_str().ok())+ .map(str::to_owned),+ };+ tracing::info!(+ session = %identity.session,+ upstream = %identity.upstream,+ status = identity.status,+ content_type = identity.content_type.as_deref(),+ "MCP event stream opened"+ );+ return Ok(response.map(|body| ObservedBody::new(body, identity).boxed_unsync()));+ }+Ok(response)}}diff --git a/crates/cursor_cloud_agents/Cargo.toml b/crates/cursor_cloud_agents/Cargo.tomlindex 7b1ff8a795..bac4ac75ea 100644--- a/crates/cursor_cloud_agents/Cargo.toml+++ b/crates/cursor_cloud_agents/Cargo.toml@@ -21,6 +21,8 @@ agent_runtime_protocol = { path = "../agent_runtime_protocol", default-featuressqlx = { workspace = true }uuid = { workspace = true }agent-client-protocol = { workspace = true }+# The raw tail of a failed stream body, logged as it was on the wire.+base64 = { workspace = true }bytes = { workspace = true }static_file_service_client = { path = "../static_file_service_client" }futures = { workspace = true }diff --git a/crates/cursor_cloud_agents/src/api.rs b/crates/cursor_cloud_agents/src/api.rsindex f5b72e43d0..afa0cc2b67 100644--- a/crates/cursor_cloud_agents/src/api.rs+++ b/crates/cursor_cloud_agents/src/api.rs@@ -17,10 +17,14 @@ mod test;/// Capturing raw SSE bytes as fixtures.pub mod record;+/// Reading a stream's body, and the report written when it fails.+mod stream;+/// Request/response DTOs for the endpoints this crate uses.pub mod wire;use crate::api::record::SseRecording;+use crate::api::stream::ConnectionFacts;use crate::api::wire::{AgentSummary, ArchiveAgentResponse, ArtifactDownloadResponse, ConversationResponse,CreateAgentRequest, CreateAgentResponse, CreateRunRequest, CreateRunResponse,@@ -35,9 +39,7 @@ use crate::domain::model::{use crate::domain::ports::{ConnectedStream, CursorAgents, CursorArtifacts, RunStream, StreamConnectError,};-use futures::{Stream, StreamExt as _};-use sse_core::SseEvent;-use std::collections::VecDeque;+use futures::Stream;use std::num::NonZeroUsize;/// The largest single SSE record payload this agent will buffer, 16 MiB.@@ -924,6 +926,17 @@ const RETENTION_HEADER: &str = "x-cursor-stream-retention-seconds";/// The SSE resume header. Not in `http`'s constant set, so it is spelled here.const LAST_EVENT_ID_HEADER: &str = "last-event-id";+/// How long a stream connect waits for the response headers before it is a+/// [`StreamConnectError::Stalled`].+///+/// A stream's headers ordinarily arrive within the second. The one case that+/// takes longer is the one this bounds: a resume the server held open for+/// minutes and answered all at once when the run ended (prod, 2026-09-26).+/// Thirty seconds is far past any honest handshake and short enough that+/// the domain, which checks the run's record on every stall, notices a run+/// that has finished behind a wedged stream within the minute.+const STREAM_HEADERS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);+impl RunStream for CursorClient {// Covers connecting the stream, retries included; reading it belongs to// the caller's `cursor.run.ingest` span.@@ -970,18 +983,46 @@ impl CursorClient {>,StreamConnectError,> {+ // `identity` is stated rather than left to the default. This client+ // is built without reqwest's decompression features, so it would+ // never decode a compressed body — but it would not have asked for+ // one either, and an intermediary that compresses an event stream+ // when nothing forbids it buffers records until a block is full.+ // Asking for identity forbids it, and a `content-encoding` on the+ // answer becomes a fact worth a warning rather than a guess.let mut request = self.http.get(self.url(&format!("/v1/agents/{agent}/runs/{run}/stream"))).basic_auth(self.config.api_key.expose(), Some(""))- .header(reqwest::header::ACCEPT, "text/event-stream");+ .header(reqwest::header::ACCEPT, "text/event-stream")+ .header(reqwest::header::ACCEPT_ENCODING, "identity");if let Some(resume_from) = resume_from {request = request.header(LAST_EVENT_ID_HEADER, resume_from);}- let response = request- .send()- .await- .map_err(|error| StreamConnectError::Other(rootcause::report!(error).into()))?;+ let asked_at = std::time::Instant::now();+ let response = match tokio::time::timeout(STREAM_HEADERS_TIMEOUT, request.send()).await {+ Ok(Ok(response)) => response,+ Ok(Err(error)) => {+ return Err(StreamConnectError::Other(rootcause::report!(+ "{}",+ stream::error_chain(&error)+ )));+ }+ Err(_elapsed) => {+ tracing::warn!(+ cursor.run.id = %run,+ cursor.stream.resumed = resume_from.is_some(),+ cursor.stream.last_event_id = resume_from,+ cursor.stream.headers_timeout_ms = STREAM_HEADERS_TIMEOUT.as_millis() as u64,+ "Cursor sent no stream headers before the timeout"+ );+ return Err(StreamConnectError::Stalled(format!(+ "no response headers after {}s",+ STREAM_HEADERS_TIMEOUT.as_secs()+ )));+ }+ };+ let headers_ms = asked_at.elapsed().as_millis() as u64;let status = response.status();// Read before the body is consumed; a resumed stream's window is the// only thing that says whether the next drop is recoverable at all.@@ -1008,71 +1049,39 @@ impl CursorClient {)));}+ let facts = ConnectionFacts::of(&response, resume_from.is_some());+ tracing::info!(+ cursor.run.id = %run,+ cursor.stream.resumed = facts.resumed,+ cursor.stream.last_event_id = resume_from,+ cursor.stream.headers_ms = headers_ms,+ cursor.stream.status = facts.status,+ cursor.stream.http_version = %facts.version,+ cursor.stream.content_type = facts.content_type.as_deref(),+ cursor.stream.content_encoding = facts.content_encoding.as_deref(),+ cursor.stream.transfer_encoding = facts.transfer_encoding.as_deref(),+ cursor.stream.server = facts.server.as_deref(),+ cursor.stream.request_id = facts.request_id.as_deref(),+ cursor.stream.retention_seconds = retention_seconds,+ "Cursor stream connected"+ );+ if facts.is_compressed() {+ tracing::warn!(+ cursor.run.id = %run,+ cursor.stream.content_encoding = facts.content_encoding.as_deref(),+ cursor.stream.request_id = facts.request_id.as_deref(),+ "Cursor stream arrived compressed despite asking for identity"+ );+ }+// Recording taps the bytes before the decoder sees them, so a// fixture is byte-identical to the wire.let recording = match &self.config.record_dir {Some(dir) => SseRecording::create(dir, agent.as_str(), run.as_str()),None => SseRecording::disabled(),};-- // Decode incrementally: SSE records straddle read boundaries, so the- // decoder holds a partial record in its own buffers and the unfold- // drains whole ones only. One read can complete several records, hence- // the queue.- let state = (- response.bytes_stream().boxed(),- sse_core::SseDecoder::with_limit(MAX_SSE_PAYLOAD),- VecDeque::new(),- recording,- );- let records = futures::stream::try_unfold(- state,- |(mut bytes, mut decoder, mut pending, mut recording)| async move {- loop {- if let Some(event) = pending.pop_front() {- return Ok(Some((event?, (bytes, decoder, pending, recording))));- }- match bytes.next().await {- Some(Ok(chunk)) => {- recording.write(&chunk);- let mut cursor = chunk;- while let Some(record) = decoder.next(&mut cursor) {- // A payload past the limit is the run's- // problem, not this stream's shape: report it- // and stop rather than resync mid-record.- let record = match record {- Ok(record) => record,- Err(error) => {- // Deliver every earlier complete record- // in this chunk before the decoder error.- pending.push_back(Err(rootcause::report!(- "cursor sse payload over {} bytes: {error}",- MAX_SSE_PAYLOAD- )- .into_dynamic()));- break;- }- };- let SseEvent::Message(message) = record else {- continue; // `retry:`; the domain drives reconnects- };- pending.push_back(Ok(crate::domain::journal::NativeRecord {- event: message.event.into_owned(),- data: message.data,- id: message.last_event_id.map(|id| id.to_string()),- }));- }- }- Some(Err(error)) => {- return Err(rootcause::report!(error).into_dynamic());- }- None => return Ok(None),- }- }- },- );Ok(ConnectedStream {- records,+ records: stream::records(response, facts, recording, MAX_SSE_PAYLOAD),retention_seconds,})}diff --git a/crates/cursor_cloud_agents/src/api/stream.rs b/crates/cursor_cloud_agents/src/api/stream.rsnew file mode 100644index 0000000000..5cc89ac9e8--- /dev/null+++ b/crates/cursor_cloud_agents/src/api/stream.rs@@ -0,0 +1,361 @@+//! Reading a run's SSE body, with the diagnostics a failure needs.+//!+//! reqwest reports every mid-body failure as `error decoding response body`+//! and, since 0.12, keeps the cause out of `Display`: the hyper, h2, or io+//! error underneath — a reset, an incomplete message, an h2 stream error —+//! is reachable only through `source()`. Eight runs in one day (prod,+//! 2026-09-26) failed with exactly that sentence and nothing else, which is+//! why this module exists. It keeps the facts of the connection and the tail+//! of its bytes, and writes them all down the moment the body fails.++#[cfg(test)]+mod test;++use crate::api::record::SseRecording;+use crate::domain::journal::NativeRecord;+use base64::Engine as _;+use futures::stream::BoxStream;+use futures::{Stream, StreamExt as _};+use sse_core::SseEvent;+use std::collections::VecDeque;+use std::num::NonZeroUsize;+use std::time::Instant;++/// Raw bytes kept from the tail of a connection, for the report a body+/// failure writes. 4 KiB: several SSE records' worth, enough to see whether+/// the last one was cut mid-frame, small enough to base64 into one log line.+pub(crate) const TAIL_BYTES: usize = 4096;++/// The header Cloudflare stamps a request with; the id to quote to Cursor.+const CF_RAY_HEADER: &str = "cf-ray";++/// Generic request-id headers, tried after [`CF_RAY_HEADER`].+const REQUEST_ID_HEADERS: [&str; 2] = ["x-request-id", "x-amzn-requestid"];++/// What a stream connection looked like when it was made.+///+/// Captured from the response headers before the body is read, because by+/// the time the body fails the response is gone and these are the only+/// things that say which server, over which protocol, with which encoding,+/// was on the other end.+#[derive(Debug, Clone)]+pub(crate) struct ConnectionFacts {+ /// The response status.+ pub status: u16,+ /// The HTTP version negotiated, `HTTP/1.1` or `HTTP/2.0`.+ pub version: String,+ /// `content-type`, expected `text/event-stream`.+ pub content_type: Option<String>,+ /// `content-encoding`. Expected absent: the request asks for identity,+ /// and a compressed event stream is buffered by whatever decodes it.+ pub content_encoding: Option<String>,+ /// `transfer-encoding`, `chunked` over HTTP/1.1.+ pub transfer_encoding: Option<String>,+ /// The `server` header, when one was sent.+ pub server: Option<String>,+ /// `cf-ray`, else `x-request-id`, else `x-amzn-requestid`.+ pub request_id: Option<String>,+ /// Whether the connection resumed from a `Last-Event-ID`.+ pub resumed: bool,+}++impl ConnectionFacts {+ /// Read the facts off a response before its body is consumed.+ pub(crate) fn of(response: &reqwest::Response, resumed: bool) -> Self {+ let header = |name: &str| {+ response+ .headers()+ .get(name)+ .and_then(|value| value.to_str().ok())+ .map(str::to_owned)+ };+ Self {+ status: response.status().as_u16(),+ version: format!("{:?}", response.version()),+ content_type: header(reqwest::header::CONTENT_TYPE.as_str()),+ content_encoding: header(reqwest::header::CONTENT_ENCODING.as_str()),+ transfer_encoding: header(reqwest::header::TRANSFER_ENCODING.as_str()),+ server: header(reqwest::header::SERVER.as_str()),+ request_id: header(CF_RAY_HEADER)+ .or_else(|| REQUEST_ID_HEADERS.iter().find_map(|name| header(name))),+ resumed,+ }+ }++ /// Whether the body arrived compressed, which the request asked it not+ /// to. Worth a warning of its own: a decoder between the socket and the+ /// SSE parser buffers until it has a whole block, and fails at whatever+ /// boundary the server cut.+ pub(crate) fn is_compressed(&self) -> bool {+ self.content_encoding+ .as_deref()+ .is_some_and(|encoding| !encoding.eq_ignore_ascii_case("identity"))+ }+}++/// The last [`TAIL_BYTES`] bytes a connection delivered, in order.+#[derive(Debug, Default)]+pub(crate) struct Tail {+ bytes: VecDeque<u8>,+}++impl Tail {+ /// Remember `chunk`, forgetting whatever is now past the window.+ pub(crate) fn push(&mut self, chunk: &[u8]) {+ if chunk.len() >= TAIL_BYTES {+ self.bytes.clear();+ self.bytes.extend(&chunk[chunk.len() - TAIL_BYTES..]);+ return;+ }+ let overflow = (self.bytes.len() + chunk.len()).saturating_sub(TAIL_BYTES);+ self.bytes.drain(..overflow);+ self.bytes.extend(chunk);+ }++ /// The window's bytes, oldest first.+ pub(crate) fn bytes(&self) -> Vec<u8> {+ self.bytes.iter().copied().collect()+ }+}++/// The bytes after the last blank line: the record a connection was in the+/// middle of when it ended, or nothing when it ended between records.+///+/// SSE ends a record with a blank line, which is `\n\n` or `\r\n\r\n` on the+/// wire; whichever came last is the boundary.+pub(crate) fn partial_frame(bytes: &[u8]) -> &[u8] {+ let after_lf = bytes+ .windows(2)+ .rposition(|window| window == b"\n\n")+ .map(|at| at + 2);+ let after_crlf = bytes+ .windows(4)+ .rposition(|window| window == b"\r\n\r\n")+ .map(|at| at + 4);+ let start = after_lf.max(after_crlf).unwrap_or(0);+ &bytes[start..]+}++/// Every message in an error's `source()` chain, outermost first, joined+/// with `: ` — what `Display` would have said before the convention that+/// each error names only itself.+pub(crate) fn error_chain(error: &(dyn std::error::Error + 'static)) -> String {+ let mut chain = error.to_string();+ let mut source = error.source();+ while let Some(cause) = source {+ let message = cause.to_string();+ // An error that already restates its cause in its own message would+ // otherwise say it twice.+ if !chain.ends_with(&message) {+ chain.push_str(": ");+ chain.push_str(&message);+ }+ source = cause.source();+ }+ chain+}++/// A lossy UTF-8 rendering of `bytes`, with control characters escaped so it+/// stays on one log line.+fn preview(bytes: &[u8]) -> String {+ String::from_utf8_lossy(bytes)+ .chars()+ .flat_map(|character| {+ if character.is_control() && character != ' ' {+ character.escape_default().collect::<Vec<_>>()+ } else {+ vec![character]+ }+ })+ .collect()+}++/// One connection's body, being read.+struct Reader {+ bytes: BoxStream<'static, reqwest::Result<bytes::Bytes>>,+ decoder: sse_core::SseDecoder,+ /// Records decoded from the last chunk and not yet yielded; one read can+ /// complete several.+ pending: VecDeque<Result<NativeRecord, rootcause::Report>>,+ recording: SseRecording,+ facts: ConnectionFacts,+ tail: Tail,+ connected_at: Instant,+ last_chunk_at: Option<Instant>,+ last_record_at: Option<Instant>,+ bytes_received: u64,+ chunks_received: u64,+ records_decoded: u64,+ limit: NonZeroUsize,+}++impl Reader {+ fn since(&self, at: Option<Instant>) -> Option<u64> {+ at.map(|at| at.elapsed().as_millis() as u64)+ }++ /// Write down everything known about a body that just failed, and+ /// answer with the report the domain journals.+ ///+ /// The log line carries the raw tail — base64 for the bytes as they+ /// were, a lossy preview for a person reading it — and the partial SSE+ /// frame the connection was cut in. The report carries only the error+ /// chain and the counts: it becomes a journal entry, and 4 KiB of+ /// base64 per interruption belongs in logs, not in the durable record.+ fn failure(&self, error: &reqwest::Error) -> rootcause::Report {+ let chain = error_chain(error);+ let tail = self.tail.bytes();+ let partial = partial_frame(&tail);+ let connected_ms = self.connected_at.elapsed().as_millis() as u64;+ let since_last_record_ms = self.since(self.last_record_at);+ tracing::warn!(+ cursor.stream.error = %chain,+ cursor.stream.error_is_decode = error.is_decode(),+ cursor.stream.error_is_timeout = error.is_timeout(),+ cursor.stream.error_is_connect = error.is_connect(),+ cursor.stream.status = self.facts.status,+ cursor.stream.http_version = %self.facts.version,+ cursor.stream.content_type = self.facts.content_type.as_deref(),+ cursor.stream.content_encoding = self.facts.content_encoding.as_deref(),+ cursor.stream.transfer_encoding = self.facts.transfer_encoding.as_deref(),+ cursor.stream.server = self.facts.server.as_deref(),+ cursor.stream.request_id = self.facts.request_id.as_deref(),+ cursor.stream.resumed = self.facts.resumed,+ cursor.stream.bytes_received = self.bytes_received,+ cursor.stream.chunks_received = self.chunks_received,+ cursor.stream.records_decoded = self.records_decoded,+ cursor.stream.connected_ms = connected_ms,+ cursor.stream.since_last_chunk_ms = self.since(self.last_chunk_at),+ cursor.stream.since_last_record_ms = since_last_record_ms,+ cursor.stream.last_event_id = self.decoder.last_event_id().map(|id| id.to_string()),+ cursor.stream.tail_bytes = tail.len(),+ cursor.stream.tail_base64 = %base64::engine::general_purpose::STANDARD.encode(&tail),+ cursor.stream.tail_preview = %preview(&tail),+ cursor.stream.partial_frame_bytes = partial.len(),+ cursor.stream.partial_frame = %preview(partial),+ cursor.stream.decoder = ?self.decoder,+ "Cursor stream body failed"+ );+ rootcause::report!(+ "{chain} (after {} bytes and {} records in {}s; {} since the last record)",+ self.bytes_received,+ self.records_decoded,+ connected_ms / 1000,+ since_last_record_ms+ .map(|ms| format!("{}s", ms / 1000))+ .unwrap_or_else(|| "none received".to_owned()),+ )+ .into_dynamic()+ }++ /// Note how a body ended. A partial frame at the end means the server+ /// cut a record in two, which the domain is about to hear as a plain+ /// close: worth a warning, so the two can be told apart later.+ fn ended(&self) {+ let tail = self.tail.bytes();+ let partial = partial_frame(&tail);+ if partial.is_empty() {+ tracing::debug!(+ cursor.stream.bytes_received = self.bytes_received,+ cursor.stream.records_decoded = self.records_decoded,+ cursor.stream.connected_ms = self.connected_at.elapsed().as_millis() as u64,+ cursor.stream.resumed = self.facts.resumed,+ "Cursor stream body ended"+ );+ return;+ }+ tracing::warn!(+ cursor.stream.bytes_received = self.bytes_received,+ cursor.stream.records_decoded = self.records_decoded,+ cursor.stream.connected_ms = self.connected_at.elapsed().as_millis() as u64,+ cursor.stream.resumed = self.facts.resumed,+ cursor.stream.request_id = self.facts.request_id.as_deref(),+ cursor.stream.partial_frame_bytes = partial.len(),+ cursor.stream.partial_frame = %preview(partial),+ "Cursor stream body ended mid-record"+ );+ }++ /// Decode one chunk into `pending`, whole records only.+ fn decode(&mut self, chunk: bytes::Bytes) {+ self.chunks_received += 1;+ self.bytes_received += chunk.len() as u64;+ self.last_chunk_at = Some(Instant::now());+ self.recording.write(&chunk);+ self.tail.push(&chunk);+ let mut cursor = chunk;+ while let Some(record) = self.decoder.next(&mut cursor) {+ // A payload past the limit is the run's problem, not this+ // stream's shape: report it and stop rather than resync+ // mid-record.+ let record = match record {+ Ok(record) => record,+ Err(error) => {+ // Deliver every earlier complete record in this chunk+ // before the decoder error.+ self.pending.push_back(Err(rootcause::report!(+ "cursor sse payload over {} bytes: {error}",+ self.limit+ )+ .into_dynamic()));+ break;+ }+ };+ let SseEvent::Message(message) = record else {+ continue; // `retry:`; the domain drives reconnects+ };+ self.records_decoded += 1;+ self.last_record_at = Some(Instant::now());+ self.pending.push_back(Ok(NativeRecord {+ event: message.event.into_owned(),+ data: message.data,+ id: message.last_event_id.map(|id| id.to_string()),+ }));+ }+ }+}++/// The complete SSE records of `response`'s body, as a stream.+///+/// Decodes incrementally: SSE records straddle read boundaries, so the+/// decoder holds a partial record in its own buffers and only whole ones are+/// yielded. `recording` taps the bytes before the decoder sees them, so a+/// fixture is byte-identical to the wire. `limit` bounds one record's+/// payload; see [`crate::api::MAX_SSE_PAYLOAD`].+pub(crate) fn records(+ response: reqwest::Response,+ facts: ConnectionFacts,+ recording: SseRecording,+ limit: NonZeroUsize,+) -> impl Stream<Item = Result<NativeRecord, rootcause::Report>> + Send {+ let reader = Reader {+ bytes: response.bytes_stream().boxed(),+ decoder: sse_core::SseDecoder::with_limit(limit),+ pending: VecDeque::new(),+ recording,+ facts,+ tail: Tail::default(),+ connected_at: Instant::now(),+ last_chunk_at: None,+ last_record_at: None,+ bytes_received: 0,+ chunks_received: 0,+ records_decoded: 0,+ limit,+ };+ futures::stream::try_unfold(reader, |mut reader| async move {+ loop {+ if let Some(record) = reader.pending.pop_front() {+ return Ok(Some((record?, reader)));+ }+ match reader.bytes.next().await {+ Some(Ok(chunk)) => reader.decode(chunk),+ Some(Err(error)) => return Err(reader.failure(&error)),+ None => {+ reader.ended();+ return Ok(None);+ }+ }+ }+ })+}diff --git a/crates/cursor_cloud_agents/src/api/stream/test.rs b/crates/cursor_cloud_agents/src/api/stream/test.rsnew file mode 100644index 0000000000..809509ef85--- /dev/null+++ b/crates/cursor_cloud_agents/src/api/stream/test.rs@@ -0,0 +1,122 @@+use super::*;++#[test]+fn the_tail_keeps_the_last_window_of_bytes_in_order() {+ let mut tail = Tail::default();+ tail.push(b"abc");+ tail.push(b"def");+ assert_eq!(tail.bytes(), b"abcdef");++ let filler = vec![b'x'; TAIL_BYTES - 4];+ tail.push(&filler);+ let bytes = tail.bytes();+ assert_eq!(bytes.len(), TAIL_BYTES);+ assert!(+ bytes.starts_with(b"cdef"),+ "the oldest bytes past the window are forgotten first"+ );++ let oversized = vec![b'y'; TAIL_BYTES + 10];+ tail.push(&oversized);+ assert_eq!(tail.bytes(), vec![b'y'; TAIL_BYTES]);+}++#[test]+fn the_partial_frame_is_whatever_follows_the_last_blank_line() {+ assert_eq!(+ partial_frame(b"event: a\ndata: 1\n\nevent: b\ndata: 2"),+ b"event: b\ndata: 2"+ );+ assert_eq!(+ partial_frame(b"event: a\r\ndata: 1\r\n\r\ndata: {\"half"),+ b"data: {\"half"+ );+ assert_eq!(partial_frame(b"event: a\ndata: 1\n\n"), b"");+ assert_eq!(+ partial_frame(b"no boundary yet"),+ b"no boundary yet",+ "a connection cut inside its first record is all partial frame"+ );+ assert_eq!(+ partial_frame(b"data: 1\r\n\r\ndata: 2\n\ndata: 3"),+ b"data: 3",+ "the later of the two line endings' boundaries wins"+ );+}++#[derive(Debug)]+struct Layer {+ message: &'static str,+ source: Option<Box<Layer>>,+}++impl std::fmt::Display for Layer {+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str(self.message)+ }+}++impl std::error::Error for Layer {+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {+ self.source+ .as_deref()+ .map(|layer| layer as &(dyn std::error::Error + 'static))+ }+}++/// The whole point: `error decoding response body` on its own names+/// nothing, and the reset or h2 error underneath is only in the chain.+#[test]+fn the_error_chain_names_every_cause() {+ let error = Layer {+ message: "error decoding response body",+ source: Some(Box::new(Layer {+ message: "error reading a body from connection",+ source: Some(Box::new(Layer {+ message: "connection reset by peer (os error 104)",+ source: None,+ })),+ })),+ };+ assert_eq!(+ error_chain(&error),+ "error decoding response body: error reading a body from connection: connection reset by peer (os error 104)"+ );+}++#[test]+fn a_cause_already_quoted_by_its_parent_is_not_repeated() {+ let error = Layer {+ message: "outer: inner",+ source: Some(Box::new(Layer {+ message: "inner",+ source: None,+ })),+ };+ assert_eq!(error_chain(&error), "outer: inner");+}++#[test]+fn compression_is_anything_but_identity() {+ let facts = |encoding: Option<&str>| ConnectionFacts {+ status: 200,+ version: "HTTP/2.0".to_owned(),+ content_type: Some("text/event-stream".to_owned()),+ content_encoding: encoding.map(str::to_owned),+ transfer_encoding: None,+ server: None,+ request_id: None,+ resumed: false,+ };+ assert!(!facts(None).is_compressed());+ assert!(!facts(Some("identity")).is_compressed());+ assert!(!facts(Some("Identity")).is_compressed());+ assert!(facts(Some("gzip")).is_compressed());+ assert!(facts(Some("br")).is_compressed());+}++#[test]+fn the_preview_keeps_control_characters_on_one_line() {+ assert_eq!(preview(b"data: a\r\n\r\n"), "data: a\\r\\n\\r\\n");+ assert_eq!(preview(&[0xff, b'o', b'k']), "\u{fffd}ok");+}diff --git a/crates/cursor_cloud_agents/src/domain/journal.rs b/crates/cursor_cloud_agents/src/domain/journal.rsindex 5921e1aa0c..514aa05262 100644--- a/crates/cursor_cloud_agents/src/domain/journal.rs+++ b/crates/cursor_cloud_agents/src/domain/journal.rs@@ -124,6 +124,12 @@ struct RunState {/// What the reader sees of `text`: the same stream minus the `<img>`/// tags Cursor writes for files only its sandbox can reach.images: InlineImageFilter,+ /// Whether the transport broke after the last content record, so that+ /// `text` may be missing its end. Set by an interruption, cleared by+ /// the next content record: a resume that continues the stream leaves+ /// no gap, and only a run whose outcome then arrives by polling is+ /// still behind when its final text is compared.+ interrupted: bool,}/// Complete live/replay state, including user prompts and terminal tool cleanup.#[derive(Debug, Default)]@@ -204,7 +210,12 @@ impl ReplayMachine {.map(|b| SessionUpdate::UserMessageChunk(ContentChunk::new(b))).collect())}- JournalInput::Sse(record) => self.event(run, record.decode()),+ JournalInput::Sse(record) => {+ if record.is_content() {+ state.interrupted = false;+ }+ self.event(run, record.decode())+ }JournalInput::ArtifactsCollected(artifacts) => {if artifacts.is_empty() {return Ok(Vec::new());@@ -251,12 +262,14 @@ impl ReplayMachine {// This closes local work, but does not claim remote capture is complete.Ok(self.translator.close_open_calls())}+ JournalInput::TransportError(_) | JournalInput::StreamInterrupted { .. } => {+ state.interrupted = true;+ Ok(Vec::new())+ }JournalInput::HistoryComplete| JournalInput::Reconciled| JournalInput::PromptAccepted(_)- | JournalInput::PromptAborted(_)- | JournalInput::TransportError(_)- | JournalInput::StreamInterrupted { .. } => Ok(Vec::new()),+ | JournalInput::PromptAborted(_) => Ok(Vec::new()),}}fn event(@@ -335,6 +348,29 @@ impl ReplayMachine {}));state.text = text;}+ // The stream broke and nothing came after it, so the+ // captured text is the front of an answer whose end+ // was never streamed. Streamed chunks cannot be taken+ // back, so the whole answer follows as its own+ // passage: the user sees the fragment and then the+ // answer, rather than the fragment alone.+ None if state.interrupted => {+ tracing::warn!(+ %run,+ captured = state.text.len(),+ restated = text.len(),+ "Cursor restated the answer after a stream gap; appending it whole"+ );+ let separator = if state.text.is_empty() || state.text.ends_with('\n') {+ ""+ } else {+ "\n\n"+ };+ updates.extend(self.translator.push(CursorEvent::Assistant {+ text: state.images.push(&format!("{separator}{text}")),+ }));+ state.text = text;+ }None => tracing::warn!(%run,captured = state.text.len(),diff --git a/crates/cursor_cloud_agents/src/domain/journal/test.rs b/crates/cursor_cloud_agents/src/domain/journal/test.rsindex 5740ec5a62..2253b27046 100644--- a/crates/cursor_cloud_agents/src/domain/journal/test.rs+++ b/crates/cursor_cloud_agents/src/domain/journal/test.rs@@ -400,6 +400,106 @@ fn a_stream_interruption_projects_to_nothing() {);}+/// The stream broke, nothing came after it, and the run's outcome arrived by+/// polling: the captured text is the front of an answer whose end was never+/// streamed, and a restatement that does not continue it is that answer, not+/// a rewrite. It follows as its own passage rather than being dropped.+#[test]+fn a_restated_answer_after_a_stream_gap_is_appended_whole() {+ let run = CursorRunId::new("gap");+ let mut machine = ReplayMachine::default();+ machine+ .push(+ Some(&run),+ &JournalInput::Sse(crate::testing::raw_record(CursorEvent::Assistant {+ text: "Looking into".into(),+ })),+ )+ .unwrap();+ machine+ .push(+ Some(&run),+ &JournalInput::StreamInterrupted {+ reason: "error decoding response body".into(),+ last_event_id: Some("evt-1".into()),+ attempt: 1,+ },+ )+ .unwrap();+ let updates = machine+ .push(+ Some(&run),+ &JournalInput::Poll(+ serde_json::json!({"status": "FINISHED", "result": "The bell icon was hidden by a stale flag."})+ .to_string(),+ ),+ )+ .unwrap();+ let text: String = updates+ .iter()+ .filter_map(|update| match update {+ SessionUpdate::AgentMessageChunk(chunk) => match &chunk.content {+ ContentBlock::Text(text) => Some(text.text.clone()),+ _ => None,+ },+ _ => None,+ })+ .collect();+ assert_eq!(+ text, "\n\nThe bell icon was hidden by a stale flag.",+ "the whole answer follows the fragment as a fresh passage"+ );+ assert_eq!(+ machine.runs[&run].text, "The bell icon was hidden by a stale flag.",+ "the answer of record is the restated one"+ );+ assert_eq!(machine.terminal_status(&run), Some(RunStatus::Finished));+}++/// A resume that continued the stream leaves no gap: content after the+/// interruption clears it, and a restatement that then differs is Cursor's+/// rewrite - inline images dropped - which the streamed text still outranks.+#[test]+fn a_resumed_stream_has_no_gap_to_fill() {+ let run = CursorRunId::new("resumed");+ let mut machine = ReplayMachine::default();+ machine+ .push(+ Some(&run),+ &JournalInput::Sse(crate::testing::raw_record(CursorEvent::Assistant {+ text: "captured ".into(),+ })),+ )+ .unwrap();+ machine+ .push(+ Some(&run),+ &JournalInput::StreamInterrupted {+ reason: "error decoding response body".into(),+ last_event_id: Some("evt-1".into()),+ attempt: 1,+ },+ )+ .unwrap();+ machine+ .push(+ Some(&run),+ &JournalInput::Sse(crate::testing::raw_record(CursorEvent::Assistant {+ text: "answer".into(),+ })),+ )+ .unwrap();+ machine+ .push(+ Some(&run),+ &JournalInput::Poll(+ serde_json::json!({"status": "FINISHED", "result": "different answer"}).to_string(),+ ),+ )+ .unwrap();+ assert_eq!(machine.runs[&run].text, "captured answer");+}+/// The screenshot turn of dev session 01a0b4ed-29e1-7634-9a67-b87aa0d3efe8:/// Cursor streamed `<img src="hello_world_browser.png" ... />` a token at a/// time, the file only ever existed in its sandbox, and the same image thendiff --git a/crates/cursor_cloud_agents/src/domain/ports.rs b/crates/cursor_cloud_agents/src/domain/ports.rsindex 2a595875fb..17c42746b4 100644--- a/crates/cursor_cloud_agents/src/domain/ports.rs+++ b/crates/cursor_cloud_agents/src/domain/ports.rs@@ -207,8 +207,10 @@ pub struct ConnectedStream<Records> {/// Why one connect did not produce a stream.////// The domain acts differently on each: an unavailable stream is retried, an-/// invalid resume position is retried without one, and an expired stream is-/// gone for good and leaves polling as the only way to learn the outcome.+/// invalid resume position is retried without one, an expired stream is+/// gone for good and leaves polling as the only way to learn the outcome,+/// and a stalled connect is checked against the run's record before it is+/// tried again.#[derive(Debug)]pub enum StreamConnectError {/// `stream_unavailable`: the stream is not there (yet). Carries the@@ -218,6 +220,16 @@ pub enum StreamConnectError {InvalidResumePosition(String),/// `stream_expired`: the retention window closed behind us.Expired(String),+ /// The provider accepted the connection but sent no response headers+ /// within the adapter's patience.+ ///+ /// Observed live (prod, 2026-09-26): a resume after a mid-run body+ /// failure was held for 6m43s with no headers, then answered with every+ /// record since the break in one burst, the moment the run finished. A+ /// connect that waits indefinitely turns that into a turn nobody can see+ /// working; this variant is what lets the domain keep polling the run's+ /// record in the meantime.+ Stalled(String),/// Every other failure.Other(rootcause::Report),}@@ -230,6 +242,12 @@ impl std::fmt::Display for StreamConnectError {write!(formatter, "Cursor rejected the resume position: {message}")}Self::Expired(message) => write!(formatter, "Cursor stream expired: {message}"),+ Self::Stalled(message) => {+ write!(+ formatter,+ "Cursor stream stalled before its headers: {message}"+ )+ }Self::Other(report) => write!(formatter, "{report}"),}}diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rsindex 14d9b934e6..76fae84bff 100644--- a/crates/cursor_cloud_agents/src/domain/service.rs+++ b/crates/cursor_cloud_agents/src/domain/service.rs@@ -155,10 +155,33 @@ const STREAM_RECONNECT_DELAYS: [std::time::Duration; 5] = [];/// Reconnects before a run gives up on its stream and polls instead. Counted-/// since the last reconnect that delivered content, so a long run that drops-/// repeatedly but recovers each time is never rationed.+/// since the last reconnect that delivered content and then stayed up (see+/// [`STREAM_HEALTHY_RESUME`]), so a long run that drops repeatedly but+/// recovers each time is never rationed.const STREAM_RECONNECT_ATTEMPTS: usize = 5;+/// How long a resumed connection has to stay up, once it has delivered,+/// before it counts as the transport working again and refills the budget.+///+/// Every resume in one day's incidents (prod, 2026-09-26, eight runs)+/// delivered its whole backlog in a burst and failed within the second, from+/// the very id it had just advanced to. Refilling on delivery alone made+/// each of those a fresh first attempt: the budget never ran down, the+/// attempt counter in every log line read `1`, and nothing could say "this+/// transport has been failing for an hour". A connection that outlives this+/// after delivering is one the provider is genuinely streaming on.+const STREAM_HEALTHY_RESUME: std::time::Duration = std::time::Duration::from_secs(5);++/// Connects that stall before their headers, on a run whose stream never+/// opened, before the turn stops trying and polls the record instead.+///+/// Once a stream has opened, stalls are not counted against anything: the+/// run's record is checked on each, so a finished run ends its turn within+/// a stall of finishing, and a run still going is still worth the stream.+/// Before it has ever opened there is nothing to resume, and a connect that+/// stalls three times running is a stream that is not going to open.+const STALLED_CONNECTS_BEFORE_POLLING: u32 = 3;+/// How long a prompt waits behind a run something else started (the same/// agent is drivable from cursor.com) before giving up, in poll intervals.const BUSY_ATTEMPTS: usize = 450;@@ -1606,6 +1629,19 @@ where// The run's latest `status` frame, kept to recognize the sticky copy// Cursor re-sends at the top of every reconnect.let mut last_status: Option<crate::domain::journal::NativeRecord> = None;+ // Connects in a row that stalled before their headers; a connection+ // that opens resets it.+ let mut stalled_connects: u32 = 0;+ // Whether the record was found terminal behind a stalled stream and+ // the one connect allowed to drain that stream has been spent.+ let mut drained_after_terminal = false;+ // When the transport last broke, and when a record last arrived on+ // any connection: together they say how long the transcript has+ // been behind, which is the figure a person watching a frozen chip+ // needs from the logs.+ let started_at = tokio::time::Instant::now();+ let mut interrupted_at: Option<tokio::time::Instant> = None;+ let mut last_record_at: Option<tokio::time::Instant> = None;loop {// The connected stream borrows the position it resumed from, so// the position this iteration sends is its own owned copy.@@ -1615,6 +1651,68 @@ where.raw_stream(agent, run, resuming.as_deref()).await{+ // The stream said nothing, but the record still answers: a+ // run that finished behind a wedged stream must not wait on+ // another stall to end its turn. Hydration takes the strict+ // arm below; it has no record to fall back on.+ Err(StreamConnectError::Stalled(detail)) if !strict => {+ stalled_connects += 1;+ let raw = self+ .fetch_run_record(session_id, agent, run, cancel)+ .await?;+ let outcome = Self::run_outcome(&raw)?;+ if outcome.is_terminal()+ && !std::mem::replace(&mut drained_after_terminal, true)+ {+ // Observed: what a stalled stream held back arrives+ // the instant the run ends, so the stream gets one+ // more connect to deliver it - and its terminal+ // frame - itself. The record is deliberately not+ // captured yet: its final text would reach the+ // transcript now and then again from the stream.+ tracing::info!(+ cursor.run.id = %run,+ cursor.stream.stalled_connects = stalled_connects,+ cursor.stream.last_event_id = last_event_id,+ "Cursor run ended behind a stalled stream; connecting once more to drain it"+ );+ } else {+ self.capture(+ session_id,+ session,+ Some(run),+ JournalInput::Poll(raw),+ emit,+ )+ .await?;+ if outcome.is_terminal() {+ // A second stall past the run's end: the+ // record's account stands.+ terminal = Some(outcome.status);+ tracing::warn!(+ cursor.run.id = %run,+ cursor.stream.stalled_connects = stalled_connects,+ cursor.stream.last_event_id = last_event_id,+ "Cursor stream still stalled after the run ended; keeping the record's account"+ );+ break;+ }+ }+ if cancel.is_cancelled() {+ break;+ }+ Some(Interruption {+ reason: format!("{detail} (stalled connect {stalled_connects})"),+ recoverable: ever_connected+ || stalled_connects < STALLED_CONNECTS_BEFORE_POLLING,+ quiet: true,+ })+ }+ Err(StreamConnectError::Stalled(detail)) => Some(Interruption {+ reason: detail,+ recoverable: false,+ quiet: false,+ }),Err(StreamConnectError::InvalidResumePosition(detail))if blind_reconnect_available && resuming.is_some() =>{@@ -1645,6 +1743,9 @@ whereOk(connected) => {let resumed = resuming.is_some();ever_connected = true;+ stalled_connects = 0;+ let opened_at = tokio::time::Instant::now();+ let mut first_record_at: Option<tokio::time::Instant> = None;let retention_seconds = connected.retention_seconds;let records = connected.records;pin_mut!(records);@@ -1664,6 +1765,9 @@ where{Ok(Some(Ok(record))) => {silence = std::time::Duration::ZERO;+ let now = tokio::time::Instant::now();+ first_record_at.get_or_insert(now);+ last_record_at = Some(now);record}Ok(Some(Err(error))) => {@@ -1792,18 +1896,33 @@ where}};if resumed {+ let connection = opened_at.elapsed();+ // A resume that delivered and then stayed up is a+ // working stream again, so the next drop gets the+ // full budget rather than whatever a much earlier one+ // left over. One that delivered its backlog and died+ // at once is the same failure continuing, and spends.+ let healthy = received_content > 0+ && (interruption.is_none() || connection >= STREAM_HEALTHY_RESUME);tracing::info!(cursor.run.id = %run,cursor.stream.attempt = reconnects,cursor.stream.last_event_id = last_event_id,cursor.stream.retention_seconds = retention_seconds,cursor.stream.resumed_records = received,+ cursor.stream.resumed_content = received_content,+ cursor.stream.waited_ms = interrupted_at.map(|at| {+ first_record_at+ .unwrap_or_else(tokio::time::Instant::now)+ .saturating_duration_since(at)+ .as_millis() as u64+ }),+ cursor.stream.connection_ms = connection.as_millis() as u64,+ cursor.stream.healthy = healthy,+ cursor.stream.fast_failed = interruption.is_some() && !healthy,"Cursor stream resumed");- // A resume that delivered is a working stream again,- // so the next drop gets the full budget rather than- // whatever a much earlier one left over.- if received_content > 0 {+ if healthy {reconnects = 0;}}@@ -1819,7 +1938,15 @@ wherebreak;};fell_back_to_poll = true;- if !ever_connected {+ interrupted_at = Some(tokio::time::Instant::now());+ // How long the transcript has been behind: since the last record+ // on any connection, or since this attempt began if none came.+ let silent_ms = last_record_at.unwrap_or(started_at).elapsed().as_millis() as u64;+ // A connect that stalled is tried again a bounded number of+ // times before this gives up on the stream, because a stall is+ // the one connect failure the client did not already retry.+ let retrying_a_stall = quiet && recoverable;+ if !ever_connected && !retrying_a_stall {// Nothing to resume: the stream never opened, and the client// has already retried an unavailable one on its own.self.capture(@@ -1833,6 +1960,8 @@ wheretracing::warn!(cursor.run.id = %run,cursor.stream.reason = %reason,+ cursor.stream.stalled_connects = stalled_connects,+ cursor.stream.silent_ms = silent_ms,"Cursor stream never connected; polling the run record instead");break;@@ -1863,6 +1992,8 @@ wherecursor.stream.last_event_id = last_event_id,cursor.stream.attempt = attempt,cursor.stream.reason = %reason,+ cursor.stream.silent_ms = silent_ms,+ cursor.stream.stalled_connects = stalled_connects,"Cursor stream is not resumable; polling the run record instead");break;@@ -1872,6 +2003,9 @@ wherecursor.stream.last_event_id = last_event_id,cursor.stream.attempt = attempt,cursor.stream.reason = %reason,+ cursor.stream.quiet = quiet,+ cursor.stream.silent_ms = silent_ms,+ cursor.stream.stalled_connects = stalled_connects,"Cursor stream interrupted; reconnecting");let delay = STREAM_RECONNECT_DELAYS@@ -1990,6 +2124,33 @@ where////// `debug` because the loop runs as often as [`POLL_DELAYS`] says; the turn span/// above carries the outcome, this carries the liveness.+ async fn poll_once(+ &self,+ session_id: &SessionId,+ session: &Session,+ agent: &CursorAgentId,+ run: &CursorRunId,+ cancel: &tokio_util::sync::CancellationToken,+ emit: Emit,+ ) -> Result<crate::domain::model::RunOutcome, SessionError> {+ let raw = self+ .fetch_run_record(session_id, agent, run, cancel)+ .await?;+ self.capture(+ session_id,+ session,+ Some(run),+ JournalInput::Poll(raw.clone()),+ emit,+ )+ .await?;+ Self::run_outcome(&raw)+ }++ /// The run's record as Cursor holds it, retried through transient+ /// failures. Not captured: [`Self::poll_once`] does that, and the one+ /// caller that reads the record without capturing it — a stalled stream+ /// checking whether its run has ended — says why there.#[tracing::instrument(name = "cursor.run.poll",level = "debug",@@ -1998,27 +2159,19 @@ whereagent.acp.session_id = ?session_id,cursor.agent.id = %agent,cursor.run.id = %run,- cursor.run.status = tracing::field::Empty,),err,)]- async fn poll_once(+ async fn fetch_run_record(&self,session_id: &SessionId,- session: &Session,agent: &CursorAgentId,run: &CursorRunId,cancel: &tokio_util::sync::CancellationToken,- emit: Emit,- ) -> Result<crate::domain::model::RunOutcome, SessionError> {- let mut raw = None;- for attempt in 0..=POLL_ERROR_TOLERANCE {+ ) -> Result<String, SessionError> {+ for _tolerated in 0..POLL_ERROR_TOLERANCE {match self.cursor.raw_result(agent, run).await {- Ok(value) => {- raw = Some(value);- break;- }- Err(error) if attempt == POLL_ERROR_TOLERANCE => return Err(error.into()),+ Ok(value) => return Ok(value),Err(error) => {tracing::warn!(error = ?error, "Cursor poll failed; retrying");if sleep_unless_cancelled(cancel, POLL_INTERVAL).await {@@ -2027,17 +2180,14 @@ where}}}- let raw = raw.expect("poll returned or failed");- self.capture(- session_id,- session,- Some(run),- JournalInput::Poll(raw.clone()),- emit,- )- .await?;+ // The last try past the tolerance is the one whose failure is final.+ self.cursor.raw_result(agent, run).await.map_err(Into::into)+ }++ /// The outcome a run record states.+ fn run_outcome(raw: &str) -> Result<crate::domain::model::RunOutcome, SessionError> {let value: serde_json::Value =- serde_json::from_str(&raw).map_err(|e| rootcause::report!(e).into_dynamic())?;+ serde_json::from_str(raw).map_err(|e| rootcause::report!(e).into_dynamic())?;Ok(crate::domain::model::RunOutcome {status: serde_json::from_value(value["status"].clone()).map_err(|e| rootcause::report!(e).into_dynamic())?,diff --git a/crates/cursor_cloud_agents/src/domain/service/test.rs b/crates/cursor_cloud_agents/src/domain/service/test.rsindex d17e2a934a..94f7a7c13d 100644--- a/crates/cursor_cloud_agents/src/domain/service/test.rs+++ b/crates/cursor_cloud_agents/src/domain/service/test.rs@@ -3602,6 +3602,362 @@ async fn a_rejected_resume_position_reconnects_once_without_one() {);}+/// Prod, 2026-09-26: a resume was held open with no headers for 6m43s and+/// answered all at once when the run ended. The connect now gives up waiting+/// for headers, and each stall checks the run's record: a run still going+/// gets its stream tried again without spending the failure budget, and a+/// run that has ended gets one more connect to drain what the stream held.+#[tokio::test(start_paused = true)]+async fn a_stalled_connect_checks_the_record_and_drains_the_stream_once_the_run_ends() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "before the stall".to_owned(),+ },+ "evt-1",+ )+ .expect("stream open");+ drop(dropped);+ // Two resumes stall. The record says the run is still going after the+ // first and finished after the second.+ cursor.script_stream_connect_error(StreamConnectError::Stalled(+ "no response headers after 30s".to_owned(),+ ));+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Running,+ text: None,+ });+ cursor.script_stream_connect_error(StreamConnectError::Stalled(+ "no response headers after 30s".to_owned(),+ ));+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Finished,+ text: Some("before the stall and after it".to_owned()),+ });+ // The drain delivers what the stalled stream held back, terminal frame+ // included.+ let drained = cursor.script_stream();+ drained+ .send(CursorEvent::Assistant {+ text: " and after it".to_owned(),+ })+ .expect("stream open");+ drained.send(finished("run-fake-1")).expect("stream open");+ drained.send(CursorEvent::Done).expect("stream open");+ drop(drained);++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the drained stream finishes the turn");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(+ cursor.resume_positions(),+ vec![+ None,+ Some("evt-1".to_owned()),+ Some("evt-1".to_owned()),+ Some("evt-1".to_owned()),+ ],+ "every connect after the break resumes from the last id heard"+ );+ assert_eq!(+ agent_texts(¬ifier.updates()),+ vec!["before the stall", " and after it"],+ "the stream's own account is delivered once, never the record's on top of it"+ );+ let entries = service.journal.read(&session).await.expect("journal");+ let stalls: Vec<_> = entries+ .iter()+ .filter_map(|entry| match &entry.input {+ JournalInput::StreamInterrupted {+ reason, attempt, ..+ } if reason.contains("stalled connect") => Some((reason.clone(), *attempt)),+ _ => None,+ })+ .collect();+ assert_eq!(+ stalls,+ vec![+ (+ "no response headers after 30s (stalled connect 1)".to_owned(),+ 2+ ),+ (+ "no response headers after 30s (stalled connect 2)".to_owned(),+ 2+ ),+ ],+ "each stall is durable, and none of them spends the reconnect budget"+ );+ assert_eq!(+ entries+ .iter()+ .filter(|entry| matches!(entry.input, JournalInput::Poll(_)))+ .count(),+ 1,+ "the record is captured while the run is going, not the terminal one that the drain overtook"+ );+}++/// When the drain stalls too, the record's account stands - and because the+/// stream broke and never came back, the restated answer is appended whole+/// rather than dropped for not continuing the fragment that was streamed.+#[tokio::test(start_paused = true)]+async fn a_stream_still_stalled_after_the_run_ends_keeps_the_record() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "Looking into".to_owned(),+ },+ "evt-1",+ )+ .expect("stream open");+ drop(dropped);+ for _ in 0..2 {+ cursor.script_stream_connect_error(StreamConnectError::Stalled(+ "no response headers after 30s".to_owned(),+ ));+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Finished,+ text: Some("The bell icon was hidden by a stale flag.".to_owned()),+ });+ }++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the record ends the turn");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(+ cursor.resume_positions().len(),+ 3,+ "the first connect, the stalled resume, and the one drain attempt"+ );+ assert_eq!(+ agent_texts(¬ifier.updates()),+ vec![+ "Looking into",+ "\n\nThe bell icon was hidden by a stale flag."+ ],+ "the answer the stream never delivered still reaches the client"+ );+}++/// Before the stream has ever opened there is nothing to resume, so stalls+/// are bounded: a few are tried, then the record is polled instead.+#[tokio::test(start_paused = true)]+async fn stalls_before_the_stream_ever_opened_fall_back_to_polling() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ for _ in 0..STALLED_CONNECTS_BEFORE_POLLING {+ cursor.script_stream_connect_error(StreamConnectError::Stalled(+ "no response headers after 30s".to_owned(),+ ));+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Running,+ text: None,+ });+ }+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Finished,+ text: Some("the whole answer".to_owned()),+ });++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the poll ends the turn");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(+ cursor.resume_positions().len(),+ STALLED_CONNECTS_BEFORE_POLLING as usize,+ "a stream that never opens is tried a bounded number of times"+ );+ assert_eq!(agent_texts(¬ifier.updates()), vec!["the whole answer"]);+ let entries = service.journal.read(&session).await.expect("journal");+ assert!(+ entries.iter().any(|entry| matches!(+ &entry.input,+ JournalInput::TransportError(reason) if reason.contains("stalled connect 3")+ )),+ "giving up on the stream is journaled as the transport failure it is"+ );+}++/// Every resume in the 2026-09-26 incidents delivered its backlog in one+/// burst and failed within the second. Delivery alone no longer refills the+/// reconnect budget: a resume that dies at once is the same failure+/// continuing, and after the budget the record is polled instead.+#[tokio::test(start_paused = true)]+async fn a_resume_that_delivers_and_dies_at_once_spends_the_budget() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "a".to_owned(),+ },+ "evt-0",+ )+ .expect("stream open");+ drop(dropped);+ for index in 1..=STREAM_RECONNECT_ATTEMPTS {+ let burst = cursor.script_stream_failing_with("error decoding response body");+ burst+ .send_with_id(+ CursorEvent::Assistant {+ text: "a".to_owned(),+ },+ &format!("evt-{index}"),+ )+ .expect("stream open");+ drop(burst);+ }+ // One more burst is queued than the budget allows; it must go unread.+ let unread = cursor.script_stream();+ cursor.script_run_result(RunOutcome {+ status: RunStatus::Finished,+ text: Some("a".repeat(1 + STREAM_RECONNECT_ATTEMPTS)),+ });++ let stop = service+ .prompt(&session, "hi")+ .await+ .expect("the poll ends the turn");+ assert_eq!(stop, StopReason::EndTurn);+ drop(unread);+ assert_eq!(+ cursor.resume_positions().len(),+ 1 + STREAM_RECONNECT_ATTEMPTS,+ "the budget is spent by resumes that fail at once, however much they delivered"+ );+ assert_eq!(+ agent_texts(¬ifier.updates()).concat(),+ "a".repeat(1 + STREAM_RECONNECT_ATTEMPTS),+ "everything each burst delivered still reached the client, once"+ );+ let entries = service.journal.read(&session).await.expect("journal");+ let attempts: Vec<u32> = entries+ .iter()+ .filter_map(|entry| match &entry.input {+ JournalInput::StreamInterrupted { attempt, .. } => Some(*attempt),+ _ => None,+ })+ .collect();+ assert_eq!(+ attempts,+ (1..=1 + STREAM_RECONNECT_ATTEMPTS as u32).collect::<Vec<_>>(),+ "the attempt counter climbs instead of reading 1 on every line"+ );+}++/// A resume that delivers and then stays up is the transport working again,+/// and the next break gets a full budget.+#[tokio::test(start_paused = true)]+async fn a_resume_that_stays_up_refills_the_budget() {+ let (service, cursor, notifier) = service(None);+ let session = service.new_session(Path::new(""), Vec::new());++ let dropped = cursor.script_stream_failing_with("error decoding response body");+ dropped+ .send_with_id(+ CursorEvent::Assistant {+ text: "a".to_owned(),+ },+ "evt-0",+ )+ .expect("stream open");+ drop(dropped);+ // The budget's worth of resumes that die at once, then one that lives.+ for index in 1..STREAM_RECONNECT_ATTEMPTS {+ let burst = cursor.script_stream_failing_with("error decoding response body");+ burst+ .send_with_id(+ CursorEvent::Assistant {+ text: "a".to_owned(),+ },+ &format!("evt-{index}"),+ )+ .expect("stream open");+ drop(burst);+ }+ let lively = cursor.script_stream_failing_with("error decoding response body");+ lively+ .send_with_id(+ CursorEvent::Assistant {+ text: "b".to_owned(),+ },+ "evt-lively",+ )+ .expect("stream open");+ // After the refill, another budget's worth of fast failures is survivable.+ for index in 0..STREAM_RECONNECT_ATTEMPTS - 1 {+ let burst = cursor.script_stream_failing_with("error decoding response body");+ burst+ .send_with_id(+ CursorEvent::Assistant {+ text: "c".to_owned(),+ },+ &format!("evt-c{index}"),+ )+ .expect("stream open");+ drop(burst);+ }+ let last = cursor.script_stream();+ last.send(finished("run-fake-1")).expect("stream open");+ last.send(CursorEvent::Done).expect("stream open");+ drop(last);++ let turn = {+ let service = Arc::clone(&service);+ let session = session.clone();+ tokio::spawn(async move { service.prompt(&session, "hi").await })+ };+ // Let the lively connection outlive the health threshold before it dies.+ // The bursts before it reconnect on the backoff schedule, which paused+ // time advances through as the turn sleeps.+ tokio::time::sleep(+ STREAM_RECONNECT_DELAYS.iter().sum::<std::time::Duration>()+ + STREAM_HEALTHY_RESUME+ + std::time::Duration::from_secs(1),+ )+ .await;+ drop(lively);++ let stop = turn+ .await+ .expect("the turn task completes")+ .expect("the stream finishes the turn");+ assert_eq!(stop, StopReason::EndTurn);+ assert_eq!(+ cursor.resume_positions().len(),+ // The first connect, the bursts before the lively resume, the lively+ // resume itself, the bursts after it, and the stream that finished.+ 1 + (STREAM_RECONNECT_ATTEMPTS - 1) + 1 + (STREAM_RECONNECT_ATTEMPTS - 1) + 1,+ "the lively resume refilled the budget for the bursts after it"+ );+ assert_eq!(+ agent_texts(¬ifier.updates()).concat(),+ format!(+ "{}b{}",+ "a".repeat(STREAM_RECONNECT_ATTEMPTS),+ "c".repeat(STREAM_RECONNECT_ATTEMPTS - 1)+ )+ );+}+/// Cursor does not heartbeat an open stream, and a run at work says nothing/// for minutes at a time - a tool call waiting on CI, a long build. Seen/// live: a run that opened its pull request a minute after its turn had