Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

chore(agents): cursor SSE ingest: diagnose body failures, survive stalled resumes, log MCP stream ends (#7067)

Wolf Mermelstein4e392cea0c7a
diff --git a/Cargo.lock b/Cargo.lock
index a2c137e7da..ef327a587e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -265,8 +265,10 @@ dependencies = [
"base64 0.22.1",
"bytes",
"chrono",
+ "futures",
"github",
"http 1.4.0",
+ "http-body 1.0.1",
"http-body-util",
"lru 0.16.4",
"macro_auth",
@@ -286,6 +288,7 @@ dependencies = [
"tower 0.5.3",
"tracing",
"url",
+ "uuid",
"workspace-hack",
]
@@ -5186,6 +5189,7 @@ dependencies = [
"agent_fold",
"agent_runtime_protocol",
"agent_session",
+ "base64 0.22.1",
"bytes",
"futures",
"insta",
diff --git a/Cargo.toml b/Cargo.toml
index 564b8bb4a6..89e11d7f59 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -242,6 +242,7 @@ hex = "0.4.3"
hmac = "0.12.1"
encoding_rs = "0.8"
http = "1"
+http-body = "1.0.1"
http-body-util = "0.1.2"
httpdate = "1"
image = "0.25"
diff --git a/crates/agent_egress/Cargo.toml b/crates/agent_egress/Cargo.toml
index ae9d860484..778f71f9bb 100644
--- a/crates/agent_egress/Cargo.toml
+++ b/crates/agent_egress/Cargo.toml
@@ -25,6 +25,8 @@ github = { path = "../github", default-features = false, features = [
"sync",
] }
http = { workspace = true }
+# Wrapping upstream event-stream bodies so their lifetime and volume get logged.
+http-body = { workspace = true }
http-body-util = { workspace = true }
lru = { workspace = true }
# Signing Macro API tokens, with the same shared vocabulary
@@ -51,5 +53,7 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
anyhow = { workspace = true }
+futures = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt"] }
+uuid = { workspace = true }
tower = { workspace = true }
diff --git a/crates/agent_egress/src/domain/mod.rs b/crates/agent_egress/src/domain/mod.rs
index 024cc230b7..bb31ddf103 100644
--- a/crates/agent_egress/src/domain/mod.rs
+++ b/crates/agent_egress/src/domain/mod.rs
@@ -5,6 +5,8 @@ pub mod error;
/// Vocabulary: grants, slugs, targets, and the transport-neutral request and
/// response the service passes through.
pub mod model;
+/// A response body that logs how long it lived and how it ended.
+pub mod observed_body;
/// The capabilities the service needs from the outside.
pub mod ports;
/// The service itself.
diff --git a/crates/agent_egress/src/domain/observed_body.rs b/crates/agent_egress/src/domain/observed_body.rs
new file mode 100644
index 0000000000..2c5b873d17
--- /dev/null
+++ b/crates/agent_egress/src/domain/observed_body.rs
@@ -0,0 +1,123 @@
+//! A response body that writes down how it ended.
+//!
+//! The proxy answers a GET for an MCP event stream with the upstream's body
+//! and hears nothing more about it: how long it stayed open, how much it
+//! carried, and whether the upstream closed it or the sandbox went away all
+//! happen after the handler has returned. When one session reopened a
+//! stream 380 times in a row (prod, 2026-09-26) the logs could say only
+//! that each open was proxied; not what the upstream answered, nor how
+//! quickly each stream ended. This wrapper is what says so.
+
+#[cfg(test)]
+mod test;
+
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use std::time::Instant;
+
+use bytes::Bytes;
+use http_body::{Body, Frame};
+
+use crate::domain::model::{AgentSessionId, BoxError, ProxyBody};
+
+/// Facts about the stream that are known when it opens, for the line
+/// written when it ends.
+#[derive(Debug, Clone)]
+pub struct StreamIdentity {
+ /// The session that opened it.
+ pub session: AgentSessionId,
+ /// The upstream it was opened on.
+ pub upstream: String,
+ /// The status the upstream answered with.
+ pub status: u16,
+ /// The upstream's `content-type`, `text/event-stream` for a live stream.
+ pub content_type: Option<String>,
+}
+
+/// A body that logs its lifetime when it ends or is dropped.
+pub struct ObservedBody {
+ inner: ProxyBody,
+ identity: StreamIdentity,
+ opened_at: Instant,
+ bytes: u64,
+ frames: u64,
+ /// Whether the upstream ended the body, as opposed to the sandbox
+ /// dropping it - or the body failing - before that.
+ ended: bool,
+ failed: bool,
+}
+
+impl ObservedBody {
+ /// Wrap `inner`, to be logged as `identity` when it ends.
+ pub fn new(inner: ProxyBody, identity: StreamIdentity) -> Self {
+ Self {
+ inner,
+ identity,
+ opened_at: Instant::now(),
+ bytes: 0,
+ frames: 0,
+ ended: false,
+ failed: false,
+ }
+ }
+
+ /// How the stream ended, in one word.
+ fn outcome(&self) -> &'static str {
+ if self.failed {
+ "failed"
+ } else if self.ended {
+ "closed by upstream"
+ } else {
+ "dropped by sandbox"
+ }
+ }
+}
+
+impl Body for ObservedBody {
+ type Data = Bytes;
+ type Error = BoxError;
+
+ fn poll_frame(
+ mut self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
+ let polled = Pin::new(&mut self.inner).poll_frame(cx);
+ match &polled {
+ Poll::Ready(Some(Ok(frame))) => {
+ self.frames += 1;
+ if let Some(data) = frame.data_ref() {
+ self.bytes += data.len() as u64;
+ }
+ }
+ Poll::Ready(Some(Err(_))) => self.failed = true,
+ Poll::Ready(None) => self.ended = true,
+ Poll::Pending => {}
+ }
+ polled
+ }
+
+ fn is_end_stream(&self) -> bool {
+ self.inner.is_end_stream()
+ }
+
+ fn size_hint(&self) -> http_body::SizeHint {
+ self.inner.size_hint()
+ }
+}
+
+impl Drop for ObservedBody {
+ fn drop(&mut self) {
+ let open_ms = self.opened_at.elapsed().as_millis() as u64;
+ tracing::info!(
+ session = %self.identity.session,
+ upstream = %self.identity.upstream,
+ status = self.identity.status,
+ content_type = self.identity.content_type.as_deref(),
+ outcome = self.outcome(),
+ open_ms,
+ bytes = self.bytes,
+ frames = self.frames,
+ "MCP event stream ended"
+ );
+ }
+}
diff --git a/crates/agent_egress/src/domain/observed_body/test.rs b/crates/agent_egress/src/domain/observed_body/test.rs
new file mode 100644
index 0000000000..e8ee9d7413
--- /dev/null
+++ b/crates/agent_egress/src/domain/observed_body/test.rs
@@ -0,0 +1,55 @@
+use super::*;
+use http_body_util::{BodyExt, Full, StreamBody};
+
+fn identity() -> StreamIdentity {
+ StreamIdentity {
+ session: AgentSessionId::new(),
+ upstream: "datadog".to_owned(),
+ status: 200,
+ content_type: Some("text/event-stream".to_owned()),
+ }
+}
+
+#[tokio::test]
+async fn a_body_read_to_its_end_counts_what_passed_and_knows_it_ended() {
+ let inner: ProxyBody = Full::new(Bytes::from_static(b"data: hello\n\n"))
+ .map_err(|never| match never {})
+ .boxed_unsync();
+ let mut observed = ObservedBody::new(inner, identity());
+ let mut collected = Vec::new();
+ while let Some(frame) = observed.frame().await {
+ if let Ok(data) = frame.expect("a full body does not fail").into_data() {
+ collected.extend_from_slice(&data);
+ }
+ }
+ assert_eq!(collected, b"data: hello\n\n");
+ assert_eq!(observed.bytes, 13);
+ assert_eq!(observed.frames, 1);
+ assert!(observed.ended);
+ assert_eq!(observed.outcome(), "closed by upstream");
+}
+
+#[tokio::test]
+async fn a_body_dropped_early_is_known_to_have_been_dropped() {
+ let inner: ProxyBody = Full::new(Bytes::from_static(b"data: hello\n\n"))
+ .map_err(|never| match never {})
+ .boxed_unsync();
+ let observed = ObservedBody::new(inner, identity());
+ assert!(!observed.ended);
+ assert_eq!(observed.outcome(), "dropped by sandbox");
+}
+
+#[tokio::test]
+async fn a_body_that_fails_is_known_to_have_failed() {
+ let failing = futures::stream::iter(vec![
+ Ok::<_, BoxError>(Frame::data(Bytes::from_static(b"data: a\n\n"))),
+ Err("connection reset".into()),
+ ]);
+ let inner: ProxyBody = StreamBody::new(failing).boxed_unsync();
+ let mut observed = ObservedBody::new(inner, identity());
+ assert!(observed.frame().await.expect("a frame").is_ok());
+ assert!(observed.frame().await.expect("an error").is_err());
+ assert!(observed.failed);
+ assert_eq!(observed.outcome(), "failed");
+ assert_eq!(observed.bytes, 9);
+}
diff --git a/crates/agent_egress/src/domain/service.rs b/crates/agent_egress/src/domain/service.rs
index 5b3a03ddd3..68df7667f7 100644
--- a/crates/agent_egress/src/domain/service.rs
+++ b/crates/agent_egress/src/domain/service.rs
@@ -11,6 +11,7 @@ use crate::domain::model::{
is_macro_staff, not_connected_tool_result, peek_json_rpc, sanitize_request_headers,
sanitize_response_headers,
};
+use crate::domain::observed_body::{ObservedBody, StreamIdentity};
use crate::domain::ports::{Forwarder, GithubTokens, McpCredentials, SessionAuthority};
#[cfg(test)]
@@ -217,6 +218,13 @@ where
request.headers_mut().insert(name.clone(), value.clone());
}
+ // A GET on an MCP target opens the server's event stream. Those are
+ // the requests a client reopens in a loop when the upstream keeps
+ // closing them, so its body is watched to its end - the one place the
+ // upstream's side of the loop can be seen.
+ let event_stream =
+ matches!(target, EgressTarget::McpServer(_)) && *request.method() == Method::GET;
+
let mut response = self.forward.forward(request).await?;
sanitize_response_headers(response.headers_mut());
@@ -233,6 +241,27 @@ where
);
}
+ if event_stream {
+ let identity = StreamIdentity {
+ session: grant.session,
+ upstream: target.name(),
+ status: response.status().as_u16(),
+ content_type: response
+ .headers()
+ .get(http::header::CONTENT_TYPE)
+ .and_then(|value| value.to_str().ok())
+ .map(str::to_owned),
+ };
+ tracing::info!(
+ session = %identity.session,
+ upstream = %identity.upstream,
+ status = identity.status,
+ content_type = identity.content_type.as_deref(),
+ "MCP event stream opened"
+ );
+ return Ok(response.map(|body| ObservedBody::new(body, identity).boxed_unsync()));
+ }
+
Ok(response)
}
}
diff --git a/crates/cursor_cloud_agents/Cargo.toml b/crates/cursor_cloud_agents/Cargo.toml
index 7b1ff8a795..bac4ac75ea 100644
--- a/crates/cursor_cloud_agents/Cargo.toml
+++ b/crates/cursor_cloud_agents/Cargo.toml
@@ -21,6 +21,8 @@ agent_runtime_protocol = { path = "../agent_runtime_protocol", default-features
sqlx = { workspace = true }
uuid = { workspace = true }
agent-client-protocol = { workspace = true }
+# The raw tail of a failed stream body, logged as it was on the wire.
+base64 = { workspace = true }
bytes = { workspace = true }
static_file_service_client = { path = "../static_file_service_client" }
futures = { workspace = true }
diff --git a/crates/cursor_cloud_agents/src/api.rs b/crates/cursor_cloud_agents/src/api.rs
index f5b72e43d0..afa0cc2b67 100644
--- a/crates/cursor_cloud_agents/src/api.rs
+++ b/crates/cursor_cloud_agents/src/api.rs
@@ -17,10 +17,14 @@ mod test;
/// Capturing raw SSE bytes as fixtures.
pub mod record;
+/// Reading a stream's body, and the report written when it fails.
+mod stream;
+
/// Request/response DTOs for the endpoints this crate uses.
pub mod wire;
use crate::api::record::SseRecording;
+use crate::api::stream::ConnectionFacts;
use crate::api::wire::{
AgentSummary, ArchiveAgentResponse, ArtifactDownloadResponse, ConversationResponse,
CreateAgentRequest, CreateAgentResponse, CreateRunRequest, CreateRunResponse,
@@ -35,9 +39,7 @@ use crate::domain::model::{
use crate::domain::ports::{
ConnectedStream, CursorAgents, CursorArtifacts, RunStream, StreamConnectError,
};
-use futures::{Stream, StreamExt as _};
-use sse_core::SseEvent;
-use std::collections::VecDeque;
+use futures::Stream;
use std::num::NonZeroUsize;
/// The largest single SSE record payload this agent will buffer, 16 MiB.
@@ -924,6 +926,17 @@ const RETENTION_HEADER: &str = "x-cursor-stream-retention-seconds";
/// The SSE resume header. Not in `http`'s constant set, so it is spelled here.
const LAST_EVENT_ID_HEADER: &str = "last-event-id";
+/// How long a stream connect waits for the response headers before it is a
+/// [`StreamConnectError::Stalled`].
+///
+/// A stream's headers ordinarily arrive within the second. The one case that
+/// takes longer is the one this bounds: a resume the server held open for
+/// minutes and answered all at once when the run ended (prod, 2026-09-26).
+/// Thirty seconds is far past any honest handshake and short enough that
+/// the domain, which checks the run's record on every stall, notices a run
+/// that has finished behind a wedged stream within the minute.
+const STREAM_HEADERS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
+
impl RunStream for CursorClient {
// Covers connecting the stream, retries included; reading it belongs to
// the caller's `cursor.run.ingest` span.
@@ -970,18 +983,46 @@ impl CursorClient {
>,
StreamConnectError,
> {
+ // `identity` is stated rather than left to the default. This client
+ // is built without reqwest's decompression features, so it would
+ // never decode a compressed body — but it would not have asked for
+ // one either, and an intermediary that compresses an event stream
+ // when nothing forbids it buffers records until a block is full.
+ // Asking for identity forbids it, and a `content-encoding` on the
+ // answer becomes a fact worth a warning rather than a guess.
let mut request = self
.http
.get(self.url(&format!("/v1/agents/{agent}/runs/{run}/stream")))
.basic_auth(self.config.api_key.expose(), Some(""))
- .header(reqwest::header::ACCEPT, "text/event-stream");
+ .header(reqwest::header::ACCEPT, "text/event-stream")
+ .header(reqwest::header::ACCEPT_ENCODING, "identity");
if let Some(resume_from) = resume_from {
request = request.header(LAST_EVENT_ID_HEADER, resume_from);
}
- let response = request
- .send()
- .await
- .map_err(|error| StreamConnectError::Other(rootcause::report!(error).into()))?;
+ let asked_at = std::time::Instant::now();
+ let response = match tokio::time::timeout(STREAM_HEADERS_TIMEOUT, request.send()).await {
+ Ok(Ok(response)) => response,
+ Ok(Err(error)) => {
+ return Err(StreamConnectError::Other(rootcause::report!(
+ "{}",
+ stream::error_chain(&error)
+ )));
+ }
+ Err(_elapsed) => {
+ tracing::warn!(
+ cursor.run.id = %run,
+ cursor.stream.resumed = resume_from.is_some(),
+ cursor.stream.last_event_id = resume_from,
+ cursor.stream.headers_timeout_ms = STREAM_HEADERS_TIMEOUT.as_millis() as u64,
+ "Cursor sent no stream headers before the timeout"
+ );
+ return Err(StreamConnectError::Stalled(format!(
+ "no response headers after {}s",
+ STREAM_HEADERS_TIMEOUT.as_secs()
+ )));
+ }
+ };
+ let headers_ms = asked_at.elapsed().as_millis() as u64;
let status = response.status();
// Read before the body is consumed; a resumed stream's window is the
// only thing that says whether the next drop is recoverable at all.
@@ -1008,71 +1049,39 @@ impl CursorClient {
)));
}
+ let facts = ConnectionFacts::of(&response, resume_from.is_some());
+ tracing::info!(
+ cursor.run.id = %run,
+ cursor.stream.resumed = facts.resumed,
+ cursor.stream.last_event_id = resume_from,
+ cursor.stream.headers_ms = headers_ms,
+ cursor.stream.status = facts.status,
+ cursor.stream.http_version = %facts.version,
+ cursor.stream.content_type = facts.content_type.as_deref(),
+ cursor.stream.content_encoding = facts.content_encoding.as_deref(),
+ cursor.stream.transfer_encoding = facts.transfer_encoding.as_deref(),
+ cursor.stream.server = facts.server.as_deref(),
+ cursor.stream.request_id = facts.request_id.as_deref(),
+ cursor.stream.retention_seconds = retention_seconds,
+ "Cursor stream connected"
+ );
+ if facts.is_compressed() {
+ tracing::warn!(
+ cursor.run.id = %run,
+ cursor.stream.content_encoding = facts.content_encoding.as_deref(),
+ cursor.stream.request_id = facts.request_id.as_deref(),
+ "Cursor stream arrived compressed despite asking for identity"
+ );
+ }
+
// Recording taps the bytes before the decoder sees them, so a
// fixture is byte-identical to the wire.
let recording = match &self.config.record_dir {
Some(dir) => SseRecording::create(dir, agent.as_str(), run.as_str()),
None => SseRecording::disabled(),
};
-
- // Decode incrementally: SSE records straddle read boundaries, so the
- // decoder holds a partial record in its own buffers and the unfold
- // drains whole ones only. One read can complete several records, hence
- // the queue.
- let state = (
- response.bytes_stream().boxed(),
- sse_core::SseDecoder::with_limit(MAX_SSE_PAYLOAD),
- VecDeque::new(),
- recording,
- );
- let records = futures::stream::try_unfold(
- state,
- |(mut bytes, mut decoder, mut pending, mut recording)| async move {
- loop {
- if let Some(event) = pending.pop_front() {
- return Ok(Some((event?, (bytes, decoder, pending, recording))));
- }
- match bytes.next().await {
- Some(Ok(chunk)) => {
- recording.write(&chunk);
- let mut cursor = chunk;
- while let Some(record) = decoder.next(&mut cursor) {
- // A payload past the limit is the run's
- // problem, not this stream's shape: report it
- // and stop rather than resync mid-record.
- let record = match record {
- Ok(record) => record,
- Err(error) => {
- // Deliver every earlier complete record
- // in this chunk before the decoder error.
- pending.push_back(Err(rootcause::report!(
- "cursor sse payload over {} bytes: {error}",
- MAX_SSE_PAYLOAD
- )
- .into_dynamic()));
- break;
- }
- };
- let SseEvent::Message(message) = record else {
- continue; // `retry:`; the domain drives reconnects
- };
- pending.push_back(Ok(crate::domain::journal::NativeRecord {
- event: message.event.into_owned(),
- data: message.data,
- id: message.last_event_id.map(|id| id.to_string()),
- }));
- }
- }
- Some(Err(error)) => {
- return Err(rootcause::report!(error).into_dynamic());
- }
- None => return Ok(None),
- }
- }
- },
- );
Ok(ConnectedStream {
- records,
+ records: stream::records(response, facts, recording, MAX_SSE_PAYLOAD),
retention_seconds,
})
}
diff --git a/crates/cursor_cloud_agents/src/api/stream.rs b/crates/cursor_cloud_agents/src/api/stream.rs
new file mode 100644
index 0000000000..5cc89ac9e8
--- /dev/null
+++ b/crates/cursor_cloud_agents/src/api/stream.rs
@@ -0,0 +1,361 @@
+//! Reading a run's SSE body, with the diagnostics a failure needs.
+//!
+//! reqwest reports every mid-body failure as `error decoding response body`
+//! and, since 0.12, keeps the cause out of `Display`: the hyper, h2, or io
+//! error underneath — a reset, an incomplete message, an h2 stream error —
+//! is reachable only through `source()`. Eight runs in one day (prod,
+//! 2026-09-26) failed with exactly that sentence and nothing else, which is
+//! why this module exists. It keeps the facts of the connection and the tail
+//! of its bytes, and writes them all down the moment the body fails.
+
+#[cfg(test)]
+mod test;
+
+use crate::api::record::SseRecording;
+use crate::domain::journal::NativeRecord;
+use base64::Engine as _;
+use futures::stream::BoxStream;
+use futures::{Stream, StreamExt as _};
+use sse_core::SseEvent;
+use std::collections::VecDeque;
+use std::num::NonZeroUsize;
+use std::time::Instant;
+
+/// Raw bytes kept from the tail of a connection, for the report a body
+/// failure writes. 4 KiB: several SSE records' worth, enough to see whether
+/// the last one was cut mid-frame, small enough to base64 into one log line.
+pub(crate) const TAIL_BYTES: usize = 4096;
+
+/// The header Cloudflare stamps a request with; the id to quote to Cursor.
+const CF_RAY_HEADER: &str = "cf-ray";
+
+/// Generic request-id headers, tried after [`CF_RAY_HEADER`].
+const REQUEST_ID_HEADERS: [&str; 2] = ["x-request-id", "x-amzn-requestid"];
+
+/// What a stream connection looked like when it was made.
+///
+/// Captured from the response headers before the body is read, because by
+/// the time the body fails the response is gone and these are the only
+/// things that say which server, over which protocol, with which encoding,
+/// was on the other end.
+#[derive(Debug, Clone)]
+pub(crate) struct ConnectionFacts {
+ /// The response status.
+ pub status: u16,
+ /// The HTTP version negotiated, `HTTP/1.1` or `HTTP/2.0`.
+ pub version: String,
+ /// `content-type`, expected `text/event-stream`.
+ pub content_type: Option<String>,
+ /// `content-encoding`. Expected absent: the request asks for identity,
+ /// and a compressed event stream is buffered by whatever decodes it.
+ pub content_encoding: Option<String>,
+ /// `transfer-encoding`, `chunked` over HTTP/1.1.
+ pub transfer_encoding: Option<String>,
+ /// The `server` header, when one was sent.
+ pub server: Option<String>,
+ /// `cf-ray`, else `x-request-id`, else `x-amzn-requestid`.
+ pub request_id: Option<String>,
+ /// Whether the connection resumed from a `Last-Event-ID`.
+ pub resumed: bool,
+}
+
+impl ConnectionFacts {
+ /// Read the facts off a response before its body is consumed.
+ pub(crate) fn of(response: &reqwest::Response, resumed: bool) -> Self {
+ let header = |name: &str| {
+ response
+ .headers()
+ .get(name)
+ .and_then(|value| value.to_str().ok())
+ .map(str::to_owned)
+ };
+ Self {
+ status: response.status().as_u16(),
+ version: format!("{:?}", response.version()),
+ content_type: header(reqwest::header::CONTENT_TYPE.as_str()),
+ content_encoding: header(reqwest::header::CONTENT_ENCODING.as_str()),
+ transfer_encoding: header(reqwest::header::TRANSFER_ENCODING.as_str()),
+ server: header(reqwest::header::SERVER.as_str()),
+ request_id: header(CF_RAY_HEADER)
+ .or_else(|| REQUEST_ID_HEADERS.iter().find_map(|name| header(name))),
+ resumed,
+ }
+ }
+
+ /// Whether the body arrived compressed, which the request asked it not
+ /// to. Worth a warning of its own: a decoder between the socket and the
+ /// SSE parser buffers until it has a whole block, and fails at whatever
+ /// boundary the server cut.
+ pub(crate) fn is_compressed(&self) -> bool {
+ self.content_encoding
+ .as_deref()
+ .is_some_and(|encoding| !encoding.eq_ignore_ascii_case("identity"))
+ }
+}
+
+/// The last [`TAIL_BYTES`] bytes a connection delivered, in order.
+#[derive(Debug, Default)]
+pub(crate) struct Tail {
+ bytes: VecDeque<u8>,
+}
+
+impl Tail {
+ /// Remember `chunk`, forgetting whatever is now past the window.
+ pub(crate) fn push(&mut self, chunk: &[u8]) {
+ if chunk.len() >= TAIL_BYTES {
+ self.bytes.clear();
+ self.bytes.extend(&chunk[chunk.len() - TAIL_BYTES..]);
+ return;
+ }
+ let overflow = (self.bytes.len() + chunk.len()).saturating_sub(TAIL_BYTES);
+ self.bytes.drain(..overflow);
+ self.bytes.extend(chunk);
+ }
+
+ /// The window's bytes, oldest first.
+ pub(crate) fn bytes(&self) -> Vec<u8> {
+ self.bytes.iter().copied().collect()
+ }
+}
+
+/// The bytes after the last blank line: the record a connection was in the
+/// middle of when it ended, or nothing when it ended between records.
+///
+/// SSE ends a record with a blank line, which is `\n\n` or `\r\n\r\n` on the
+/// wire; whichever came last is the boundary.
+pub(crate) fn partial_frame(bytes: &[u8]) -> &[u8] {
+ let after_lf = bytes
+ .windows(2)
+ .rposition(|window| window == b"\n\n")
+ .map(|at| at + 2);
+ let after_crlf = bytes
+ .windows(4)
+ .rposition(|window| window == b"\r\n\r\n")
+ .map(|at| at + 4);
+ let start = after_lf.max(after_crlf).unwrap_or(0);
+ &bytes[start..]
+}
+
+/// Every message in an error's `source()` chain, outermost first, joined
+/// with `: ` — what `Display` would have said before the convention that
+/// each error names only itself.
+pub(crate) fn error_chain(error: &(dyn std::error::Error + 'static)) -> String {
+ let mut chain = error.to_string();
+ let mut source = error.source();
+ while let Some(cause) = source {
+ let message = cause.to_string();
+ // An error that already restates its cause in its own message would
+ // otherwise say it twice.
+ if !chain.ends_with(&message) {
+ chain.push_str(": ");
+ chain.push_str(&message);
+ }
+ source = cause.source();
+ }
+ chain
+}
+
+/// A lossy UTF-8 rendering of `bytes`, with control characters escaped so it
+/// stays on one log line.
+fn preview(bytes: &[u8]) -> String {
+ String::from_utf8_lossy(bytes)
+ .chars()
+ .flat_map(|character| {
+ if character.is_control() && character != ' ' {
+ character.escape_default().collect::<Vec<_>>()
+ } else {
+ vec![character]
+ }
+ })
+ .collect()
+}
+
+/// One connection's body, being read.
+struct Reader {
+ bytes: BoxStream<'static, reqwest::Result<bytes::Bytes>>,
+ decoder: sse_core::SseDecoder,
+ /// Records decoded from the last chunk and not yet yielded; one read can
+ /// complete several.
+ pending: VecDeque<Result<NativeRecord, rootcause::Report>>,
+ recording: SseRecording,
+ facts: ConnectionFacts,
+ tail: Tail,
+ connected_at: Instant,
+ last_chunk_at: Option<Instant>,
+ last_record_at: Option<Instant>,
+ bytes_received: u64,
+ chunks_received: u64,
+ records_decoded: u64,
+ limit: NonZeroUsize,
+}
+
+impl Reader {
+ fn since(&self, at: Option<Instant>) -> Option<u64> {
+ at.map(|at| at.elapsed().as_millis() as u64)
+ }
+
+ /// Write down everything known about a body that just failed, and
+ /// answer with the report the domain journals.
+ ///
+ /// The log line carries the raw tail — base64 for the bytes as they
+ /// were, a lossy preview for a person reading it — and the partial SSE
+ /// frame the connection was cut in. The report carries only the error
+ /// chain and the counts: it becomes a journal entry, and 4 KiB of
+ /// base64 per interruption belongs in logs, not in the durable record.
+ fn failure(&self, error: &reqwest::Error) -> rootcause::Report {
+ let chain = error_chain(error);
+ let tail = self.tail.bytes();
+ let partial = partial_frame(&tail);
+ let connected_ms = self.connected_at.elapsed().as_millis() as u64;
+ let since_last_record_ms = self.since(self.last_record_at);
+ tracing::warn!(
+ cursor.stream.error = %chain,
+ cursor.stream.error_is_decode = error.is_decode(),
+ cursor.stream.error_is_timeout = error.is_timeout(),
+ cursor.stream.error_is_connect = error.is_connect(),
+ cursor.stream.status = self.facts.status,
+ cursor.stream.http_version = %self.facts.version,
+ cursor.stream.content_type = self.facts.content_type.as_deref(),
+ cursor.stream.content_encoding = self.facts.content_encoding.as_deref(),
+ cursor.stream.transfer_encoding = self.facts.transfer_encoding.as_deref(),
+ cursor.stream.server = self.facts.server.as_deref(),
+ cursor.stream.request_id = self.facts.request_id.as_deref(),
+ cursor.stream.resumed = self.facts.resumed,
+ cursor.stream.bytes_received = self.bytes_received,
+ cursor.stream.chunks_received = self.chunks_received,
+ cursor.stream.records_decoded = self.records_decoded,
+ cursor.stream.connected_ms = connected_ms,
+ cursor.stream.since_last_chunk_ms = self.since(self.last_chunk_at),
+ cursor.stream.since_last_record_ms = since_last_record_ms,
+ cursor.stream.last_event_id = self.decoder.last_event_id().map(|id| id.to_string()),
+ cursor.stream.tail_bytes = tail.len(),
+ cursor.stream.tail_base64 = %base64::engine::general_purpose::STANDARD.encode(&tail),
+ cursor.stream.tail_preview = %preview(&tail),
+ cursor.stream.partial_frame_bytes = partial.len(),
+ cursor.stream.partial_frame = %preview(partial),
+ cursor.stream.decoder = ?self.decoder,
+ "Cursor stream body failed"
+ );
+ rootcause::report!(
+ "{chain} (after {} bytes and {} records in {}s; {} since the last record)",
+ self.bytes_received,
+ self.records_decoded,
+ connected_ms / 1000,
+ since_last_record_ms
+ .map(|ms| format!("{}s", ms / 1000))
+ .unwrap_or_else(|| "none received".to_owned()),
+ )
+ .into_dynamic()
+ }
+
+ /// Note how a body ended. A partial frame at the end means the server
+ /// cut a record in two, which the domain is about to hear as a plain
+ /// close: worth a warning, so the two can be told apart later.
+ fn ended(&self) {
+ let tail = self.tail.bytes();
+ let partial = partial_frame(&tail);
+ if partial.is_empty() {
+ tracing::debug!(
+ cursor.stream.bytes_received = self.bytes_received,
+ cursor.stream.records_decoded = self.records_decoded,
+ cursor.stream.connected_ms = self.connected_at.elapsed().as_millis() as u64,
+ cursor.stream.resumed = self.facts.resumed,
+ "Cursor stream body ended"
+ );
+ return;
+ }
+ tracing::warn!(
+ cursor.stream.bytes_received = self.bytes_received,
+ cursor.stream.records_decoded = self.records_decoded,
+ cursor.stream.connected_ms = self.connected_at.elapsed().as_millis() as u64,
+ cursor.stream.resumed = self.facts.resumed,
+ cursor.stream.request_id = self.facts.request_id.as_deref(),
+ cursor.stream.partial_frame_bytes = partial.len(),
+ cursor.stream.partial_frame = %preview(partial),
+ "Cursor stream body ended mid-record"
+ );
+ }
+
+ /// Decode one chunk into `pending`, whole records only.
+ fn decode(&mut self, chunk: bytes::Bytes) {
+ self.chunks_received += 1;
+ self.bytes_received += chunk.len() as u64;
+ self.last_chunk_at = Some(Instant::now());
+ self.recording.write(&chunk);
+ self.tail.push(&chunk);
+ let mut cursor = chunk;
+ while let Some(record) = self.decoder.next(&mut cursor) {
+ // A payload past the limit is the run's problem, not this
+ // stream's shape: report it and stop rather than resync
+ // mid-record.
+ let record = match record {
+ Ok(record) => record,
+ Err(error) => {
+ // Deliver every earlier complete record in this chunk
+ // before the decoder error.
+ self.pending.push_back(Err(rootcause::report!(
+ "cursor sse payload over {} bytes: {error}",
+ self.limit
+ )
+ .into_dynamic()));
+ break;
+ }
+ };
+ let SseEvent::Message(message) = record else {
+ continue; // `retry:`; the domain drives reconnects
+ };
+ self.records_decoded += 1;
+ self.last_record_at = Some(Instant::now());
+ self.pending.push_back(Ok(NativeRecord {
+ event: message.event.into_owned(),
+ data: message.data,
+ id: message.last_event_id.map(|id| id.to_string()),
+ }));
+ }
+ }
+}
+
+/// The complete SSE records of `response`'s body, as a stream.
+///
+/// Decodes incrementally: SSE records straddle read boundaries, so the
+/// decoder holds a partial record in its own buffers and only whole ones are
+/// yielded. `recording` taps the bytes before the decoder sees them, so a
+/// fixture is byte-identical to the wire. `limit` bounds one record's
+/// payload; see [`crate::api::MAX_SSE_PAYLOAD`].
+pub(crate) fn records(
+ response: reqwest::Response,
+ facts: ConnectionFacts,
+ recording: SseRecording,
+ limit: NonZeroUsize,
+) -> impl Stream<Item = Result<NativeRecord, rootcause::Report>> + Send {
+ let reader = Reader {
+ bytes: response.bytes_stream().boxed(),
+ decoder: sse_core::SseDecoder::with_limit(limit),
+ pending: VecDeque::new(),
+ recording,
+ facts,
+ tail: Tail::default(),
+ connected_at: Instant::now(),
+ last_chunk_at: None,
+ last_record_at: None,
+ bytes_received: 0,
+ chunks_received: 0,
+ records_decoded: 0,
+ limit,
+ };
+ futures::stream::try_unfold(reader, |mut reader| async move {
+ loop {
+ if let Some(record) = reader.pending.pop_front() {
+ return Ok(Some((record?, reader)));
+ }
+ match reader.bytes.next().await {
+ Some(Ok(chunk)) => reader.decode(chunk),
+ Some(Err(error)) => return Err(reader.failure(&error)),
+ None => {
+ reader.ended();
+ return Ok(None);
+ }
+ }
+ }
+ })
+}
diff --git a/crates/cursor_cloud_agents/src/api/stream/test.rs b/crates/cursor_cloud_agents/src/api/stream/test.rs
new file mode 100644
index 0000000000..809509ef85
--- /dev/null
+++ b/crates/cursor_cloud_agents/src/api/stream/test.rs
@@ -0,0 +1,122 @@
+use super::*;
+
+#[test]
+fn the_tail_keeps_the_last_window_of_bytes_in_order() {
+ let mut tail = Tail::default();
+ tail.push(b"abc");
+ tail.push(b"def");
+ assert_eq!(tail.bytes(), b"abcdef");
+
+ let filler = vec![b'x'; TAIL_BYTES - 4];
+ tail.push(&filler);
+ let bytes = tail.bytes();
+ assert_eq!(bytes.len(), TAIL_BYTES);
+ assert!(
+ bytes.starts_with(b"cdef"),
+ "the oldest bytes past the window are forgotten first"
+ );
+
+ let oversized = vec![b'y'; TAIL_BYTES + 10];
+ tail.push(&oversized);
+ assert_eq!(tail.bytes(), vec![b'y'; TAIL_BYTES]);
+}
+
+#[test]
+fn the_partial_frame_is_whatever_follows_the_last_blank_line() {
+ assert_eq!(
+ partial_frame(b"event: a\ndata: 1\n\nevent: b\ndata: 2"),
+ b"event: b\ndata: 2"
+ );
+ assert_eq!(
+ partial_frame(b"event: a\r\ndata: 1\r\n\r\ndata: {\"half"),
+ b"data: {\"half"
+ );
+ assert_eq!(partial_frame(b"event: a\ndata: 1\n\n"), b"");
+ assert_eq!(
+ partial_frame(b"no boundary yet"),
+ b"no boundary yet",
+ "a connection cut inside its first record is all partial frame"
+ );
+ assert_eq!(
+ partial_frame(b"data: 1\r\n\r\ndata: 2\n\ndata: 3"),
+ b"data: 3",
+ "the later of the two line endings' boundaries wins"
+ );
+}
+
+#[derive(Debug)]
+struct Layer {
+ message: &'static str,
+ source: Option<Box<Layer>>,
+}
+
+impl std::fmt::Display for Layer {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str(self.message)
+ }
+}
+
+impl std::error::Error for Layer {
+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+ self.source
+ .as_deref()
+ .map(|layer| layer as &(dyn std::error::Error + 'static))
+ }
+}
+
+/// The whole point: `error decoding response body` on its own names
+/// nothing, and the reset or h2 error underneath is only in the chain.
+#[test]
+fn the_error_chain_names_every_cause() {
+ let error = Layer {
+ message: "error decoding response body",
+ source: Some(Box::new(Layer {
+ message: "error reading a body from connection",
+ source: Some(Box::new(Layer {
+ message: "connection reset by peer (os error 104)",
+ source: None,
+ })),
+ })),
+ };
+ assert_eq!(
+ error_chain(&error),
+ "error decoding response body: error reading a body from connection: connection reset by peer (os error 104)"
+ );
+}
+
+#[test]
+fn a_cause_already_quoted_by_its_parent_is_not_repeated() {
+ let error = Layer {
+ message: "outer: inner",
+ source: Some(Box::new(Layer {
+ message: "inner",
+ source: None,
+ })),
+ };
+ assert_eq!(error_chain(&error), "outer: inner");
+}
+
+#[test]
+fn compression_is_anything_but_identity() {
+ let facts = |encoding: Option<&str>| ConnectionFacts {
+ status: 200,
+ version: "HTTP/2.0".to_owned(),
+ content_type: Some("text/event-stream".to_owned()),
+ content_encoding: encoding.map(str::to_owned),
+ transfer_encoding: None,
+ server: None,
+ request_id: None,
+ resumed: false,
+ };
+ assert!(!facts(None).is_compressed());
+ assert!(!facts(Some("identity")).is_compressed());
+ assert!(!facts(Some("Identity")).is_compressed());
+ assert!(facts(Some("gzip")).is_compressed());
+ assert!(facts(Some("br")).is_compressed());
+}
+
+#[test]
+fn the_preview_keeps_control_characters_on_one_line() {
+ assert_eq!(preview(b"data: a\r\n\r\n"), "data: a\\r\\n\\r\\n");
+ assert_eq!(preview(&[0xff, b'o', b'k']), "\u{fffd}ok");
+}
diff --git a/crates/cursor_cloud_agents/src/domain/journal.rs b/crates/cursor_cloud_agents/src/domain/journal.rs
index 5921e1aa0c..514aa05262 100644
--- a/crates/cursor_cloud_agents/src/domain/journal.rs
+++ b/crates/cursor_cloud_agents/src/domain/journal.rs
@@ -124,6 +124,12 @@ struct RunState {
/// What the reader sees of `text`: the same stream minus the `<img>`
/// tags Cursor writes for files only its sandbox can reach.
images: InlineImageFilter,
+ /// Whether the transport broke after the last content record, so that
+ /// `text` may be missing its end. Set by an interruption, cleared by
+ /// the next content record: a resume that continues the stream leaves
+ /// no gap, and only a run whose outcome then arrives by polling is
+ /// still behind when its final text is compared.
+ interrupted: bool,
}
/// Complete live/replay state, including user prompts and terminal tool cleanup.
#[derive(Debug, Default)]
@@ -204,7 +210,12 @@ impl ReplayMachine {
.map(|b| SessionUpdate::UserMessageChunk(ContentChunk::new(b)))
.collect())
}
- JournalInput::Sse(record) => self.event(run, record.decode()),
+ JournalInput::Sse(record) => {
+ if record.is_content() {
+ state.interrupted = false;
+ }
+ self.event(run, record.decode())
+ }
JournalInput::ArtifactsCollected(artifacts) => {
if artifacts.is_empty() {
return Ok(Vec::new());
@@ -251,12 +262,14 @@ impl ReplayMachine {
// This closes local work, but does not claim remote capture is complete.
Ok(self.translator.close_open_calls())
}
+ JournalInput::TransportError(_) | JournalInput::StreamInterrupted { .. } => {
+ state.interrupted = true;
+ Ok(Vec::new())
+ }
JournalInput::HistoryComplete
| JournalInput::Reconciled
| JournalInput::PromptAccepted(_)
- | JournalInput::PromptAborted(_)
- | JournalInput::TransportError(_)
- | JournalInput::StreamInterrupted { .. } => Ok(Vec::new()),
+ | JournalInput::PromptAborted(_) => Ok(Vec::new()),
}
}
fn event(
@@ -335,6 +348,29 @@ impl ReplayMachine {
}));
state.text = text;
}
+ // The stream broke and nothing came after it, so the
+ // captured text is the front of an answer whose end
+ // was never streamed. Streamed chunks cannot be taken
+ // back, so the whole answer follows as its own
+ // passage: the user sees the fragment and then the
+ // answer, rather than the fragment alone.
+ None if state.interrupted => {
+ tracing::warn!(
+ %run,
+ captured = state.text.len(),
+ restated = text.len(),
+ "Cursor restated the answer after a stream gap; appending it whole"
+ );
+ let separator = if state.text.is_empty() || state.text.ends_with('\n') {
+ ""
+ } else {
+ "\n\n"
+ };
+ updates.extend(self.translator.push(CursorEvent::Assistant {
+ text: state.images.push(&format!("{separator}{text}")),
+ }));
+ state.text = text;
+ }
None => tracing::warn!(
%run,
captured = state.text.len(),
diff --git a/crates/cursor_cloud_agents/src/domain/journal/test.rs b/crates/cursor_cloud_agents/src/domain/journal/test.rs
index 5740ec5a62..2253b27046 100644
--- a/crates/cursor_cloud_agents/src/domain/journal/test.rs
+++ b/crates/cursor_cloud_agents/src/domain/journal/test.rs
@@ -400,6 +400,106 @@ fn a_stream_interruption_projects_to_nothing() {
);
}
+/// The stream broke, nothing came after it, and the run's outcome arrived by
+/// polling: the captured text is the front of an answer whose end was never
+/// streamed, and a restatement that does not continue it is that answer, not
+/// a rewrite. It follows as its own passage rather than being dropped.
+#[test]
+fn a_restated_answer_after_a_stream_gap_is_appended_whole() {
+ let run = CursorRunId::new("gap");
+ let mut machine = ReplayMachine::default();
+ machine
+ .push(
+ Some(&run),
+ &JournalInput::Sse(crate::testing::raw_record(CursorEvent::Assistant {
+ text: "Looking into".into(),
+ })),
+ )
+ .unwrap();
+ machine
+ .push(
+ Some(&run),
+ &JournalInput::StreamInterrupted {
+ reason: "error decoding response body".into(),
+ last_event_id: Some("evt-1".into()),
+ attempt: 1,
+ },
+ )
+ .unwrap();
+ let updates = machine
+ .push(
+ Some(&run),
+ &JournalInput::Poll(
+ serde_json::json!({"status": "FINISHED", "result": "The bell icon was hidden by a stale flag."})
+ .to_string(),
+ ),
+ )
+ .unwrap();
+ let text: String = updates
+ .iter()
+ .filter_map(|update| match update {
+ SessionUpdate::AgentMessageChunk(chunk) => match &chunk.content {
+ ContentBlock::Text(text) => Some(text.text.clone()),
+ _ => None,
+ },
+ _ => None,
+ })
+ .collect();
+ assert_eq!(
+ text, "\n\nThe bell icon was hidden by a stale flag.",
+ "the whole answer follows the fragment as a fresh passage"
+ );
+ assert_eq!(
+ machine.runs[&run].text, "The bell icon was hidden by a stale flag.",
+ "the answer of record is the restated one"
+ );
+ assert_eq!(machine.terminal_status(&run), Some(RunStatus::Finished));
+}
+
+/// A resume that continued the stream leaves no gap: content after the
+/// interruption clears it, and a restatement that then differs is Cursor's
+/// rewrite - inline images dropped - which the streamed text still outranks.
+#[test]
+fn a_resumed_stream_has_no_gap_to_fill() {
+ let run = CursorRunId::new("resumed");
+ let mut machine = ReplayMachine::default();
+ machine
+ .push(
+ Some(&run),
+ &JournalInput::Sse(crate::testing::raw_record(CursorEvent::Assistant {
+ text: "captured ".into(),
+ })),
+ )
+ .unwrap();
+ machine
+ .push(
+ Some(&run),
+ &JournalInput::StreamInterrupted {
+ reason: "error decoding response body".into(),
+ last_event_id: Some("evt-1".into()),
+ attempt: 1,
+ },
+ )
+ .unwrap();
+ machine
+ .push(
+ Some(&run),
+ &JournalInput::Sse(crate::testing::raw_record(CursorEvent::Assistant {
+ text: "answer".into(),
+ })),
+ )
+ .unwrap();
+ machine
+ .push(
+ Some(&run),
+ &JournalInput::Poll(
+ serde_json::json!({"status": "FINISHED", "result": "different answer"}).to_string(),
+ ),
+ )
+ .unwrap();
+ assert_eq!(machine.runs[&run].text, "captured answer");
+}
+
/// The screenshot turn of dev session 01a0b4ed-29e1-7634-9a67-b87aa0d3efe8:
/// Cursor streamed `<img src="hello_world_browser.png" ... />` a token at a
/// time, the file only ever existed in its sandbox, and the same image then
diff --git a/crates/cursor_cloud_agents/src/domain/ports.rs b/crates/cursor_cloud_agents/src/domain/ports.rs
index 2a595875fb..17c42746b4 100644
--- a/crates/cursor_cloud_agents/src/domain/ports.rs
+++ b/crates/cursor_cloud_agents/src/domain/ports.rs
@@ -207,8 +207,10 @@ pub struct ConnectedStream<Records> {
/// Why one connect did not produce a stream.
///
/// The domain acts differently on each: an unavailable stream is retried, an
-/// invalid resume position is retried without one, and an expired stream is
-/// gone for good and leaves polling as the only way to learn the outcome.
+/// invalid resume position is retried without one, an expired stream is
+/// gone for good and leaves polling as the only way to learn the outcome,
+/// and a stalled connect is checked against the run's record before it is
+/// tried again.
#[derive(Debug)]
pub enum StreamConnectError {
/// `stream_unavailable`: the stream is not there (yet). Carries the
@@ -218,6 +220,16 @@ pub enum StreamConnectError {
InvalidResumePosition(String),
/// `stream_expired`: the retention window closed behind us.
Expired(String),
+ /// The provider accepted the connection but sent no response headers
+ /// within the adapter's patience.
+ ///
+ /// Observed live (prod, 2026-09-26): a resume after a mid-run body
+ /// failure was held for 6m43s with no headers, then answered with every
+ /// record since the break in one burst, the moment the run finished. A
+ /// connect that waits indefinitely turns that into a turn nobody can see
+ /// working; this variant is what lets the domain keep polling the run's
+ /// record in the meantime.
+ Stalled(String),
/// Every other failure.
Other(rootcause::Report),
}
@@ -230,6 +242,12 @@ impl std::fmt::Display for StreamConnectError {
write!(formatter, "Cursor rejected the resume position: {message}")
}
Self::Expired(message) => write!(formatter, "Cursor stream expired: {message}"),
+ Self::Stalled(message) => {
+ write!(
+ formatter,
+ "Cursor stream stalled before its headers: {message}"
+ )
+ }
Self::Other(report) => write!(formatter, "{report}"),
}
}
diff --git a/crates/cursor_cloud_agents/src/domain/service.rs b/crates/cursor_cloud_agents/src/domain/service.rs
index 14d9b934e6..76fae84bff 100644
--- a/crates/cursor_cloud_agents/src/domain/service.rs
+++ b/crates/cursor_cloud_agents/src/domain/service.rs
@@ -155,10 +155,33 @@ const STREAM_RECONNECT_DELAYS: [std::time::Duration; 5] = [
];
/// Reconnects before a run gives up on its stream and polls instead. Counted
-/// since the last reconnect that delivered content, so a long run that drops
-/// repeatedly but recovers each time is never rationed.
+/// since the last reconnect that delivered content and then stayed up (see
+/// [`STREAM_HEALTHY_RESUME`]), so a long run that drops repeatedly but
+/// recovers each time is never rationed.
const STREAM_RECONNECT_ATTEMPTS: usize = 5;
+/// How long a resumed connection has to stay up, once it has delivered,
+/// before it counts as the transport working again and refills the budget.
+///
+/// Every resume in one day's incidents (prod, 2026-09-26, eight runs)
+/// delivered its whole backlog in a burst and failed within the second, from
+/// the very id it had just advanced to. Refilling on delivery alone made
+/// each of those a fresh first attempt: the budget never ran down, the
+/// attempt counter in every log line read `1`, and nothing could say "this
+/// transport has been failing for an hour". A connection that outlives this
+/// after delivering is one the provider is genuinely streaming on.
+const STREAM_HEALTHY_RESUME: std::time::Duration = std::time::Duration::from_secs(5);
+
+/// Connects that stall before their headers, on a run whose stream never
+/// opened, before the turn stops trying and polls the record instead.
+///
+/// Once a stream has opened, stalls are not counted against anything: the
+/// run's record is checked on each, so a finished run ends its turn within
+/// a stall of finishing, and a run still going is still worth the stream.
+/// Before it has ever opened there is nothing to resume, and a connect that
+/// stalls three times running is a stream that is not going to open.
+const STALLED_CONNECTS_BEFORE_POLLING: u32 = 3;
+
/// How long a prompt waits behind a run something else started (the same
/// agent is drivable from cursor.com) before giving up, in poll intervals.
const BUSY_ATTEMPTS: usize = 450;
@@ -1606,6 +1629,19 @@ where
// The run's latest `status` frame, kept to recognize the sticky copy
// Cursor re-sends at the top of every reconnect.
let mut last_status: Option<crate::domain::journal::NativeRecord> = None;
+ // Connects in a row that stalled before their headers; a connection
+ // that opens resets it.
+ let mut stalled_connects: u32 = 0;
+ // Whether the record was found terminal behind a stalled stream and
+ // the one connect allowed to drain that stream has been spent.
+ let mut drained_after_terminal = false;
+ // When the transport last broke, and when a record last arrived on
+ // any connection: together they say how long the transcript has
+ // been behind, which is the figure a person watching a frozen chip
+ // needs from the logs.
+ let started_at = tokio::time::Instant::now();
+ let mut interrupted_at: Option<tokio::time::Instant> = None;
+ let mut last_record_at: Option<tokio::time::Instant> = None;
loop {
// The connected stream borrows the position it resumed from, so
// the position this iteration sends is its own owned copy.
@@ -1615,6 +1651,68 @@ where
.raw_stream(agent, run, resuming.as_deref())
.await
{
+ // The stream said nothing, but the record still answers: a
+ // run that finished behind a wedged stream must not wait on
+ // another stall to end its turn. Hydration takes the strict
+ // arm below; it has no record to fall back on.
+ Err(StreamConnectError::Stalled(detail)) if !strict => {
+ stalled_connects += 1;
+ let raw = self
+ .fetch_run_record(session_id, agent, run, cancel)
+ .await?;
+ let outcome = Self::run_outcome(&raw)?;
+ if outcome.is_terminal()
+ && !std::mem::replace(&mut drained_after_terminal, true)
+ {
+ // Observed: what a stalled stream held back arrives
+ // the instant the run ends, so the stream gets one
+ // more connect to deliver it - and its terminal
+ // frame - itself. The record is deliberately not
+ // captured yet: its final text would reach the
+ // transcript now and then again from the stream.
+ tracing::info!(
+ cursor.run.id = %run,
+ cursor.stream.stalled_connects = stalled_connects,
+ cursor.stream.last_event_id = last_event_id,
+ "Cursor run ended behind a stalled stream; connecting once more to drain it"
+ );
+ } else {
+ self.capture(
+ session_id,
+ session,
+ Some(run),
+ JournalInput::Poll(raw),
+ emit,
+ )
+ .await?;
+ if outcome.is_terminal() {
+ // A second stall past the run's end: the
+ // record's account stands.
+ terminal = Some(outcome.status);
+ tracing::warn!(
+ cursor.run.id = %run,
+ cursor.stream.stalled_connects = stalled_connects,
+ cursor.stream.last_event_id = last_event_id,
+ "Cursor stream still stalled after the run ended; keeping the record's account"
+ );
+ break;
+ }
+ }
+ if cancel.is_cancelled() {
+ break;
+ }
+ Some(Interruption {
+ reason: format!("{detail} (stalled connect {stalled_connects})"),
+ recoverable: ever_connected
+ || stalled_connects < STALLED_CONNECTS_BEFORE_POLLING,
+ quiet: true,
+ })
+ }
+ Err(StreamConnectError::Stalled(detail)) => Some(Interruption {
+ reason: detail,
+ recoverable: false,
+ quiet: false,
+ }),
Err(StreamConnectError::InvalidResumePosition(detail))
if blind_reconnect_available && resuming.is_some() =>
{
@@ -1645,6 +1743,9 @@ where
Ok(connected) => {
let resumed = resuming.is_some();
ever_connected = true;
+ stalled_connects = 0;
+ let opened_at = tokio::time::Instant::now();
+ let mut first_record_at: Option<tokio::time::Instant> = None;
let retention_seconds = connected.retention_seconds;
let records = connected.records;
pin_mut!(records);
@@ -1664,6 +1765,9 @@ where
{
Ok(Some(Ok(record))) => {
silence = std::time::Duration::ZERO;
+ let now = tokio::time::Instant::now();
+ first_record_at.get_or_insert(now);
+ last_record_at = Some(now);
record
}
Ok(Some(Err(error))) => {
@@ -1792,18 +1896,33 @@ where
}
};
if resumed {
+ let connection = opened_at.elapsed();
+ // A resume that delivered and then stayed up is a
+ // working stream again, so the next drop gets the
+ // full budget rather than whatever a much earlier one
+ // left over. One that delivered its backlog and died
+ // at once is the same failure continuing, and spends.
+ let healthy = received_content > 0
+ && (interruption.is_none() || connection >= STREAM_HEALTHY_RESUME);
tracing::info!(
cursor.run.id = %run,
cursor.stream.attempt = reconnects,
cursor.stream.last_event_id = last_event_id,
cursor.stream.retention_seconds = retention_seconds,
cursor.stream.resumed_records = received,
+ cursor.stream.resumed_content = received_content,
+ cursor.stream.waited_ms = interrupted_at.map(|at| {
+ first_record_at
+ .unwrap_or_else(tokio::time::Instant::now)
+ .saturating_duration_since(at)
+ .as_millis() as u64
+ }),
+ cursor.stream.connection_ms = connection.as_millis() as u64,
+ cursor.stream.healthy = healthy,
+ cursor.stream.fast_failed = interruption.is_some() && !healthy,
"Cursor stream resumed"
);
- // A resume that delivered is a working stream again,
- // so the next drop gets the full budget rather than
- // whatever a much earlier one left over.
- if received_content > 0 {
+ if healthy {
reconnects = 0;
}
}
@@ -1819,7 +1938,15 @@ where
break;
};
fell_back_to_poll = true;
- if !ever_connected {
+ interrupted_at = Some(tokio::time::Instant::now());
+ // How long the transcript has been behind: since the last record
+ // on any connection, or since this attempt began if none came.
+ let silent_ms = last_record_at.unwrap_or(started_at).elapsed().as_millis() as u64;
+ // A connect that stalled is tried again a bounded number of
+ // times before this gives up on the stream, because a stall is
+ // the one connect failure the client did not already retry.
+ let retrying_a_stall = quiet && recoverable;
+ if !ever_connected && !retrying_a_stall {
// Nothing to resume: the stream never opened, and the client
// has already retried an unavailable one on its own.
self.capture(
@@ -1833,6 +1960,8 @@ where
tracing::warn!(
cursor.run.id = %run,
cursor.stream.reason = %reason,
+ cursor.stream.stalled_connects = stalled_connects,
+ cursor.stream.silent_ms = silent_ms,
"Cursor stream never connected; polling the run record instead"
);
break;
@@ -1863,6 +1992,8 @@ where
cursor.stream.last_event_id = last_event_id,
cursor.stream.attempt = attempt,
cursor.stream.reason = %reason,
+ cursor.stream.silent_ms = silent_ms,
+ cursor.stream.stalled_connects = stalled_connects,
"Cursor stream is not resumable; polling the run record instead"
);
break;
@@ -1872,6 +2003,9 @@ where
cursor.stream.last_event_id = last_event_id,
cursor.stream.attempt = attempt,
cursor.stream.reason = %reason,
+ cursor.stream.quiet = quiet,
+ cursor.stream.silent_ms = silent_ms,
+ cursor.stream.stalled_connects = stalled_connects,
"Cursor stream interrupted; reconnecting"
);
let delay = STREAM_RECONNECT_DELAYS
@@ -1990,6 +2124,33 @@ where
///
/// `debug` because the loop runs as often as [`POLL_DELAYS`] says; the turn span
/// above carries the outcome, this carries the liveness.
+ async fn poll_once(
+ &self,
+ session_id: &SessionId,
+ session: &Session,
+ agent: &CursorAgentId,
+ run: &CursorRunId,
+ cancel: &tokio_util::sync::CancellationToken,
+ emit: Emit,
+ ) -> Result<crate::domain::model::RunOutcome, SessionError> {
+ let raw = self
+ .fetch_run_record(session_id, agent, run, cancel)
+ .await?;
+ self.capture(
+ session_id,
+ session,
+ Some(run),
+ JournalInput::Poll(raw.clone()),
+ emit,
+ )
+ .await?;
+ Self::run_outcome(&raw)
+ }
+
+ /// The run's record as Cursor holds it, retried through transient
+ /// failures. Not captured: [`Self::poll_once`] does that, and the one
+ /// caller that reads the record without capturing it — a stalled stream
+ /// checking whether its run has ended — says why there.
#[tracing::instrument(
name = "cursor.run.poll",
level = "debug",
@@ -1998,27 +2159,19 @@ where
agent.acp.session_id = ?session_id,
cursor.agent.id = %agent,
cursor.run.id = %run,
- cursor.run.status = tracing::field::Empty,
),
err,
)]
- async fn poll_once(
+ async fn fetch_run_record(
&self,
session_id: &SessionId,
- session: &Session,
agent: &CursorAgentId,
run: &CursorRunId,
cancel: &tokio_util::sync::CancellationToken,
- emit: Emit,
- ) -> Result<crate::domain::model::RunOutcome, SessionError> {
- let mut raw = None;
- for attempt in 0..=POLL_ERROR_TOLERANCE {
+ ) -> Result<String, SessionError> {
+ for _tolerated in 0..POLL_ERROR_TOLERANCE {
match self.cursor.raw_result(agent, run).await {
- Ok(value) => {
- raw = Some(value);
- break;
- }
- Err(error) if attempt == POLL_ERROR_TOLERANCE => return Err(error.into()),
+ Ok(value) => return Ok(value),
Err(error) => {
tracing::warn!(error = ?error, "Cursor poll failed; retrying");
if sleep_unless_cancelled(cancel, POLL_INTERVAL).await {
@@ -2027,17 +2180,14 @@ where
}
}
}
- let raw = raw.expect("poll returned or failed");
- self.capture(
- session_id,
- session,
- Some(run),
- JournalInput::Poll(raw.clone()),
- emit,
- )
- .await?;
+ // The last try past the tolerance is the one whose failure is final.
+ self.cursor.raw_result(agent, run).await.map_err(Into::into)
+ }
+
+ /// The outcome a run record states.
+ fn run_outcome(raw: &str) -> Result<crate::domain::model::RunOutcome, SessionError> {
let value: serde_json::Value =
- serde_json::from_str(&raw).map_err(|e| rootcause::report!(e).into_dynamic())?;
+ serde_json::from_str(raw).map_err(|e| rootcause::report!(e).into_dynamic())?;
Ok(crate::domain::model::RunOutcome {
status: serde_json::from_value(value["status"].clone())
.map_err(|e| rootcause::report!(e).into_dynamic())?,
diff --git a/crates/cursor_cloud_agents/src/domain/service/test.rs b/crates/cursor_cloud_agents/src/domain/service/test.rs
index d17e2a934a..94f7a7c13d 100644
--- a/crates/cursor_cloud_agents/src/domain/service/test.rs
+++ b/crates/cursor_cloud_agents/src/domain/service/test.rs
@@ -3602,6 +3602,362 @@ async fn a_rejected_resume_position_reconnects_once_without_one() {
);
}
+/// Prod, 2026-09-26: a resume was held open with no headers for 6m43s and
+/// answered all at once when the run ended. The connect now gives up waiting
+/// for headers, and each stall checks the run's record: a run still going
+/// gets its stream tried again without spending the failure budget, and a
+/// run that has ended gets one more connect to drain what the stream held.
+#[tokio::test(start_paused = true)]
+async fn a_stalled_connect_checks_the_record_and_drains_the_stream_once_the_run_ends() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "before the stall".to_owned(),
+ },
+ "evt-1",
+ )
+ .expect("stream open");
+ drop(dropped);
+ // Two resumes stall. The record says the run is still going after the
+ // first and finished after the second.
+ cursor.script_stream_connect_error(StreamConnectError::Stalled(
+ "no response headers after 30s".to_owned(),
+ ));
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Running,
+ text: None,
+ });
+ cursor.script_stream_connect_error(StreamConnectError::Stalled(
+ "no response headers after 30s".to_owned(),
+ ));
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Finished,
+ text: Some("before the stall and after it".to_owned()),
+ });
+ // The drain delivers what the stalled stream held back, terminal frame
+ // included.
+ let drained = cursor.script_stream();
+ drained
+ .send(CursorEvent::Assistant {
+ text: " and after it".to_owned(),
+ })
+ .expect("stream open");
+ drained.send(finished("run-fake-1")).expect("stream open");
+ drained.send(CursorEvent::Done).expect("stream open");
+ drop(drained);
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the drained stream finishes the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(
+ cursor.resume_positions(),
+ vec![
+ None,
+ Some("evt-1".to_owned()),
+ Some("evt-1".to_owned()),
+ Some("evt-1".to_owned()),
+ ],
+ "every connect after the break resumes from the last id heard"
+ );
+ assert_eq!(
+ agent_texts(&notifier.updates()),
+ vec!["before the stall", " and after it"],
+ "the stream's own account is delivered once, never the record's on top of it"
+ );
+ let entries = service.journal.read(&session).await.expect("journal");
+ let stalls: Vec<_> = entries
+ .iter()
+ .filter_map(|entry| match &entry.input {
+ JournalInput::StreamInterrupted {
+ reason, attempt, ..
+ } if reason.contains("stalled connect") => Some((reason.clone(), *attempt)),
+ _ => None,
+ })
+ .collect();
+ assert_eq!(
+ stalls,
+ vec![
+ (
+ "no response headers after 30s (stalled connect 1)".to_owned(),
+ 2
+ ),
+ (
+ "no response headers after 30s (stalled connect 2)".to_owned(),
+ 2
+ ),
+ ],
+ "each stall is durable, and none of them spends the reconnect budget"
+ );
+ assert_eq!(
+ entries
+ .iter()
+ .filter(|entry| matches!(entry.input, JournalInput::Poll(_)))
+ .count(),
+ 1,
+ "the record is captured while the run is going, not the terminal one that the drain overtook"
+ );
+}
+
+/// When the drain stalls too, the record's account stands - and because the
+/// stream broke and never came back, the restated answer is appended whole
+/// rather than dropped for not continuing the fragment that was streamed.
+#[tokio::test(start_paused = true)]
+async fn a_stream_still_stalled_after_the_run_ends_keeps_the_record() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "Looking into".to_owned(),
+ },
+ "evt-1",
+ )
+ .expect("stream open");
+ drop(dropped);
+ for _ in 0..2 {
+ cursor.script_stream_connect_error(StreamConnectError::Stalled(
+ "no response headers after 30s".to_owned(),
+ ));
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Finished,
+ text: Some("The bell icon was hidden by a stale flag.".to_owned()),
+ });
+ }
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the record ends the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(
+ cursor.resume_positions().len(),
+ 3,
+ "the first connect, the stalled resume, and the one drain attempt"
+ );
+ assert_eq!(
+ agent_texts(&notifier.updates()),
+ vec![
+ "Looking into",
+ "\n\nThe bell icon was hidden by a stale flag."
+ ],
+ "the answer the stream never delivered still reaches the client"
+ );
+}
+
+/// Before the stream has ever opened there is nothing to resume, so stalls
+/// are bounded: a few are tried, then the record is polled instead.
+#[tokio::test(start_paused = true)]
+async fn stalls_before_the_stream_ever_opened_fall_back_to_polling() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ for _ in 0..STALLED_CONNECTS_BEFORE_POLLING {
+ cursor.script_stream_connect_error(StreamConnectError::Stalled(
+ "no response headers after 30s".to_owned(),
+ ));
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Running,
+ text: None,
+ });
+ }
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Finished,
+ text: Some("the whole answer".to_owned()),
+ });
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the poll ends the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(
+ cursor.resume_positions().len(),
+ STALLED_CONNECTS_BEFORE_POLLING as usize,
+ "a stream that never opens is tried a bounded number of times"
+ );
+ assert_eq!(agent_texts(&notifier.updates()), vec!["the whole answer"]);
+ let entries = service.journal.read(&session).await.expect("journal");
+ assert!(
+ entries.iter().any(|entry| matches!(
+ &entry.input,
+ JournalInput::TransportError(reason) if reason.contains("stalled connect 3")
+ )),
+ "giving up on the stream is journaled as the transport failure it is"
+ );
+}
+
+/// Every resume in the 2026-09-26 incidents delivered its backlog in one
+/// burst and failed within the second. Delivery alone no longer refills the
+/// reconnect budget: a resume that dies at once is the same failure
+/// continuing, and after the budget the record is polled instead.
+#[tokio::test(start_paused = true)]
+async fn a_resume_that_delivers_and_dies_at_once_spends_the_budget() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "a".to_owned(),
+ },
+ "evt-0",
+ )
+ .expect("stream open");
+ drop(dropped);
+ for index in 1..=STREAM_RECONNECT_ATTEMPTS {
+ let burst = cursor.script_stream_failing_with("error decoding response body");
+ burst
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "a".to_owned(),
+ },
+ &format!("evt-{index}"),
+ )
+ .expect("stream open");
+ drop(burst);
+ }
+ // One more burst is queued than the budget allows; it must go unread.
+ let unread = cursor.script_stream();
+ cursor.script_run_result(RunOutcome {
+ status: RunStatus::Finished,
+ text: Some("a".repeat(1 + STREAM_RECONNECT_ATTEMPTS)),
+ });
+
+ let stop = service
+ .prompt(&session, "hi")
+ .await
+ .expect("the poll ends the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ drop(unread);
+ assert_eq!(
+ cursor.resume_positions().len(),
+ 1 + STREAM_RECONNECT_ATTEMPTS,
+ "the budget is spent by resumes that fail at once, however much they delivered"
+ );
+ assert_eq!(
+ agent_texts(&notifier.updates()).concat(),
+ "a".repeat(1 + STREAM_RECONNECT_ATTEMPTS),
+ "everything each burst delivered still reached the client, once"
+ );
+ let entries = service.journal.read(&session).await.expect("journal");
+ let attempts: Vec<u32> = entries
+ .iter()
+ .filter_map(|entry| match &entry.input {
+ JournalInput::StreamInterrupted { attempt, .. } => Some(*attempt),
+ _ => None,
+ })
+ .collect();
+ assert_eq!(
+ attempts,
+ (1..=1 + STREAM_RECONNECT_ATTEMPTS as u32).collect::<Vec<_>>(),
+ "the attempt counter climbs instead of reading 1 on every line"
+ );
+}
+
+/// A resume that delivers and then stays up is the transport working again,
+/// and the next break gets a full budget.
+#[tokio::test(start_paused = true)]
+async fn a_resume_that_stays_up_refills_the_budget() {
+ let (service, cursor, notifier) = service(None);
+ let session = service.new_session(Path::new(""), Vec::new());
+
+ let dropped = cursor.script_stream_failing_with("error decoding response body");
+ dropped
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "a".to_owned(),
+ },
+ "evt-0",
+ )
+ .expect("stream open");
+ drop(dropped);
+ // The budget's worth of resumes that die at once, then one that lives.
+ for index in 1..STREAM_RECONNECT_ATTEMPTS {
+ let burst = cursor.script_stream_failing_with("error decoding response body");
+ burst
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "a".to_owned(),
+ },
+ &format!("evt-{index}"),
+ )
+ .expect("stream open");
+ drop(burst);
+ }
+ let lively = cursor.script_stream_failing_with("error decoding response body");
+ lively
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "b".to_owned(),
+ },
+ "evt-lively",
+ )
+ .expect("stream open");
+ // After the refill, another budget's worth of fast failures is survivable.
+ for index in 0..STREAM_RECONNECT_ATTEMPTS - 1 {
+ let burst = cursor.script_stream_failing_with("error decoding response body");
+ burst
+ .send_with_id(
+ CursorEvent::Assistant {
+ text: "c".to_owned(),
+ },
+ &format!("evt-c{index}"),
+ )
+ .expect("stream open");
+ drop(burst);
+ }
+ let last = cursor.script_stream();
+ last.send(finished("run-fake-1")).expect("stream open");
+ last.send(CursorEvent::Done).expect("stream open");
+ drop(last);
+
+ let turn = {
+ let service = Arc::clone(&service);
+ let session = session.clone();
+ tokio::spawn(async move { service.prompt(&session, "hi").await })
+ };
+ // Let the lively connection outlive the health threshold before it dies.
+ // The bursts before it reconnect on the backoff schedule, which paused
+ // time advances through as the turn sleeps.
+ tokio::time::sleep(
+ STREAM_RECONNECT_DELAYS.iter().sum::<std::time::Duration>()
+ + STREAM_HEALTHY_RESUME
+ + std::time::Duration::from_secs(1),
+ )
+ .await;
+ drop(lively);
+
+ let stop = turn
+ .await
+ .expect("the turn task completes")
+ .expect("the stream finishes the turn");
+ assert_eq!(stop, StopReason::EndTurn);
+ assert_eq!(
+ cursor.resume_positions().len(),
+ // The first connect, the bursts before the lively resume, the lively
+ // resume itself, the bursts after it, and the stream that finished.
+ 1 + (STREAM_RECONNECT_ATTEMPTS - 1) + 1 + (STREAM_RECONNECT_ATTEMPTS - 1) + 1,
+ "the lively resume refilled the budget for the bursts after it"
+ );
+ assert_eq!(
+ agent_texts(&notifier.updates()).concat(),
+ format!(
+ "{}b{}",
+ "a".repeat(STREAM_RECONNECT_ATTEMPTS),
+ "c".repeat(STREAM_RECONNECT_ATTEMPTS - 1)
+ )
+ );
+}
+
/// Cursor does not heartbeat an open stream, and a run at work says nothing
/// for minutes at a time - a tool call waiting on CI, a long build. Seen
/// live: a run that opened its pull request a minute after its turn had