Lightweight coding agent that runs in your terminal
Rust
Make diagnostic report uploads resilient to slow networks (#42096)
chessa17ee5705c27
diff --git a/codex-rs/app-server/src/request_processors/feedback_processor.rs b/codex-rs/app-server/src/request_processors/feedback_processor.rsindex 0d9cc2abac..1365cf7ecb 100644--- a/codex-rs/app-server/src/request_processors/feedback_processor.rs+++ b/codex-rs/app-server/src/request_processors/feedback_processor.rs@@ -1,5 +1,6 @@use super::feedback_thread_index::FeedbackThreadIndex;use super::*;+use crate::error_code::OVERLOADED_ERROR_CODE;use codex_connectors::ConnectorDirectoryCacheContext;use codex_connectors::ConnectorDirectoryCacheKey;use codex_connectors::connector_runtime_cache_path;@@ -11,6 +12,7 @@ use codex_feedback::guardian_review_failures;use codex_rollout::RolloutRecorder;use sha2::Digest;use sha2::Sha256;+use tokio::sync::Semaphore;#[derive(Clone)]pub(crate) struct FeedbackRequestProcessor {@@ -20,6 +22,7 @@ pub(crate) struct FeedbackRequestProcessor {feedback: CodexFeedback,log_db: Option<LogDbLayer>,state_db: Option<StateDbHandle>,+ uploads: Arc<Semaphore>,}impl FeedbackRequestProcessor {@@ -38,6 +41,7 @@ impl FeedbackRequestProcessor {feedback,log_db,state_db,+ uploads: Arc::new(Semaphore::new(/*permits*/ 3)),}}@@ -59,6 +63,17 @@ impl FeedbackRequestProcessor {"sending feedback is disabled by configuration",));}+ let permit = self+ .uploads+ .clone()+ .try_acquire_owned()+ .map_err(|_| JSONRPCErrorError {+ code: OVERLOADED_ERROR_CODE,+ message:+ "Three feedback uploads are already in progress; try again after one finishes"+ .to_string(),+ data: None,+ })?;let FeedbackUploadParams {classification,@@ -269,6 +284,8 @@ impl FeedbackRequestProcessor {let runtime_handle = tokio::runtime::Handle::current();let upload_result = tokio::task::spawn_blocking(move || {+ // Cancelling the RPC waiter must not release a still-running upload's slot.+ let _permit = permit;let tags = (!upload_tags.is_empty()).then_some(&upload_tags);runtime_handle.block_on(snapshot.upload_feedback(FeedbackUploadOptions {diff --git a/codex-rs/app-server/tests/suite/v2/feedback.rs b/codex-rs/app-server/tests/suite/v2/feedback.rsindex d2e21dc5d0..fd6a2ef936 100644--- a/codex-rs/app-server/tests/suite/v2/feedback.rs+++ b/codex-rs/app-server/tests/suite/v2/feedback.rs@@ -5,21 +5,16 @@ use app_test_support::TestAppServer;use codex_app_server_protocol::RequestId;use pretty_assertions::assert_eq;use serde_json::json;+use tokio::io::AsyncBufReadExt;+use tokio::io::AsyncWriteExt;+use tokio::io::BufReader;+use tokio::net::TcpListener;use tokio::time::timeout;-use wiremock::Mock;-use wiremock::MockServer;-use wiremock::ResponseTemplate;-use wiremock::matchers::method;#[tokio::test]-async fn feedback_upload_reports_transport_failure_as_json_rpc_error() -> Result<()> {- let proxy = MockServer::start().await;- Mock::given(method("CONNECT"))- .respond_with(ResponseTemplate::new(503))- .expect(1)- .mount(&proxy)- .await;- let proxy_uri = proxy.uri();+async fn feedback_upload_limits_concurrency_and_releases_failed_uploads() -> Result<()> {+ let proxy = TcpListener::bind("127.0.0.1:0").await?;+ let proxy_uri = format!("http://{}", proxy.local_addr()?);let mut app_server = TestAppServer::builder().with_env_overrides(&[("HTTPS_PROXY", Some(proxy_uri.as_str())),@@ -30,19 +25,68 @@ async fn feedback_upload_reports_transport_failure_as_json_rpc_error() -> Result.build_initialized().await?;+ let mut pending = Vec::new();+ for _ in 0..3 {+ let request_id = app_server+ .send_raw_request(+ "feedback/upload",+ Some(json!({ "classification": "bug", "includeLogs": false })),+ )+ .await?;+ let (stream, _) = timeout(Duration::from_secs(/*secs*/ 15), proxy.accept()).await??;+ let mut stream = BufReader::new(stream);+ let mut request = String::new();+ timeout(+ Duration::from_secs(/*secs*/ 15),+ stream.read_line(&mut request),+ )+ .await??;+ assert!(request.starts_with("CONNECT "));+ pending.push((request_id, stream.into_inner()));+ }++ let excess_id = app_server+ .send_raw_request(+ "feedback/upload",+ Some(json!({ "classification": "bug", "includeLogs": false })),+ )+ .await?;+ let error = timeout(+ Duration::from_secs(/*secs*/ 15),+ app_server.read_stream_until_error_message(RequestId::Integer(excess_id)),+ )+ .await??;++ assert_eq!(error.error.code, -32001);++ let unavailable =+ b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";+ for (request_id, mut stream) in pending {+ stream.write_all(unavailable).await?;+ stream.shutdown().await?;+ let error = timeout(+ Duration::from_secs(/*secs*/ 15),+ app_server.read_stream_until_error_message(RequestId::Integer(request_id)),+ )+ .await??;+ assert_eq!(error.error.code, -32603);+ assert!(error.error.message.contains("failed to upload feedback"));+ }+let request_id = app_server.send_raw_request("feedback/upload",Some(json!({ "classification": "bug", "includeLogs": false })),).await?;+ let (mut stream, _) = timeout(Duration::from_secs(/*secs*/ 15), proxy.accept()).await??;+ stream.write_all(unavailable).await?;+ stream.shutdown().await?;let error = timeout(- Duration::from_secs(15),+ Duration::from_secs(/*secs*/ 15),app_server.read_stream_until_error_message(RequestId::Integer(request_id)),).await??;-assert_eq!(error.error.code, -32603);- assert!(error.error.message.contains("failed to upload feedback"));Ok(())}diff --git a/codex-rs/feedback/src/lib.rs b/codex-rs/feedback/src/lib.rsindex 8921545018..4fe0db21b4 100644--- a/codex-rs/feedback/src/lib.rs+++ b/codex-rs/feedback/src/lib.rs@@ -56,7 +56,7 @@ pub const WINDOWS_SANDBOX_LOG_ATTACHMENT_FILENAME: &str = "windows-sandbox.log";const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024; // 4 MiBconst SENTRY_DSN: &str ="https://[email protected]/4510195390611458";-const UPLOAD_TIMEOUT_SECS: u64 = 10;+const UPLOAD_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 300);// Raw collection budgets used by the report API, not the interactive upload.pub const MAX_ATTACHMENT_BYTES: usize = 64 * 1024 * 1024;pub const MAX_ATTACHMENTS_BYTES: usize = 126 * 1024 * 1024;@@ -553,8 +553,13 @@ impl FeedbackSnapshot {options: FeedbackUploadOptions<'_>,http_client_factory: &HttpClientFactory,) -> Result<()> {- self.upload_feedback_with_dsn(options, http_client_factory, SENTRY_DSN)- .await+ self.upload_feedback_with_dsn(+ options,+ http_client_factory,+ SENTRY_DSN,+ Instant::now() + UPLOAD_TIMEOUT,+ )+ .await}async fn upload_feedback_with_dsn(@@ -562,6 +567,7 @@ impl FeedbackSnapshot {options: FeedbackUploadOptions<'_>,http_client_factory: &HttpClientFactory,dsn: &str,+ deadline: Instant,) -> Result<()> {use std::str::FromStr;@@ -592,15 +598,15 @@ impl FeedbackSnapshot {http_client_factory.clone(),ClientRouteClass::Other,);- // Acknowledge the report before reading diagnostic files. Keep one network wait- // budget for the whole submission; file reads and compression do not consume it.- let mut remaining_upload_time = Duration::from_secs(UPLOAD_TIMEOUT_SECS);+ // Accept the report before reading diagnostics; all envelopes share one deadline.+ let mut rate_limited = false;let status = upload::send_gzip_envelope(&client_pool,&dsn,event_body,upload::EnvelopeKind::Event,- &mut remaining_upload_time,+ deadline,+ &mut rate_limited,).await?;anyhow::ensure!(@@ -608,7 +614,7 @@ impl FeedbackSnapshot {"Sentry rejected feedback upload with HTTP status {status}");- let attachments = self.feedback_attachments(+ let mut attachments = self.feedback_attachments(options.include_logs,options.extra_attachments,options.extra_attachment_paths,@@ -617,8 +623,16 @@ impl FeedbackSnapshot {let mut uploaded_attachments = 0;let mut attachments_failed = false;// Keep attachments linked to the accepted event, without replaying its contents.- for attachment in attachments {- if remaining_upload_time.is_zero() {+ // Inspect the upper bound without reading the next file after the deadline.+ while attachments.size_hint().1 != Some(0) {+ if rate_limited || Instant::now() >= deadline {+ attachments_failed = true;+ break;+ }+ let Some(attachment) = attachments.next() else {+ break;+ };+ if Instant::now() >= deadline {attachments_failed = true;break;}@@ -630,7 +644,8 @@ impl FeedbackSnapshot {&dsn,body,upload::EnvelopeKind::Attachment,- &mut remaining_upload_time,+ deadline,+ &mut rate_limited,).await?;status = Some(response_status.as_u16());@@ -855,6 +870,7 @@ mod tests {use std::fs;use std::sync::atomic::AtomicUsize;use std::sync::atomic::Ordering;+ use std::time::Duration;use super::*;use crate::FeedbackDiagnostic;@@ -967,10 +983,94 @@ mod tests {},&HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),dsn,+ Instant::now() + UPLOAD_TIMEOUT,).await}+ #[tokio::test]+ async fn feedback_upload_allows_slow_reports() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/42/envelope/"))+ .respond_with(+ ResponseTemplate::new(StatusCode::OK)+ .set_delay(Duration::from_secs(/*secs*/ 4)),+ )+ .expect(/*r*/ 3)+ .mount(&server)+ .await;++ let dsn = format!("http://public@{}/42", server.address());+ let attachment = FeedbackAttachment {+ filename: "later.txt".to_string(),+ content_type: None,+ buffer: b"later diagnostic".to_vec(),+ };+ upload_test_feedback(&CodexFeedback::new(), &dsn, &[attachment])+ .await+ .expect("all three envelopes should finish across twelve seconds of network waits");+ }++ #[tokio::test]+ async fn feedback_upload_deadline_stops_retries_and_later_attachments() {+ let server = MockServer::start().await;+ let attempt = AtomicUsize::default();+ Mock::given(method("POST"))+ .and(path("/api/42/envelope/"))+ .respond_with(move |_: &wiremock::Request| {+ match attempt.fetch_add(/*val*/ 1, Ordering::SeqCst) {+ 0 | 2 => ResponseTemplate::new(StatusCode::OK)+ .set_delay(Duration::from_secs(/*secs*/ 1)),+ 1 => ResponseTemplate::new(StatusCode::SERVICE_UNAVAILABLE)+ .set_delay(Duration::from_secs(/*secs*/ 1)),+ _ => ResponseTemplate::new(StatusCode::OK)+ .set_delay(Duration::from_secs(/*secs*/ 20)),+ }+ })+ .expect(/*r*/ 4)+ .mount(&server)+ .await;++ let dsn = format!("http://public@{}/42", server.address());+ let attachments = ["stalled.txt", "later.txt"].map(|filename| FeedbackAttachment {+ filename: filename.to_string(),+ content_type: None,+ buffer: filename.as_bytes().to_vec(),+ });+ let snapshot = CodexFeedback::new()+ .snapshot(/*session_id*/ None)+ .with_feedback_diagnostics(FeedbackDiagnostics::default());+ let error = tokio::time::timeout(+ Duration::from_secs(/*secs*/ 10),+ snapshot.upload_feedback_with_dsn(+ FeedbackUploadOptions {+ classification: "bug",+ reason: None,+ tags: None,+ include_logs: true,+ extra_attachments: &attachments,+ extra_attachment_paths: &[],+ session_source: Some(SessionSource::Cli),+ logs_override: Some(b"log contents".to_vec()),+ },+ &HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),+ &dsn,+ Instant::now() + Duration::from_secs(/*secs*/ 8),+ ),+ )+ .await+ .expect("each request must use only the time left in the report deadline")+ .expect_err("the accepted report must report incomplete attachments");+ assert_eq!(+ error.to_string(),+ "feedback report was accepted, but some attachments failed to upload"+ );+ let requests = server.received_requests().await.unwrap();+ assert_eq!(requests.len(), 4);+ assert_eq!(requests[1].body, requests[2].body);+ }+#[tokio::test]async fn feedback_upload_retries_diagnostics_without_replaying_core() {let server = MockServer::start().await;@@ -1114,6 +1214,7 @@ mod tests {},&HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),&format!("http://public@{}/42", server.address()),+ Instant::now() + UPLOAD_TIMEOUT,).await.unwrap();@@ -1198,6 +1299,16 @@ mod tests {0,ResponseTemplate::new(StatusCode::OK).insert_header("Retry-After", "60"),),+ (+ 1,+ ResponseTemplate::new(StatusCode::SERVICE_UNAVAILABLE)+ .insert_header("Retry-After", "300"),+ ),+ (+ 1,+ ResponseTemplate::new(StatusCode::SERVICE_UNAVAILABLE)+ .insert_header("Retry-After", "invalid"),+ ),] {let server = MockServer::start().await;let attempt = AtomicUsize::default();diff --git a/codex-rs/feedback/src/upload.rs b/codex-rs/feedback/src/upload.rsindex c617e84a5b..dd62543dca 100644--- a/codex-rs/feedback/src/upload.rs+++ b/codex-rs/feedback/src/upload.rs@@ -46,24 +46,21 @@ pub(super) fn gzip_envelope_request(.timeout(timeout)}-/// Send an already-serialized envelope, charging requests and backoff to the shared budget.+/// Send an already-serialized envelope within the report's shared deadline.pub(super) async fn send_gzip_envelope(client_pool: &RouteAwareClientPool,dsn: &Dsn,body: Vec<u8>,kind: EnvelopeKind,- remaining_upload_time: &mut Duration,+ deadline: Instant,+ rate_limited: &mut bool,) -> Result<StatusCode> {- let request_started_at = Instant::now();let body = Bytes::from(body);// Retain the exact gzip bytes for at most two diagnostic retries. Never replay the core.let mut retry_delays = [250, 500].map(Duration::from_millis).into_iter();let response = loop {- let timeout = remaining_upload_time.saturating_sub(request_started_at.elapsed());- if timeout.is_zero() {- *remaining_upload_time = Duration::ZERO;- anyhow::bail!("feedback upload network wait budget exhausted");- }+ let timeout = deadline.saturating_duration_since(Instant::now());+ anyhow::ensure!(!timeout.is_zero(), "feedback upload deadline exceeded");let response = gzip_envelope_request(client_pool, dsn, body.clone(), timeout).send().await;@@ -77,7 +74,7 @@ pub(super) async fn send_gzip_envelope(}) {// Even accepted requests can impose a quota on events or attachments.// https://develop.sentry.dev/sdk/foundations/transport/rate-limiting/- *remaining_upload_time = Duration::ZERO;+ *rate_limited = true;break response;}let retryable = match &response {@@ -96,9 +93,8 @@ pub(super) async fn send_gzip_envelope(response.status() == StatusCode::TOO_MANY_REQUESTS&& (retry_delay.is_none() || !response.headers().contains_key("Retry-After"))}) {- // Bare 429s default to a cooldown longer than this upload budget.// Never let later files bypass an exhausted rate limit.- *remaining_upload_time = Duration::ZERO;+ *rate_limited = true;break response;}let Some(retry_delay) = retry_delay else {@@ -111,20 +107,16 @@ pub(super) async fn send_gzip_envelope(.map(|value| {// An invalid cooldown is not permission to send immediately.parse_retry_after(value.to_str().unwrap_or_default())- .unwrap_or(*remaining_upload_time)+ .unwrap_or_else(|| deadline.saturating_duration_since(Instant::now()))});let delay = retry_delay.max(retry_after.unwrap_or_default());- if delay >= remaining_upload_time.saturating_sub(request_started_at.elapsed()) {- // The cooldown applies to later files too, not just this retry.- *remaining_upload_time = Duration::ZERO;+ if delay >= deadline.saturating_duration_since(Instant::now()) {+ // Leave long cooldowns for a later submission, including its remaining files.+ *rate_limited = true;break response;}tokio::time::sleep(delay).await;- if request_started_at.elapsed() >= *remaining_upload_time {- break response;- }};- *remaining_upload_time = remaining_upload_time.saturating_sub(request_started_at.elapsed());response.map(|response| response.status()).context("failed to upload feedback to Sentry")