Lightweight coding agent that runs in your terminal
Rust
Bundle rollout attachments into a gzip tar archive (#50446)
Dmitriy Kovalenko9b0a676d5dca
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lockindex cb7a7ba70c..24e4f109df 100644--- a/codex-rs/Cargo.lock+++ b/codex-rs/Cargo.lock@@ -3439,6 +3439,8 @@ dependencies = ["pretty_assertions","sentry","serde_json",+ "tar",+ "tempfile","tokio","tracing","tracing-subscriber",diff --git a/codex-rs/app-server/src/request_processors/feedback_processor.rs b/codex-rs/app-server/src/request_processors/feedback_processor.rsindex 3d4763770d..a5192d03ea 100644--- a/codex-rs/app-server/src/request_processors/feedback_processor.rs+++ b/codex-rs/app-server/src/request_processors/feedback_processor.rs@@ -284,7 +284,7 @@ impl FeedbackRequestProcessor {reason: reason.as_deref(),tags,include_logs,- extra_attachments: &extra_attachments,+ extra_attachments,extra_attachment_paths: &attachment_paths,session_source: Some(session_source),logs_override: sqlite_feedback_logs,diff --git a/codex-rs/app-server/src/request_processors/feedback_thread_index.rs b/codex-rs/app-server/src/request_processors/feedback_thread_index.rsindex 5446c38b94..624477af70 100644--- a/codex-rs/app-server/src/request_processors/feedback_thread_index.rs+++ b/codex-rs/app-server/src/request_processors/feedback_thread_index.rs@@ -1,5 +1,6 @@//! Selects a bounded feedback subtree, preserving the reported thread and prioritizing-//! children with retained failed reviews. The index describes selection, not delivery.+//! children with retained failed reviews. Filenames name archive entries or individual+//! attachments when archiving fails; the index describes selection, not delivery.use codex_feedback::FeedbackAttachment;use codex_feedback::GuardianReviewFailures;@@ -13,6 +14,7 @@ const MAX_LISTED_OMISSIONS: usize = 64;#[derive(Serialize)]pub(super) struct FeedbackThreadIndex {+ rollout_archive_filename: &'static str,pub threads: Vec<FeedbackThread>,retained_failure_thread_ids: Vec<ThreadId>,omitted_thread_ids: Vec<ThreadId>,@@ -56,6 +58,7 @@ impl FeedbackThreadIndex {.take(MAX_LISTED_OMISSIONS).collect::<Vec<_>>();Self {+ rollout_archive_filename: "rollouts.tar.gz",threads: thread_ids.into_iter().map(|thread_id| FeedbackThread {diff --git a/codex-rs/app-server/src/request_processors/feedback_thread_index_tests.rs b/codex-rs/app-server/src/request_processors/feedback_thread_index_tests.rsindex af5df1926a..075bc69ba8 100644--- a/codex-rs/app-server/src/request_processors/feedback_thread_index_tests.rs+++ b/codex-rs/app-server/src/request_processors/feedback_thread_index_tests.rs@@ -28,6 +28,7 @@ fn old_failed_child_precedes_new_children_and_index_explains_selection() -> anyhassert_eq!(serde_json::from_slice::<serde_json::Value>(&attachment.buffer)?,json!({+ "rollout_archive_filename": "rollouts.tar.gz","threads": selected.into_iter().map(|i| json!({"thread_id": ids[i],"rollout_filename": match i {diff --git a/codex-rs/feedback/Cargo.toml b/codex-rs/feedback/Cargo.tomlindex 2fab1dd906..ee56043b1c 100644--- a/codex-rs/feedback/Cargo.toml+++ b/codex-rs/feedback/Cargo.toml@@ -21,6 +21,8 @@ httpdate = { workspace = true }mime_guess = { workspace = true }sentry = { version = "0.46", default-features = false }serde_json = { workspace = true }+tar = { workspace = true }+tempfile = { workspace = true }tokio = { workspace = true, features = ["time"] }tracing = { workspace = true }tracing-subscriber = { workspace = true }diff --git a/codex-rs/feedback/src/feedback_event_tests.rs b/codex-rs/feedback/src/feedback_event_tests.rsindex b8bcfa4314..49bf41cd1e 100644--- a/codex-rs/feedback/src/feedback_event_tests.rs+++ b/codex-rs/feedback/src/feedback_event_tests.rs@@ -200,7 +200,7 @@ async fn feedback_upload_sends_safe_reason_tag_and_full_comment() {reason: Some(&reason),tags: Some(&tags),include_logs: false,- extra_attachments: &[],+ extra_attachments: Vec::new(),extra_attachment_paths: &[],session_source: None,logs_override: None,diff --git a/codex-rs/feedback/src/lib.rs b/codex-rs/feedback/src/lib.rsindex e383cafa30..1ed8674bef 100644--- a/codex-rs/feedback/src/lib.rs+++ b/codex-rs/feedback/src/lib.rs@@ -36,6 +36,7 @@ pub use daemon_logs::daemon_log_attachments;pub(crate) mod feedback_diagnostics;mod guardian;mod report_upload;+mod rollout_archive;mod upload;pub use feedback_diagnostics::FEEDBACK_DIAGNOSTICS_ATTACHMENT_FILENAME;pub use feedback_diagnostics::FeedbackDiagnostic;@@ -508,12 +509,11 @@ pub struct FeedbackUploadOptions<'a> {pub reason: Option<&'a str>,pub tags: Option<&'a BTreeMap<String, String>>,pub include_logs: bool,- /// Generated attachments that are already buffered and safe to upload.+ /// Owned diagnostics and frozen rollout prefixes that are safe to upload.///- /// These are included after `codex-logs.log` and before path-backed rollout- /// attachments. They are only passed by the caller after any user consent- /// gate has decided logs and diagnostics should be uploaded.- pub extra_attachments: &'a [FeedbackAttachment],+ /// Diagnostics follow `codex-logs.log`; rollout prefixes join the archive with+ /// path-backed rollouts. Callers must apply any user consent gate first.+ pub extra_attachments: Vec<FeedbackAttachment>,pub extra_attachment_paths: &'a [FeedbackAttachmentPath],pub session_source: Option<SessionSource>,pub logs_override: Option<Vec<u8>>,@@ -676,6 +676,7 @@ impl FeedbackSnapshot {);let mut attachments = self.feedback_attachments(+ &headers,options.include_logs,options.extra_attachments,options.extra_attachment_paths,@@ -817,16 +818,74 @@ impl FeedbackSnapshot {fn feedback_attachments<'a>(&'a self,+ headers: &'a sentry::protocol::EnvelopeHeaders,include_logs: bool,- extra_attachments: &'a [FeedbackAttachment],+ extra_attachments: Vec<FeedbackAttachment>,extra_attachment_paths: &'a [FeedbackAttachmentPath],logs_override: Option<Vec<u8>>,) -> impl Iterator<Item = Result<sentry::protocol::Attachment>> + 'a {use sentry::protocol::Attachment;- // Priority: logs, generated attachments (doctor report), connectivity diagnostics,- // then files in caller order. Measure each file’s envelope independently;- // the size limit applies per request, not to the sum of all files.+ // Partition metadata without reading files before the event is accepted. Keep+ // generated diagnostics first, then rollouts, then remaining files in caller order.+ let (rollouts, diagnostics_attachments): (Vec<_>, Vec<_>) =+ extra_attachments.into_iter().partition(|attachment| {+ codex_rollout::rollout_id_from_path(Path::new(&attachment.filename)).is_some()+ });+ let (rollout_paths, diagnostic_paths): (Vec<_>, Vec<_>) =+ extra_attachment_paths.iter().partition(|attachment| {+ codex_rollout::rollout_id_from_path(&attachment.path).is_some()+ });+ let has_rollouts = !rollouts.is_empty() || !rollout_paths.is_empty();+ let archive = has_rollouts+ .then_some((rollout_paths, rollouts))+ .into_iter()+ .flat_map(move |(paths, rollouts)| {+ let archive =+ tempfile::tempdir()+ .map_err(anyhow::Error::from)+ .and_then(|directory| {+ let output_path = directory.path().join("rollouts.tar.gz");+ let Some(archive) = rollout_archive::archive_rollouts(+ &output_path,+ paths.iter().copied(),+ &rollouts,+ MAX_DECODED_UPLOAD_BYTES,+ )?+ else {+ return Ok(None);+ };+ let attachment = archive+ .read_attachment(MAX_DECODED_UPLOAD_BYTES)?+ .context("rollout archive is missing or exceeds the size limit")?;+ // Measure framing without copying the archive. Account for the extra+ // decimal digits when the empty attachment's length becomes nonzero.+ let mut envelope =+ sentry::protocol::Envelope::new().with_headers(headers.clone());+ envelope.add_item(Attachment {+ filename: attachment.filename.clone(),+ content_type: attachment.content_type.clone(),+ ..Default::default()+ });+ let mut framing = Vec::new();+ envelope.to_writer(&mut framing)?;+ let archive_bytes = attachment.buffer.len();+ let envelope_bytes = framing.len()+ + archive_bytes+ + archive_bytes.checked_ilog10().unwrap_or(0) as usize;+ anyhow::ensure!(+ envelope_bytes <= MAX_DECODED_UPLOAD_BYTES,+ "rollout archive envelope exceeds the size limit"+ );+ Ok(Some(attachment))+ });+ rollout_archive::with_individual_fallback(+ archive,+ paths,+ rollouts,+ MAX_DECODED_UPLOAD_BYTES,+ )+ });let logs = include_logs.then(|| self.log_attachment(logs_override));let diagnostics = self.feedback_diagnostics_attachment_text(include_logs)@@ -837,18 +896,11 @@ impl FeedbackSnapshot {});logs.into_iter()- .chain(- extra_attachments- .iter()- .map(|attachment| FeedbackAttachment {- buffer: attachment.buffer.clone(),- filename: attachment.filename.clone(),- content_type: attachment.content_type.clone(),- }),- )+ .chain(diagnostics_attachments).chain(diagnostics).map(Ok)- .chain(extra_attachment_paths.iter().map(|attachment_path| {+ .chain(archive)+ .chain(diagnostic_paths.into_iter().map(|attachment_path| {attachment_path.read_attachment(MAX_DECODED_UPLOAD_BYTES)?.context("feedback attachment is not a regular file or exceeds the size limit")@@ -1068,7 +1120,7 @@ mod tests {async fn upload_test_feedback(feedback: &CodexFeedback,dsn: &str,- extra_attachments: &[FeedbackAttachment],+ extra_attachments: Vec<FeedbackAttachment>,) -> Result<()> {feedback.snapshot(/*session_id*/ None)@@ -1110,7 +1162,7 @@ mod tests {content_type: None,buffer: b"later diagnostic".to_vec(),};- upload_test_feedback(&CodexFeedback::new(), &dsn, &[attachment])+ upload_test_feedback(&CodexFeedback::new(), &dsn, vec![attachment]).await.expect("all three envelopes should finish across twelve seconds of network waits");}@@ -1152,7 +1204,7 @@ mod tests {reason: None,tags: None,include_logs: true,- extra_attachments: &attachments,+ extra_attachments: attachments.into(),extra_attachment_paths: &[],session_source: Some(SessionSource::Cli),logs_override: Some(b"log contents".to_vec()),@@ -1211,7 +1263,7 @@ mod tests {buffer: filename.as_bytes().to_vec(),});let started = Instant::now();- upload_test_feedback(&CodexFeedback::new(), &dsn, &attachments)+ upload_test_feedback(&CodexFeedback::new(), &dsn, attachments.into()).await.expect_err("legacy uploads report incomplete diagnostics");assert!(started.elapsed() >= Duration::from_secs(2));@@ -1292,7 +1344,7 @@ mod tests {reason: Some("large diagnostic upload"),tags: None,include_logs: false,- extra_attachments: &[],+ extra_attachments: Vec::new(),extra_attachment_paths: &[FeedbackAttachmentPath {path: first_path.clone(),@@ -1392,7 +1444,7 @@ mod tests {reason: None,tags: None,include_logs: false,- extra_attachments: &[],+ extra_attachments: Vec::new(),extra_attachment_paths: &[FeedbackAttachmentPath {path,attachment_filename_override: None,@@ -1473,7 +1525,7 @@ mod tests {content_type: None,buffer: b"later diagnostic".to_vec(),};- upload_test_feedback(&CodexFeedback::new(), &dsn, &[later])+ upload_test_feedback(&CodexFeedback::new(), &dsn, vec![later]).await.expect_err("legacy uploads report rate-limited diagnostics");}@@ -1492,7 +1544,7 @@ mod tests {let feedback = CodexFeedback::new();let dsn = format!("http://public@{}/42", server.address());- let error = upload_test_feedback(&feedback, &dsn, &[])+ let error = upload_test_feedback(&feedback, &dsn, Vec::new()).await.expect_err("rejected Sentry responses must fail feedback uploads");@@ -1515,7 +1567,7 @@ mod tests {.mount(&sentry_server).await;let dsn = format!("http://public@{}/42", sentry_server.address());- let error = upload_test_feedback(&CodexFeedback::new(), &dsn, &[])+ let error = upload_test_feedback(&CodexFeedback::new(), &dsn, Vec::new()).await.expect_err("redirected feedback uploads must be rejected");@@ -1539,7 +1591,7 @@ mod tests {drop(listener);let dsn = format!("http://public@{address}/42");- let error = upload_test_feedback(&CodexFeedback::new(), &dsn, &[])+ let error = upload_test_feedback(&CodexFeedback::new(), &dsn, Vec::new()).await.expect_err("transport failures must fail feedback uploads");@@ -1570,8 +1622,9 @@ mod tests {let attachments_with_diagnostics = snapshot_with_diagnostics.feedback_attachments(+ &sentry::protocol::EnvelopeHeaders::default(),/*include_logs*/ true,- &[FeedbackAttachment {+ vec![FeedbackAttachment {filename: DOCTOR_REPORT_ATTACHMENT_FILENAME.to_string(),content_type: Some("application/json".to_string()),buffer: b"{\"overallStatus\":\"ok\"}".to_vec(),@@ -1615,7 +1668,13 @@ mod tests {let attachments_without_diagnostics = CodexFeedback::new().snapshot(/*session_id*/ None).with_feedback_diagnostics(FeedbackDiagnostics::default())- .feedback_attachments(/*include_logs*/ true, &[], &[], Some(vec![1]))+ .feedback_attachments(+ &sentry::protocol::EnvelopeHeaders::default(),+ /*include_logs*/ true,+ Vec::new(),+ &[],+ Some(vec![1]),+ ).collect::<Result<Vec<_>>>().unwrap();@@ -1645,8 +1704,9 @@ mod tests {let attachments = CodexFeedback::new().snapshot(/*session_id*/ None).feedback_attachments(+ &sentry::protocol::EnvelopeHeaders::default(),/*include_logs*/ false,- &[],+ Vec::new(),&[FeedbackAttachmentPath {path: gzip_path.clone(),diff --git a/codex-rs/feedback/src/rollout_archive.rs b/codex-rs/feedback/src/rollout_archive.rsnew file mode 100644index 0000000000..c68bd9a997--- /dev/null+++ b/codex-rs/feedback/src/rollout_archive.rs@@ -0,0 +1,148 @@+//! Writes selected rollouts to a bounded gzip tar at a caller-provided local path.+//! Invalid inputs are logged and skipped; only a complete archive is published.+//! Archive failures fall back to lazy individual attachments using the original sources.++use std::collections::HashSet;+use std::fs;+use std::io;+use std::io::BufWriter;+use std::io::Write;+use std::path::Path;++use anyhow::Context;+use anyhow::Result;+use flate2::Compression;+use flate2::write::GzEncoder;++use crate::FeedbackAttachment;+use crate::FeedbackAttachmentPath;++pub(super) fn archive_rollouts<'a>(+ output_path: &Path,+ rollout_paths: impl IntoIterator<Item = &'a FeedbackAttachmentPath>,+ rollouts: impl IntoIterator<Item = &'a FeedbackAttachment>,+ max_bytes: usize,+) -> Result<Option<FeedbackAttachmentPath>> {+ let parent = output_path+ .parent()+ .filter(|path| !path.as_os_str().is_empty())+ .unwrap_or_else(|| Path::new("."));+ fs::create_dir_all(parent).context("failed to create rollout archive directory")?;+ let file = tempfile::NamedTempFile::new_in(parent)?;+ let writer = BoundedWriter {+ inner: BufWriter::new(file),+ written: 0,+ max_bytes,+ };+ let mut archive = tar::Builder::new(GzEncoder::new(writer, Compression::default()));+ let mut filenames = HashSet::new();+ for path in rollout_paths {+ match path.read_attachment(max_bytes) {+ Ok(Some(rollout)) => append_rollout(&mut archive, &mut filenames, &rollout, max_bytes)?,+ Ok(None) => {} // The attachment reader already logs oversized or nonregular files.+ Err(error) => tracing::error!(%error, "failed to read rollout; skipping attachment"),+ }+ }+ for rollout in rollouts {+ append_rollout(&mut archive, &mut filenames, rollout, max_bytes)?;+ }+ if filenames.is_empty() {+ return Ok(None);+ }+ let writer = archive+ .into_inner()?+ .finish()+ .context("failed to finish rollout archive")?;+ let file = writer.inner.into_inner()?;+ // Keep failed writes and incomplete gzip streams out of the destination path.+ file.persist_noclobber(output_path)?;+ Ok(Some(FeedbackAttachmentPath {+ path: output_path.to_path_buf(),+ attachment_filename_override: None,+ }))+}++pub(super) fn with_individual_fallback<'a>(+ archive: Result<Option<FeedbackAttachment>>,+ rollout_paths: Vec<&'a FeedbackAttachmentPath>,+ rollouts: Vec<FeedbackAttachment>,+ max_bytes: usize,+) -> impl Iterator<Item = Result<FeedbackAttachment>> + 'a {+ let (archive, fallback) = match archive {+ Ok(archive) => (archive, false),+ Err(error) => {+ tracing::error!(%error, "rollout archiving failed; uploading individual attachments");+ (None, true)+ }+ };+ archive+ .into_iter()+ .map(Ok)+ .chain(fallback.then_some(rollouts).into_iter().flatten().map(Ok))+ .chain(+ fallback+ .then_some(rollout_paths)+ .into_iter()+ .flatten()+ .map(move |path| {+ path.read_attachment(max_bytes)?.context(+ "feedback attachment is not a regular file or exceeds the size limit",+ )+ }),+ )+}++fn append_rollout<W: Write>(+ archive: &mut tar::Builder<W>,+ filenames: &mut HashSet<String>,+ rollout: &FeedbackAttachment,+ max_bytes: usize,+) -> Result<()> {+ if rollout.buffer.len() > max_bytes {+ tracing::error!(+ bytes = rollout.buffer.len(),+ max_bytes,+ "rollout skipped: size limit exceeded"+ );+ return Ok(());+ }+ if rollout.filename.is_empty()+ || matches!(rollout.filename.as_str(), "." | "..")+ || rollout.filename.contains(['/', '\\', ':'])+ || filenames.contains(&rollout.filename)+ {+ tracing::error!("rollout skipped: invalid or duplicate archive filename");+ return Ok(());+ }+ let mut header = tar::Header::new_gnu();+ header.set_size(rollout.buffer.len() as u64);+ header.set_mode(0o600);+ header.set_mtime(0);+ header.set_entry_type(tar::EntryType::Regular);+ archive+ .append_data(&mut header, &rollout.filename, rollout.buffer.as_slice())+ .context("failed to write rollout archive")?;+ filenames.insert(rollout.filename.clone());+ Ok(())+}++struct BoundedWriter<W> {+ inner: W,+ written: usize,+ max_bytes: usize,+}++impl<W: Write> Write for BoundedWriter<W> {+ fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {+ if buffer.len() > self.max_bytes.saturating_sub(self.written) {+ return Err(io::Error::other("rollout archive exceeds the size limit"));+ }+ let written = self.inner.write(buffer)?;+ self.written += written;+ Ok(written)+ }++ fn flush(&mut self) -> io::Result<()> {+ self.inner.flush()+ }+}diff --git a/codex-rs/feedback/src/rollout_attachment_tests.rs b/codex-rs/feedback/src/rollout_attachment_tests.rsindex 537ffbad1e..736c610af1 100644--- a/codex-rs/feedback/src/rollout_attachment_tests.rs+++ b/codex-rs/feedback/src/rollout_attachment_tests.rs@@ -46,40 +46,88 @@ impl Drop for Fixture {}}-#[test]-fn unloaded_compressed_rollout_is_included_as_jsonl_attachment() {+#[tokio::test]+async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read() -> Result<()> {+ use codex_http_client::OutboundProxyPolicy;+ use flate2::read::GzDecoder;+ use sentry::protocol::Envelope;+ use sentry::protocol::EnvelopeItem;+ use wiremock::Mock;+ use wiremock::MockServer;+ use wiremock::ResponseTemplate;+ use wiremock::matchers::method;+ use wiremock::matchers::path;+let fixture = Fixture::new();- // The app-server supplies the DB's logical .jsonl path without loading this thread.- let paths = [fixture.attachment()];- let snapshot = CodexFeedback::new().snapshot(/*session_id*/ None);- let attachments = snapshot- .feedback_attachments(- /*include_logs*/ false,- &[],- &paths,- /*logs_override*/ None,+ let second_filename = format!("rollout-2026-09-09T12-00-00-{}.jsonl", ThreadId::new());+ let second_bytes = b"{\"message\":\"second rollout\"}\n";+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/42/envelope/"))+ .respond_with(ResponseTemplate::new(200))+ .expect(/*r*/ 2)+ .mount(&server)+ .await;+ let dsn = format!("http://public@{}/42", server.address());+ CodexFeedback::new()+ .snapshot(/*session_id*/ None)+ .with_feedback_diagnostics(FeedbackDiagnostics::default())+ .upload_feedback_with_dsn(+ FeedbackUploadOptions {+ classification: "bug",+ reason: None,+ tags: None,+ include_logs: false,+ extra_attachments: vec![FeedbackAttachment {+ filename: second_filename.clone(),+ content_type: Some("text/plain".to_string()),+ buffer: second_bytes.to_vec(),+ }],+ extra_attachment_paths: &[fixture.attachment()],+ session_source: None,+ logs_override: None,+ },+ &HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),+ &dsn,+ Instant::now() + UPLOAD_TIMEOUT,)- .collect::<Result<Vec<_>>>()- .unwrap();+ .await?;++ let requests = server.received_requests().await.unwrap();+ assert_eq!(requests.len(), 2);+ let mut body = requests[1].body.clone();+ if body.starts_with(&[0x1f, 0x8b]) {+ let mut decoded = Vec::new();+ GzDecoder::new(body.as_slice()).read_to_end(&mut decoded)?;+ body = decoded;+ }+ let envelope = Envelope::from_slice(&body)?;+ let mut items = envelope.items();+ let Some(EnvelopeItem::Attachment(attachment)) = items.next() else {+ anyhow::bail!("expected a rollout archive attachment");+ };+ assert_eq!(attachment.filename, "rollouts.tar.gz");+ assert!(items.next().is_none());++ let extracted = tempfile::tempdir()?;+ tar::Archive::new(GzDecoder::new(attachment.buffer.as_slice())).unpack(extracted.path())?;+ let files = fs::read_dir(extracted.path())?+ .map(|entry| {+ let entry = entry?;+ Ok((entry.file_name(), fs::read(entry.path())?))+ })+ .collect::<io::Result<BTreeMap<_, _>>>()?;assert_eq!(- attachments- .iter()- .map(|attachment| (- attachment.filename.as_str(),- attachment.content_type.as_deref(),- attachment.buffer.as_slice(),- ))- .collect::<Vec<_>>(),- vec![(- fixture.plain.file_name().unwrap().to_str().unwrap(),- Some("text/plain"),- JSONL- )]- );- assert!(- !fixture.plain.exists(),- "feedback must not materialize the durable rollout"+ files,+ BTreeMap::from([+ (+ fixture.plain.file_name().unwrap().to_owned(),+ JSONL.to_vec()+ ),+ (second_filename.into(), second_bytes.to_vec()),+ ]));+ Ok(())}#[test]