Lightweight coding agent that runs in your terminal
Rust
Improve diagnostics for compressed rollout read failures (#55545)
jif4be9cfdf30bc
diff --git a/codex-rs/feedback/src/lib.rs b/codex-rs/feedback/src/lib.rsindex 1d1fad2567..9920fbf022 100644--- a/codex-rs/feedback/src/lib.rs+++ b/codex-rs/feedback/src/lib.rs@@ -415,20 +415,45 @@ mod rollout_attachment_tests;impl FeedbackAttachmentPath {/// Read a whole regular file within the caller's size limit.pub fn read_attachment(&self, max_bytes: usize) -> io::Result<Option<FeedbackAttachment>> {- let rollout_path = codex_rollout::rollout_id_from_path(&self.path)+ let mut attachment_path = self.path.clone();+ let mut rollout_path = codex_rollout::rollout_id_from_path(&self.path).map(|_| codex_rollout::plain_rollout_path(&self.path));- let buffer = if rollout_path.is_some() {- let Some(buffer) =- codex_rollout::read_rollout_prefix(&self.path, max_bytes.saturating_add(1))?- else {- tracing::error!(- "feedback attachment skipped: rollout is missing or not a regular file"- );- return Ok(None);- };+ let mut raw_rollout = false;+ let decoded = if let Some(plain_path) = &rollout_path {+ match codex_rollout::read_rollout_prefix(&self.path, max_bytes.saturating_add(1)) {+ Ok(Some(buffer)) => Some(buffer),+ Ok(None) => {+ tracing::error!(+ "feedback attachment skipped: rollout is missing or not a regular file"+ );+ return Ok(None);+ }+ Err(error)+ if error.raw_os_error().is_none()+ && matches!(+ error.kind(),+ io::ErrorKind::UnexpectedEof+ | io::ErrorKind::InvalidData+ | io::ErrorKind::Other+ )+ && !plain_path.try_exists().unwrap_or(true) =>+ {+ // Preserve a selected, undecodable archive for diagnosis. Never replace+ // a preferred plain rollout with its potentially stale compressed sibling.+ attachment_path = plain_path.with_extension("jsonl.zst");+ rollout_path = None;+ raw_rollout = true;+ None+ }+ Err(error) => return Err(error),+ }+ } else {+ None+ };+ let buffer = if let Some(buffer) = decoded {buffer} else {- let metadata = fs::metadata(&self.path)?;+ let metadata = fs::metadata(&attachment_path)?;if !metadata.is_file() {tracing::error!("feedback attachment skipped: not a regular file");return Ok(None);@@ -443,7 +468,7 @@ impl FeedbackAttachmentPath {}let mut buffer = Vec::new();// Detect files that grow past the limit after the metadata check.- fs::File::open(&self.path)?+ fs::File::open(&attachment_path)?.take(max_bytes as u64 + 1).read_to_end(&mut buffer)?;buffer@@ -456,17 +481,20 @@ impl FeedbackAttachmentPath {);return Ok(None);}- let filename = self+ let mut filename = self.attachment_filename_override.clone().unwrap_or_else(|| {rollout_path.as_ref()- .unwrap_or(&self.path)+ .unwrap_or(&attachment_path).file_name().map(|name| name.to_string_lossy().to_string()).unwrap_or_else(|| "extra-log.log".to_string())});+ if raw_rollout && !filename.ends_with(".zst") {+ filename.push_str(".zst");+ }let content_type = match Path::new(&filename).extension().and_then(|extension| extension.to_str())diff --git a/codex-rs/feedback/src/rollout_attachment_tests.rs b/codex-rs/feedback/src/rollout_attachment_tests.rsindex 736c610af1..c4163b8ad0 100644--- a/codex-rs/feedback/src/rollout_attachment_tests.rs+++ b/codex-rs/feedback/src/rollout_attachment_tests.rs@@ -47,7 +47,7 @@ impl Drop for Fixture {}#[tokio::test]-async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read() -> Result<()> {+async fn feedback_upload_archives_readable_rollouts_and_undecodable_sources() -> Result<()> {use codex_http_client::OutboundProxyPolicy;use flate2::read::GzDecoder;use sentry::protocol::Envelope;@@ -59,6 +59,9 @@ async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read()use wiremock::matchers::path;let fixture = Fixture::new();+ let damaged = Fixture::new();+ let damaged_bytes = &COMPRESSED[..10];+ fs::write(&damaged.compressed, damaged_bytes)?;let second_filename = format!("rollout-2026-09-09T12-00-00-{}.jsonl", ThreadId::new());let second_bytes = b"{\"message\":\"second rollout\"}\n";let server = MockServer::start().await;@@ -83,7 +86,7 @@ async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read()content_type: Some("text/plain".to_string()),buffer: second_bytes.to_vec(),}],- extra_attachment_paths: &[fixture.attachment()],+ extra_attachment_paths: &[fixture.attachment(), damaged.attachment()],session_source: None,logs_override: None,},@@ -125,6 +128,10 @@ async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read()JSONL.to_vec()),(second_filename.into(), second_bytes.to_vec()),+ (+ damaged.compressed.file_name().unwrap().to_owned(),+ damaged_bytes.to_vec(),+ ),]));Ok(())@@ -234,6 +241,34 @@ fn compressed_attachment_bounds_decoded_bytes() {assert!(!fixture.plain.exists());}+#[test]+fn undecodable_attachment_preserves_raw_bytes_with_bounds_and_filename_override() {+ let fixture = Fixture::new();+ let damaged_bytes = &COMPRESSED[..10];+ fs::write(&fixture.compressed, damaged_bytes).unwrap();+ for path in [&fixture.plain, &fixture.compressed] {+ let attachment = FeedbackAttachmentPath {+ path: path.clone(),+ attachment_filename_override: Some("diagnostic.jsonl".to_string()),+ };+ assert!(+ attachment+ .read_attachment(damaged_bytes.len() - 1)+ .unwrap()+ .is_none()+ );+ let uploaded = attachment+ .read_attachment(damaged_bytes.len())+ .unwrap()+ .unwrap();+ assert_eq!(+ (uploaded.filename, uploaded.buffer),+ ("diagnostic.jsonl.zst".to_string(), damaged_bytes.to_vec())+ );+ }+ assert_eq!(fs::read(&fixture.compressed).unwrap(), damaged_bytes);+}+#[test]fn nonregular_rollout_is_omitted_and_unrelated_zstd_attachment_stays_opaque() {let fixture = Fixture::new();diff --git a/codex-rs/rollout/src/compression.rs b/codex-rs/rollout/src/compression.rsindex c0f33f14af..603fe3383c 100644--- a/codex-rs/rollout/src/compression.rs+++ b/codex-rs/rollout/src/compression.rs@@ -14,6 +14,7 @@ mod access_metadata;mod blocking_reader;mod error_metrics;mod path_metadata;+mod read_diagnostics;mod read_metrics;pub(crate) use blocking_reader::read_rollout_lines;@@ -1389,7 +1390,7 @@ mod reader {metrics.format = "zstd";let reader = tokio::task::spawn_blocking(move || {let input = File::open(path.as_path())?;- let decoder = zstd::stream::read::Decoder::new(input)?;+ let decoder = super::read_diagnostics::DiagnosticDecoder::new(input)?;Ok::<_, io::Error>(io::BufReader::new(Box::new(decoder) as Box<dyn Read + Send>).lines(),)diff --git a/codex-rs/rollout/src/compression/read_diagnostics.rs b/codex-rs/rollout/src/compression/read_diagnostics.rsnew file mode 100644index 0000000000..7b8756e639--- /dev/null+++ b/codex-rs/rollout/src/compression/read_diagnostics.rs@@ -0,0 +1,125 @@+//! Captures byte counts and file changes on the first compressed-stream failure.+//! Healthy reads only retain the first frame header and count decoded bytes. Logs contain+//! no paths or contents and are limited to one failure per process per minute.++use std::fs::File;+use std::fs::Metadata;+use std::io;+use std::io::BufReader;+use std::io::Read;+use std::sync::Mutex;+use std::sync::PoisonError;+use std::time::Duration;+use std::time::Instant;++use super::error_metrics::error_kind;++pub(super) struct DiagnosticDecoder {+ decoder: zstd::stream::read::Decoder<'static, BufReader<Source>>,+ decoded_bytes: u64,+ failure: Option<FailureDetails>,+}++#[derive(Debug, PartialEq, Eq)]+struct FailureDetails {+ compressed_bytes: Option<u64>,+ first_frame_content_bytes: Option<u64>,+ decoded_bytes: u64,+ file_changed: Option<bool>,+}++struct Source {+ file: File,+ opened_metadata: Option<Metadata>,+ header: [u8; 18],+ header_len: usize,+}++impl DiagnosticDecoder {+ pub(super) fn new(file: File) -> io::Result<Self> {+ let source = Source {+ opened_metadata: file.metadata().ok(),+ file,+ header: [0; 18],+ header_len: 0,+ };+ Ok(Self {+ decoder: zstd::stream::read::Decoder::new(source)?,+ decoded_bytes: 0,+ failure: None,+ })+ }+}++impl Read for DiagnosticDecoder {+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {+ let result = self.decoder.read(buffer);+ match &result {+ Ok(bytes) => self.decoded_bytes = self.decoded_bytes.saturating_add(*bytes as u64),+ Err(error) if self.failure.is_none() => {+ let source = self.decoder.get_ref().get_ref();+ let metadata = source.file.metadata().ok();+ let details = FailureDetails {+ compressed_bytes: metadata.as_ref().map(Metadata::len),+ // This is the first frame's size, not the total for concatenated frames.+ first_frame_content_bytes: zstd::zstd_safe::get_frame_content_size(+ &source.header[..source.header_len],+ )+ .ok()+ .flatten(),+ decoded_bytes: self.decoded_bytes,+ file_changed: source+ .opened_metadata+ .as_ref()+ .zip(metadata.as_ref())+ .and_then(|(before, after)| {+ Some(+ before.len() != after.len()+ || before.modified().ok()? != after.modified().ok()?,+ )+ }),+ };+ static LAST_REPORT: Mutex<Option<Instant>> = Mutex::new(None);+ let report = {+ let mut last = LAST_REPORT.lock().unwrap_or_else(PoisonError::into_inner);+ if last+ .is_none_or(|time| time.elapsed() >= Duration::from_secs(/*secs*/ 60))+ {+ *last = Some(Instant::now());+ true+ } else {+ false+ }+ };+ if report {+ tracing::warn!(+ error_kind = error_kind(error),+ compressed_bytes = ?details.compressed_bytes,+ first_frame_content_bytes = ?details.first_frame_content_bytes,+ decoded_bytes = details.decoded_bytes,+ file_changed = ?details.file_changed,+ "compressed rollout read failed"+ );+ }+ self.failure = Some(details);+ }+ Err(_) => {}+ }+ result+ }+}++impl Read for Source {+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {+ let bytes = self.file.read(buffer)?;+ let header_bytes = bytes.min(self.header.len() - self.header_len);+ self.header[self.header_len..self.header_len + header_bytes]+ .copy_from_slice(&buffer[..header_bytes]);+ self.header_len += header_bytes;+ Ok(bytes)+ }+}++#[cfg(test)]+#[path = "read_diagnostics_tests.rs"]+mod tests;diff --git a/codex-rs/rollout/src/compression/read_diagnostics_tests.rs b/codex-rs/rollout/src/compression/read_diagnostics_tests.rsnew file mode 100644index 0000000000..3feed76317--- /dev/null+++ b/codex-rs/rollout/src/compression/read_diagnostics_tests.rs@@ -0,0 +1,64 @@+//! Checks failure details against real truncated and concatenated zstd streams.++use std::fs;+use std::io::Read;+use std::io::Write;++use pretty_assertions::assert_eq;++use super::*;++#[test]+fn incomplete_frames_report_decoded_bytes_and_changes_to_the_open_file() -> io::Result<()> {+ let contents = b"valid utf8\n";+ let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), /*level*/ 0)?;+ encoder.set_pledged_src_size(Some(contents.len() as u64))?;+ encoder.write_all(contents)?;+ let complete = encoder.finish()?;++ for (suffix, change_after_open, expected_contents) in [+ (None, false, &contents[..contents.len() - 1]),+ (None, true, &contents[..contents.len() - 1]),+ (Some(0x28), false, contents.as_slice()),+ ] {+ let home = tempfile::tempdir()?;+ let path = home.path().join("rollout.jsonl.zst");+ let mut damaged = complete.clone();+ if let Some(byte) = suffix {+ damaged.push(byte); // A complete payload followed by an unfinished frame header.+ } else {+ damaged.pop();+ }+ fs::write(+ &path,+ if change_after_open {+ &complete+ } else {+ &damaged+ },+ )?;+ let mut reader = DiagnosticDecoder::new(File::open(&path)?)?;+ if change_after_open {+ fs::OpenOptions::new()+ .write(true)+ .open(&path)?+ .set_len(damaged.len() as u64)?;+ }+ let mut decoded = Vec::new();+ let error = reader.read_to_end(&mut decoded).unwrap_err();+ assert_eq!(+ (error.kind(), decoded, reader.failure),+ (+ io::ErrorKind::UnexpectedEof,+ expected_contents.to_vec(),+ Some(FailureDetails {+ compressed_bytes: Some(damaged.len() as u64),+ first_frame_content_bytes: Some(contents.len() as u64),+ decoded_bytes: expected_contents.len() as u64,+ file_changed: Some(change_after_open),+ }),+ )+ );+ }+ Ok(())+}