Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Improve diagnostics for compressed rollout read failures (#55545)

jif4be9cfdf30bc
diff --git a/codex-rs/feedback/src/lib.rs b/codex-rs/feedback/src/lib.rs
index 1d1fad2567..9920fbf022 100644
--- a/codex-rs/feedback/src/lib.rs
+++ b/codex-rs/feedback/src/lib.rs
@@ -415,20 +415,45 @@ mod rollout_attachment_tests;
impl FeedbackAttachmentPath {
/// Read a whole regular file within the caller's size limit.
pub fn read_attachment(&self, max_bytes: usize) -> io::Result<Option<FeedbackAttachment>> {
- let rollout_path = codex_rollout::rollout_id_from_path(&self.path)
+ let mut attachment_path = self.path.clone();
+ let mut rollout_path = codex_rollout::rollout_id_from_path(&self.path)
.map(|_| codex_rollout::plain_rollout_path(&self.path));
- let buffer = if rollout_path.is_some() {
- let Some(buffer) =
- codex_rollout::read_rollout_prefix(&self.path, max_bytes.saturating_add(1))?
- else {
- tracing::error!(
- "feedback attachment skipped: rollout is missing or not a regular file"
- );
- return Ok(None);
- };
+ let mut raw_rollout = false;
+ let decoded = if let Some(plain_path) = &rollout_path {
+ match codex_rollout::read_rollout_prefix(&self.path, max_bytes.saturating_add(1)) {
+ Ok(Some(buffer)) => Some(buffer),
+ Ok(None) => {
+ tracing::error!(
+ "feedback attachment skipped: rollout is missing or not a regular file"
+ );
+ return Ok(None);
+ }
+ Err(error)
+ if error.raw_os_error().is_none()
+ && matches!(
+ error.kind(),
+ io::ErrorKind::UnexpectedEof
+ | io::ErrorKind::InvalidData
+ | io::ErrorKind::Other
+ )
+ && !plain_path.try_exists().unwrap_or(true) =>
+ {
+ // Preserve a selected, undecodable archive for diagnosis. Never replace
+ // a preferred plain rollout with its potentially stale compressed sibling.
+ attachment_path = plain_path.with_extension("jsonl.zst");
+ rollout_path = None;
+ raw_rollout = true;
+ None
+ }
+ Err(error) => return Err(error),
+ }
+ } else {
+ None
+ };
+ let buffer = if let Some(buffer) = decoded {
buffer
} else {
- let metadata = fs::metadata(&self.path)?;
+ let metadata = fs::metadata(&attachment_path)?;
if !metadata.is_file() {
tracing::error!("feedback attachment skipped: not a regular file");
return Ok(None);
@@ -443,7 +468,7 @@ impl FeedbackAttachmentPath {
}
let mut buffer = Vec::new();
// Detect files that grow past the limit after the metadata check.
- fs::File::open(&self.path)?
+ fs::File::open(&attachment_path)?
.take(max_bytes as u64 + 1)
.read_to_end(&mut buffer)?;
buffer
@@ -456,17 +481,20 @@ impl FeedbackAttachmentPath {
);
return Ok(None);
}
- let filename = self
+ let mut filename = self
.attachment_filename_override
.clone()
.unwrap_or_else(|| {
rollout_path
.as_ref()
- .unwrap_or(&self.path)
+ .unwrap_or(&attachment_path)
.file_name()
.map(|name| name.to_string_lossy().to_string())
.unwrap_or_else(|| "extra-log.log".to_string())
});
+ if raw_rollout && !filename.ends_with(".zst") {
+ filename.push_str(".zst");
+ }
let content_type = match Path::new(&filename)
.extension()
.and_then(|extension| extension.to_str())
diff --git a/codex-rs/feedback/src/rollout_attachment_tests.rs b/codex-rs/feedback/src/rollout_attachment_tests.rs
index 736c610af1..c4163b8ad0 100644
--- a/codex-rs/feedback/src/rollout_attachment_tests.rs
+++ b/codex-rs/feedback/src/rollout_attachment_tests.rs
@@ -47,7 +47,7 @@ impl Drop for Fixture {
}
#[tokio::test]
-async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read() -> Result<()> {
+async fn feedback_upload_archives_readable_rollouts_and_undecodable_sources() -> Result<()> {
use codex_http_client::OutboundProxyPolicy;
use flate2::read::GzDecoder;
use sentry::protocol::Envelope;
@@ -59,6 +59,9 @@ async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read()
use wiremock::matchers::path;
let fixture = Fixture::new();
+ let damaged = Fixture::new();
+ let damaged_bytes = &COMPRESSED[..10];
+ fs::write(&damaged.compressed, damaged_bytes)?;
let second_filename = format!("rollout-2026-09-09T12-00-00-{}.jsonl", ThreadId::new());
let second_bytes = b"{\"message\":\"second rollout\"}\n";
let server = MockServer::start().await;
@@ -83,7 +86,7 @@ async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read()
content_type: Some("text/plain".to_string()),
buffer: second_bytes.to_vec(),
}],
- extra_attachment_paths: &[fixture.attachment()],
+ extra_attachment_paths: &[fixture.attachment(), damaged.attachment()],
session_source: None,
logs_override: None,
},
@@ -125,6 +128,10 @@ async fn feedback_upload_archives_two_rollouts_that_can_be_extracted_and_read()
JSONL.to_vec()
),
(second_filename.into(), second_bytes.to_vec()),
+ (
+ damaged.compressed.file_name().unwrap().to_owned(),
+ damaged_bytes.to_vec(),
+ ),
])
);
Ok(())
@@ -234,6 +241,34 @@ fn compressed_attachment_bounds_decoded_bytes() {
assert!(!fixture.plain.exists());
}
+#[test]
+fn undecodable_attachment_preserves_raw_bytes_with_bounds_and_filename_override() {
+ let fixture = Fixture::new();
+ let damaged_bytes = &COMPRESSED[..10];
+ fs::write(&fixture.compressed, damaged_bytes).unwrap();
+ for path in [&fixture.plain, &fixture.compressed] {
+ let attachment = FeedbackAttachmentPath {
+ path: path.clone(),
+ attachment_filename_override: Some("diagnostic.jsonl".to_string()),
+ };
+ assert!(
+ attachment
+ .read_attachment(damaged_bytes.len() - 1)
+ .unwrap()
+ .is_none()
+ );
+ let uploaded = attachment
+ .read_attachment(damaged_bytes.len())
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ (uploaded.filename, uploaded.buffer),
+ ("diagnostic.jsonl.zst".to_string(), damaged_bytes.to_vec())
+ );
+ }
+ assert_eq!(fs::read(&fixture.compressed).unwrap(), damaged_bytes);
+}
+
#[test]
fn nonregular_rollout_is_omitted_and_unrelated_zstd_attachment_stays_opaque() {
let fixture = Fixture::new();
diff --git a/codex-rs/rollout/src/compression.rs b/codex-rs/rollout/src/compression.rs
index c0f33f14af..603fe3383c 100644
--- a/codex-rs/rollout/src/compression.rs
+++ b/codex-rs/rollout/src/compression.rs
@@ -14,6 +14,7 @@ mod access_metadata;
mod blocking_reader;
mod error_metrics;
mod path_metadata;
+mod read_diagnostics;
mod read_metrics;
pub(crate) use blocking_reader::read_rollout_lines;
@@ -1389,7 +1390,7 @@ mod reader {
metrics.format = "zstd";
let reader = tokio::task::spawn_blocking(move || {
let input = File::open(path.as_path())?;
- let decoder = zstd::stream::read::Decoder::new(input)?;
+ let decoder = super::read_diagnostics::DiagnosticDecoder::new(input)?;
Ok::<_, io::Error>(
io::BufReader::new(Box::new(decoder) as Box<dyn Read + Send>).lines(),
)
diff --git a/codex-rs/rollout/src/compression/read_diagnostics.rs b/codex-rs/rollout/src/compression/read_diagnostics.rs
new file mode 100644
index 0000000000..7b8756e639
--- /dev/null
+++ b/codex-rs/rollout/src/compression/read_diagnostics.rs
@@ -0,0 +1,125 @@
+//! Captures byte counts and file changes on the first compressed-stream failure.
+//! Healthy reads only retain the first frame header and count decoded bytes. Logs contain
+//! no paths or contents and are limited to one failure per process per minute.
+
+use std::fs::File;
+use std::fs::Metadata;
+use std::io;
+use std::io::BufReader;
+use std::io::Read;
+use std::sync::Mutex;
+use std::sync::PoisonError;
+use std::time::Duration;
+use std::time::Instant;
+
+use super::error_metrics::error_kind;
+
+pub(super) struct DiagnosticDecoder {
+ decoder: zstd::stream::read::Decoder<'static, BufReader<Source>>,
+ decoded_bytes: u64,
+ failure: Option<FailureDetails>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct FailureDetails {
+ compressed_bytes: Option<u64>,
+ first_frame_content_bytes: Option<u64>,
+ decoded_bytes: u64,
+ file_changed: Option<bool>,
+}
+
+struct Source {
+ file: File,
+ opened_metadata: Option<Metadata>,
+ header: [u8; 18],
+ header_len: usize,
+}
+
+impl DiagnosticDecoder {
+ pub(super) fn new(file: File) -> io::Result<Self> {
+ let source = Source {
+ opened_metadata: file.metadata().ok(),
+ file,
+ header: [0; 18],
+ header_len: 0,
+ };
+ Ok(Self {
+ decoder: zstd::stream::read::Decoder::new(source)?,
+ decoded_bytes: 0,
+ failure: None,
+ })
+ }
+}
+
+impl Read for DiagnosticDecoder {
+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+ let result = self.decoder.read(buffer);
+ match &result {
+ Ok(bytes) => self.decoded_bytes = self.decoded_bytes.saturating_add(*bytes as u64),
+ Err(error) if self.failure.is_none() => {
+ let source = self.decoder.get_ref().get_ref();
+ let metadata = source.file.metadata().ok();
+ let details = FailureDetails {
+ compressed_bytes: metadata.as_ref().map(Metadata::len),
+ // This is the first frame's size, not the total for concatenated frames.
+ first_frame_content_bytes: zstd::zstd_safe::get_frame_content_size(
+ &source.header[..source.header_len],
+ )
+ .ok()
+ .flatten(),
+ decoded_bytes: self.decoded_bytes,
+ file_changed: source
+ .opened_metadata
+ .as_ref()
+ .zip(metadata.as_ref())
+ .and_then(|(before, after)| {
+ Some(
+ before.len() != after.len()
+ || before.modified().ok()? != after.modified().ok()?,
+ )
+ }),
+ };
+ static LAST_REPORT: Mutex<Option<Instant>> = Mutex::new(None);
+ let report = {
+ let mut last = LAST_REPORT.lock().unwrap_or_else(PoisonError::into_inner);
+ if last
+ .is_none_or(|time| time.elapsed() >= Duration::from_secs(/*secs*/ 60))
+ {
+ *last = Some(Instant::now());
+ true
+ } else {
+ false
+ }
+ };
+ if report {
+ tracing::warn!(
+ error_kind = error_kind(error),
+ compressed_bytes = ?details.compressed_bytes,
+ first_frame_content_bytes = ?details.first_frame_content_bytes,
+ decoded_bytes = details.decoded_bytes,
+ file_changed = ?details.file_changed,
+ "compressed rollout read failed"
+ );
+ }
+ self.failure = Some(details);
+ }
+ Err(_) => {}
+ }
+ result
+ }
+}
+
+impl Read for Source {
+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+ let bytes = self.file.read(buffer)?;
+ let header_bytes = bytes.min(self.header.len() - self.header_len);
+ self.header[self.header_len..self.header_len + header_bytes]
+ .copy_from_slice(&buffer[..header_bytes]);
+ self.header_len += header_bytes;
+ Ok(bytes)
+ }
+}
+
+#[cfg(test)]
+#[path = "read_diagnostics_tests.rs"]
+mod tests;
diff --git a/codex-rs/rollout/src/compression/read_diagnostics_tests.rs b/codex-rs/rollout/src/compression/read_diagnostics_tests.rs
new file mode 100644
index 0000000000..3feed76317
--- /dev/null
+++ b/codex-rs/rollout/src/compression/read_diagnostics_tests.rs
@@ -0,0 +1,64 @@
+//! Checks failure details against real truncated and concatenated zstd streams.
+
+use std::fs;
+use std::io::Read;
+use std::io::Write;
+
+use pretty_assertions::assert_eq;
+
+use super::*;
+
+#[test]
+fn incomplete_frames_report_decoded_bytes_and_changes_to_the_open_file() -> io::Result<()> {
+ let contents = b"valid utf8\n";
+ let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), /*level*/ 0)?;
+ encoder.set_pledged_src_size(Some(contents.len() as u64))?;
+ encoder.write_all(contents)?;
+ let complete = encoder.finish()?;
+
+ for (suffix, change_after_open, expected_contents) in [
+ (None, false, &contents[..contents.len() - 1]),
+ (None, true, &contents[..contents.len() - 1]),
+ (Some(0x28), false, contents.as_slice()),
+ ] {
+ let home = tempfile::tempdir()?;
+ let path = home.path().join("rollout.jsonl.zst");
+ let mut damaged = complete.clone();
+ if let Some(byte) = suffix {
+ damaged.push(byte); // A complete payload followed by an unfinished frame header.
+ } else {
+ damaged.pop();
+ }
+ fs::write(
+ &path,
+ if change_after_open {
+ &complete
+ } else {
+ &damaged
+ },
+ )?;
+ let mut reader = DiagnosticDecoder::new(File::open(&path)?)?;
+ if change_after_open {
+ fs::OpenOptions::new()
+ .write(true)
+ .open(&path)?
+ .set_len(damaged.len() as u64)?;
+ }
+ let mut decoded = Vec::new();
+ let error = reader.read_to_end(&mut decoded).unwrap_err();
+ assert_eq!(
+ (error.kind(), decoded, reader.failure),
+ (
+ io::ErrorKind::UnexpectedEof,
+ expected_contents.to_vec(),
+ Some(FailureDetails {
+ compressed_bytes: Some(damaged.len() as u64),
+ first_frame_content_bytes: Some(contents.len() as u64),
+ decoded_bytes: expected_contents.len() as u64,
+ file_changed: Some(change_after_open),
+ }),
+ )
+ );
+ }
+ Ok(())
+}