Lightweight coding agent that runs in your terminal
Rust
Classify rollout read failures by reason and progress (#47565)
jifd6c4c6aea404
diff --git a/codex-rs/rollout/src/compression.rs b/codex-rs/rollout/src/compression.rsindex 6e5c09c6ca..ba7eb24c5b 100644--- a/codex-rs/rollout/src/compression.rs+++ b/codex-rs/rollout/src/compression.rs@@ -20,6 +20,7 @@ mod error_metrics;mod read_metrics;use error_metrics::FailureMetric;+use read_metrics::ReadFailureSource;use read_metrics::ReadMetrics;const COMPRESSED_SUFFIX: &str = ".zst";@@ -85,7 +86,7 @@ pub async fn open_rollout_line_reader(path: &Path) -> io::Result<RolloutLineReadmatch result {Ok(inner) => Ok(RolloutLineReader { inner, metrics }),Err(err) => {- metrics.failed("open", &err);+ metrics.failed("open", ReadFailureSource::Stream, &err);Err(err)}}@@ -265,17 +266,22 @@ impl RolloutLineReader {pub async fn next_line(&mut self) -> io::Result<Option<String>> {let started_at = Instant::now();self.metrics.reached_eof = false;+ let mut failure_source = ReadFailureSource::Stream;let result = async {match &mut self.inner {RolloutLineReaderInner::Plain(lines) => lines.next_line().await,RolloutLineReaderInner::Blocking(slot) => {let Some(mut reader) = slot.take() else {+ failure_source = ReadFailureSource::ReaderBusy;return Err(io::Error::other("compressed rollout reader is busy"));};let (line, reader) =tokio::task::spawn_blocking(move || (reader.next().transpose(), reader)).await- .map_err(io::Error::other)?;+ .map_err(|err| {+ failure_source = ReadFailureSource::TaskJoin;+ io::Error::other(err)+ })?;*slot = Some(reader);line}@@ -284,8 +290,11 @@ impl RolloutLineReader {.await;self.metrics.duration = self.metrics.duration.saturating_add(started_at.elapsed());match &result {- Ok(line) => self.metrics.reached_eof = line.is_none(),- Err(err) => self.metrics.failed("read", err),+ Ok(line) => {+ self.metrics.reached_eof = line.is_none();+ self.metrics.read_any_line |= line.is_some();+ }+ Err(err) => self.metrics.failed("read", failure_source, err),}result}@@ -1331,6 +1340,7 @@ mod reader {use std::io::Read;use std::path::Path;+ use super::ReadFailureSource;use super::ReadMetrics;use super::RolloutLineReaderInner;use super::path;@@ -1353,7 +1363,8 @@ mod reader {)}).await- .map_err(io::Error::other)??;+ .map_err(io::Error::other)+ .inspect_err(|err| metrics.failed("open", ReadFailureSource::TaskJoin, err))??;return Ok(RolloutLineReaderInner::Blocking(Some(reader)));}metrics.format = "plain";diff --git a/codex-rs/rollout/src/compression/read_metrics.rs b/codex-rs/rollout/src/compression/read_metrics.rsindex 3840f0215a..ea984085c9 100644--- a/codex-rs/rollout/src/compression/read_metrics.rs+++ b/codex-rs/rollout/src/compression/read_metrics.rs@@ -2,6 +2,8 @@//! Duration sums completed open/retry and read calls, excluding caller processing//! and any canceled call. Partial readers must not be treated as successful EOFs.//! Aggregate locally rather than exporting a metric for every JSONL record.+//! Failure reasons and progress are bounded labels captured at the first failure;+//! error messages, file paths, and rollout contents are never exported.use std::io;use std::time::Duration;@@ -12,7 +14,21 @@ pub(super) struct ReadMetrics {pub(super) format: &'static str,pub(super) reached_eof: bool,pub(super) duration: Duration,- failure: Option<(&'static str, &'static str)>,+ pub(super) read_any_line: bool,+ failure: Option<ReadFailure>,+}++pub(super) enum ReadFailureSource {+ Stream,+ ReaderBusy,+ TaskJoin,+}++struct ReadFailure {+ stage: &'static str,+ error_kind: &'static str,+ reason: &'static str,+ read_progress: &'static str,}impl Default for ReadMetrics {@@ -21,14 +37,56 @@ impl Default for ReadMetrics {format: "unknown",reached_eof: false,duration: Duration::ZERO,+ read_any_line: false,failure: None,}}}impl ReadMetrics {- pub(super) fn failed(&mut self, stage: &'static str, error: &io::Error) {- self.failure.get_or_insert((stage, error_kind(error)));+ pub(super) fn failed(+ &mut self,+ stage: &'static str,+ source: ReadFailureSource,+ error: &io::Error,+ ) {+ self.failure.get_or_insert_with(|| ReadFailure {+ stage,+ error_kind: error_kind(error),+ reason: match source {+ ReadFailureSource::ReaderBusy => "reader_busy",+ ReadFailureSource::TaskJoin => "task_join",+ ReadFailureSource::Stream if error.raw_os_error().is_some() => "os_error",+ ReadFailureSource::Stream+ if self.format == "zstd" && error.kind() == io::ErrorKind::Other =>+ {+ // zstd converts its numeric error code to an io::Error containing+ // only a message. Match exact known messages, never export that text.+ // An unfamiliar message must remain a generic stream error.+ match error.to_string().as_str() {+ "Unknown frame descriptor" => "zstd_invalid_frame",+ "Data corruption detected"+ | "Header of Literals' block doesn't respect format specification" => {+ "zstd_corrupt_block"+ }+ "Restored data doesn't match checksum" => "zstd_checksum",+ "Frame requires too much memory for decoding"+ | "Allocation error : not enough memory" => "zstd_resource_limit",+ "Version not supported" | "Unsupported frame parameter" => {+ "zstd_unsupported_frame"+ }+ "Dictionary is corrupted" | "Dictionary mismatch" => "zstd_dictionary",+ _ => "stream_error",+ }+ }+ ReadFailureSource::Stream => "stream_error",+ },+ read_progress: if self.read_any_line {+ "after_first_line"+ } else {+ "before_first_line"+ },+ });}}@@ -37,16 +95,24 @@ impl Drop for ReadMetrics {let Some(metrics) = codex_otel::global() else {return;};- let (outcome, stage, error_kind) = match self.failure {- Some((stage, error_kind)) => ("failed", stage, error_kind),- None if self.reached_eof => ("eof", "none", "none"),- None => ("partial", "none", "none"),+ let (outcome, stage, error_kind, reason, read_progress) = match &self.failure {+ Some(failure) => (+ "failed",+ failure.stage,+ failure.error_kind,+ failure.reason,+ failure.read_progress,+ ),+ None if self.reached_eof => ("eof", "none", "none", "none", "none"),+ None => ("partial", "none", "none", "none", "none"),};let tags = [("format", self.format),("outcome", outcome),("stage", stage),("error_kind", error_kind),+ ("reason", reason),+ ("read_progress", read_progress),];let _ = metrics.counter("codex.rollout_compression.read", /*inc*/ 1, &tags);let _ = metrics.record_duration(