Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Classify rollout read failures by reason and progress (#47565)

jifd6c4c6aea404
diff --git a/codex-rs/rollout/src/compression.rs b/codex-rs/rollout/src/compression.rs
index 6e5c09c6ca..ba7eb24c5b 100644
--- a/codex-rs/rollout/src/compression.rs
+++ b/codex-rs/rollout/src/compression.rs
@@ -20,6 +20,7 @@ mod error_metrics;
mod read_metrics;
use error_metrics::FailureMetric;
+use read_metrics::ReadFailureSource;
use read_metrics::ReadMetrics;
const COMPRESSED_SUFFIX: &str = ".zst";
@@ -85,7 +86,7 @@ pub async fn open_rollout_line_reader(path: &Path) -> io::Result<RolloutLineRead
match result {
Ok(inner) => Ok(RolloutLineReader { inner, metrics }),
Err(err) => {
- metrics.failed("open", &err);
+ metrics.failed("open", ReadFailureSource::Stream, &err);
Err(err)
}
}
@@ -265,17 +266,22 @@ impl RolloutLineReader {
pub async fn next_line(&mut self) -> io::Result<Option<String>> {
let started_at = Instant::now();
self.metrics.reached_eof = false;
+ let mut failure_source = ReadFailureSource::Stream;
let result = async {
match &mut self.inner {
RolloutLineReaderInner::Plain(lines) => lines.next_line().await,
RolloutLineReaderInner::Blocking(slot) => {
let Some(mut reader) = slot.take() else {
+ failure_source = ReadFailureSource::ReaderBusy;
return Err(io::Error::other("compressed rollout reader is busy"));
};
let (line, reader) =
tokio::task::spawn_blocking(move || (reader.next().transpose(), reader))
.await
- .map_err(io::Error::other)?;
+ .map_err(|err| {
+ failure_source = ReadFailureSource::TaskJoin;
+ io::Error::other(err)
+ })?;
*slot = Some(reader);
line
}
@@ -284,8 +290,11 @@ impl RolloutLineReader {
.await;
self.metrics.duration = self.metrics.duration.saturating_add(started_at.elapsed());
match &result {
- Ok(line) => self.metrics.reached_eof = line.is_none(),
- Err(err) => self.metrics.failed("read", err),
+ Ok(line) => {
+ self.metrics.reached_eof = line.is_none();
+ self.metrics.read_any_line |= line.is_some();
+ }
+ Err(err) => self.metrics.failed("read", failure_source, err),
}
result
}
@@ -1331,6 +1340,7 @@ mod reader {
use std::io::Read;
use std::path::Path;
+ use super::ReadFailureSource;
use super::ReadMetrics;
use super::RolloutLineReaderInner;
use super::path;
@@ -1353,7 +1363,8 @@ mod reader {
)
})
.await
- .map_err(io::Error::other)??;
+ .map_err(io::Error::other)
+ .inspect_err(|err| metrics.failed("open", ReadFailureSource::TaskJoin, err))??;
return Ok(RolloutLineReaderInner::Blocking(Some(reader)));
}
metrics.format = "plain";
diff --git a/codex-rs/rollout/src/compression/read_metrics.rs b/codex-rs/rollout/src/compression/read_metrics.rs
index 3840f0215a..ea984085c9 100644
--- a/codex-rs/rollout/src/compression/read_metrics.rs
+++ b/codex-rs/rollout/src/compression/read_metrics.rs
@@ -2,6 +2,8 @@
//! Duration sums completed open/retry and read calls, excluding caller processing
//! and any canceled call. Partial readers must not be treated as successful EOFs.
//! Aggregate locally rather than exporting a metric for every JSONL record.
+//! Failure reasons and progress are bounded labels captured at the first failure;
+//! error messages, file paths, and rollout contents are never exported.
use std::io;
use std::time::Duration;
@@ -12,7 +14,21 @@ pub(super) struct ReadMetrics {
pub(super) format: &'static str,
pub(super) reached_eof: bool,
pub(super) duration: Duration,
- failure: Option<(&'static str, &'static str)>,
+ pub(super) read_any_line: bool,
+ failure: Option<ReadFailure>,
+}
+
+pub(super) enum ReadFailureSource {
+ Stream,
+ ReaderBusy,
+ TaskJoin,
+}
+
+struct ReadFailure {
+ stage: &'static str,
+ error_kind: &'static str,
+ reason: &'static str,
+ read_progress: &'static str,
}
impl Default for ReadMetrics {
@@ -21,14 +37,56 @@ impl Default for ReadMetrics {
format: "unknown",
reached_eof: false,
duration: Duration::ZERO,
+ read_any_line: false,
failure: None,
}
}
}
impl ReadMetrics {
- pub(super) fn failed(&mut self, stage: &'static str, error: &io::Error) {
- self.failure.get_or_insert((stage, error_kind(error)));
+ pub(super) fn failed(
+ &mut self,
+ stage: &'static str,
+ source: ReadFailureSource,
+ error: &io::Error,
+ ) {
+ self.failure.get_or_insert_with(|| ReadFailure {
+ stage,
+ error_kind: error_kind(error),
+ reason: match source {
+ ReadFailureSource::ReaderBusy => "reader_busy",
+ ReadFailureSource::TaskJoin => "task_join",
+ ReadFailureSource::Stream if error.raw_os_error().is_some() => "os_error",
+ ReadFailureSource::Stream
+ if self.format == "zstd" && error.kind() == io::ErrorKind::Other =>
+ {
+ // zstd converts its numeric error code to an io::Error containing
+ // only a message. Match exact known messages, never export that text.
+ // An unfamiliar message must remain a generic stream error.
+ match error.to_string().as_str() {
+ "Unknown frame descriptor" => "zstd_invalid_frame",
+ "Data corruption detected"
+ | "Header of Literals' block doesn't respect format specification" => {
+ "zstd_corrupt_block"
+ }
+ "Restored data doesn't match checksum" => "zstd_checksum",
+ "Frame requires too much memory for decoding"
+ | "Allocation error : not enough memory" => "zstd_resource_limit",
+ "Version not supported" | "Unsupported frame parameter" => {
+ "zstd_unsupported_frame"
+ }
+ "Dictionary is corrupted" | "Dictionary mismatch" => "zstd_dictionary",
+ _ => "stream_error",
+ }
+ }
+ ReadFailureSource::Stream => "stream_error",
+ },
+ read_progress: if self.read_any_line {
+ "after_first_line"
+ } else {
+ "before_first_line"
+ },
+ });
}
}
@@ -37,16 +95,24 @@ impl Drop for ReadMetrics {
let Some(metrics) = codex_otel::global() else {
return;
};
- let (outcome, stage, error_kind) = match self.failure {
- Some((stage, error_kind)) => ("failed", stage, error_kind),
- None if self.reached_eof => ("eof", "none", "none"),
- None => ("partial", "none", "none"),
+ let (outcome, stage, error_kind, reason, read_progress) = match &self.failure {
+ Some(failure) => (
+ "failed",
+ failure.stage,
+ failure.error_kind,
+ failure.reason,
+ failure.read_progress,
+ ),
+ None if self.reached_eof => ("eof", "none", "none", "none", "none"),
+ None => ("partial", "none", "none", "none", "none"),
};
let tags = [
("format", self.format),
("outcome", outcome),
("stage", stage),
("error_kind", error_kind),
+ ("reason", reason),
+ ("read_progress", read_progress),
];
let _ = metrics.counter("codex.rollout_compression.read", /*inc*/ 1, &tags);
let _ = metrics.record_duration(