Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add diagnostic labels to shell snapshot capture metrics (#43454)

jifc84003c7e18d
diff --git a/codex-rs/exec-server/src/shell_snapshot.rs b/codex-rs/exec-server/src/shell_snapshot.rs
index fe3e2a2a16..7254c1bda5 100644
--- a/codex-rs/exec-server/src/shell_snapshot.rs
+++ b/codex-rs/exec-server/src/shell_snapshot.rs
@@ -56,6 +56,9 @@ struct ShellSnapshot {
environment: HashMap<String, String>,
}
+// Keep a bounded metric label alongside the original RPC error.
+type CaptureResult = Result<ShellSnapshot, (&'static str, JSONRPCErrorError)>;
+
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum CapturePurpose {
Execution,
@@ -98,7 +101,7 @@ impl ShellSnapshotCache {
}
};
- let snapshot = {
+ let (snapshot, attempt) = {
let mut entries = self.entries.lock().await;
let position = entries.iter().position(|entry| {
&entry.request == request
@@ -119,8 +122,9 @@ impl ShellSnapshotCache {
entry.snapshot = Arc::new(OnceCell::new());
}
let snapshot = Arc::clone(&entry.snapshot);
+ let attempt = entry.attempts;
entries.push_back(entry);
- Some(snapshot)
+ Some((snapshot, attempt))
});
if let Some(snapshot) = cached {
snapshot
@@ -139,17 +143,28 @@ impl ShellSnapshotCache {
entries.pop_front();
}
- snapshot
+ (snapshot, 1)
}
};
let capture = async {
+ let attempt = attempt.to_string();
+ let purpose = match purpose {
+ CapturePurpose::Execution => "execution",
+ CapturePurpose::Prewarm => "prewarm",
+ };
let started_at = std::time::Instant::now();
let result = capture_snapshot(params, prepared, shell_type).await;
telemetry.shell_snapshot_captured(
started_at.elapsed(),
- result.as_ref().map(|_| ()).map_err(|_| "capture_failed"),
+ result.as_ref().map(|_| ()).map_err(|(reason, _)| *reason),
+ &[
+ ("purpose", purpose),
+ ("attempt", &attempt),
+ ("shell", request.shell.name.as_str()),
+ ("sandbox", prepared.sandbox.as_metric_tag()),
+ ],
);
- result
+ result.map_err(|(_, error)| error)
};
let snapshot = match purpose {
CapturePurpose::Execution => {
@@ -239,15 +254,22 @@ async fn capture_snapshot(
params: &ExecParams,
prepared: &PreparedExecRequest,
shell_type: ShellType,
-) -> Result<ShellSnapshot, JSONRPCErrorError> {
- let script = snapshot_state_and_environment_script(shell_type)
- .ok_or_else(|| invalid_params("unsupported shell snapshot script".to_string()))?;
+) -> CaptureResult {
+ let script = snapshot_state_and_environment_script(shell_type).ok_or_else(|| {
+ (
+ "unsupported_shell",
+ invalid_params("unsupported shell snapshot script".to_string()),
+ )
+ })?;
let shell_start = prepared.command.len() - params.argv.len();
let mut argv = prepared.command.clone();
argv[shell_start + 2] = script;
- let (program, args) = argv
- .split_first()
- .ok_or_else(|| internal_error("missing shell snapshot command".to_string()))?;
+ let (program, args) = argv.split_first().ok_or_else(|| {
+ (
+ "missing_command",
+ internal_error("missing shell snapshot command".to_string()),
+ )
+ })?;
let mut command = Command::new(program);
command
@@ -262,59 +284,86 @@ async fn capture_snapshot(
if let Some(arg0) = &prepared.arg0 {
command.arg0(arg0);
}
- let mut child = command
- .spawn()
- .map_err(|err| internal_error(format!("cannot capture shell snapshot: {err}")))?;
- let stdout = child
- .stdout
- .take()
- .ok_or_else(|| internal_error("missing shell snapshot output".to_string()))?;
+ let mut child = command.spawn().map_err(|err| {
+ (
+ "spawn_failed",
+ internal_error(format!("cannot capture shell snapshot: {err}")),
+ )
+ })?;
+ let stdout = child.stdout.take().ok_or_else(|| {
+ (
+ "missing_output",
+ internal_error("missing shell snapshot output".to_string()),
+ )
+ })?;
let capture = async {
let mut output = Vec::new();
stdout
.take((MAX_SNAPSHOT_BYTES + 1) as u64)
.read_to_end(&mut output)
.await
- .map_err(|err| internal_error(format!("cannot read shell snapshot: {err}")))?;
+ .map_err(|err| {
+ (
+ "read_failed",
+ internal_error(format!("cannot read shell snapshot: {err}")),
+ )
+ })?;
if output.len() > MAX_SNAPSHOT_BYTES {
- return Err(internal_error(format!(
- "shell snapshot exceeds {MAX_SNAPSHOT_BYTES} bytes"
- )));
+ return Err((
+ "too_large",
+ internal_error(format!("shell snapshot exceeds {MAX_SNAPSHOT_BYTES} bytes")),
+ ));
}
- let status = child
- .wait()
- .await
- .map_err(|err| internal_error(format!("cannot finish shell snapshot: {err}")))?;
+ let status = child.wait().await.map_err(|err| {
+ (
+ "wait_failed",
+ internal_error(format!("cannot finish shell snapshot: {err}")),
+ )
+ })?;
if !status.success() {
- return Err(internal_error(format!(
- "shell snapshot capture exited with {status}"
- )));
+ return Err((
+ "nonzero_exit",
+ internal_error(format!("shell snapshot capture exited with {status}")),
+ ));
}
Ok(output)
};
let output = tokio::time::timeout(SNAPSHOT_TIMEOUT, capture)
.await
- .map_err(|_| internal_error("shell snapshot capture timed out".to_string()))??;
+ .map_err(|_| {
+ (
+ "timeout",
+ internal_error("shell snapshot capture timed out".to_string()),
+ )
+ })??;
parse_snapshot(&output, params.env_policy.as_ref())
}
-fn parse_snapshot(
- output: &[u8],
- env_policy: Option<&ExecEnvPolicy>,
-) -> Result<ShellSnapshot, JSONRPCErrorError> {
- let separator = output
- .iter()
- .position(|byte| *byte == 0)
- .ok_or_else(|| internal_error("shell snapshot is missing its environment".to_string()))?;
+fn parse_snapshot(output: &[u8], env_policy: Option<&ExecEnvPolicy>) -> CaptureResult {
+ let separator = output.iter().position(|byte| *byte == 0).ok_or_else(|| {
+ (
+ "missing_environment",
+ internal_error("shell snapshot is missing its environment".to_string()),
+ )
+ })?;
let state = &output[..separator];
let marker = b"# Snapshot file";
let start = state
.windows(marker.len())
.position(|window| window == marker)
- .ok_or_else(|| internal_error("shell snapshot is missing its state marker".to_string()))?;
- let state = std::str::from_utf8(&state[start..])
- .map_err(|err| internal_error(format!("shell snapshot state is not UTF-8: {err}")))?;
+ .ok_or_else(|| {
+ (
+ "missing_state",
+ internal_error("shell snapshot is missing its state marker".to_string()),
+ )
+ })?;
+ let state = std::str::from_utf8(&state[start..]).map_err(|err| {
+ (
+ "invalid_utf8",
+ internal_error(format!("shell snapshot state is not UTF-8: {err}")),
+ )
+ })?;
let mut environment = output[separator + 1..]
.split(|byte| *byte == 0)
diff --git a/codex-rs/exec-server/src/shell_snapshot_tests.rs b/codex-rs/exec-server/src/shell_snapshot_tests.rs
index b5df524235..74e5c63cf6 100644
--- a/codex-rs/exec-server/src/shell_snapshot_tests.rs
+++ b/codex-rs/exec-server/src/shell_snapshot_tests.rs
@@ -199,24 +199,27 @@ async fn snapshot_failure_retries_are_bounded_and_single_flight(
}
let mut expected_counters = BTreeMap::new();
let mut expected_durations = BTreeMap::new();
- let failures =
- (recovery_attempt - 1).min(MAX_SNAPSHOT_ATTEMPTS) as u64 + u64::from(prewarm_fails_first);
- for (success, count) in [
- ("false", failures),
- ("true", u64::from(recovery_attempt <= MAX_SNAPSHOT_ATTEMPTS)),
- ] {
- if count == 0 {
- continue;
- }
+ let captures = prewarm_fails_first
+ .then_some(("prewarm", 1))
+ .into_iter()
+ .chain(
+ (1..=recovery_attempt.min(MAX_SNAPSHOT_ATTEMPTS)).map(|attempt| ("execution", attempt)),
+ );
+ for (purpose, attempt) in captures {
+ let success = purpose == "execution" && attempt == recovery_attempt;
let mut tags = BTreeMap::from([
("version".to_string(), "v2".to_string()),
("success".to_string(), success.to_string()),
+ ("purpose".to_string(), purpose.to_string()),
+ ("attempt".to_string(), attempt.to_string()),
+ ("shell".to_string(), "bash".to_string()),
+ ("sandbox".to_string(), "none".to_string()),
]);
- expected_durations.insert(tags.clone(), count);
- if success == "false" {
- tags.insert("failure_reason".to_string(), "capture_failed".to_string());
+ if !success {
+ tags.insert("failure_reason".to_string(), "nonzero_exit".to_string());
}
- expected_counters.insert(tags, count);
+ expected_durations.insert(tags.clone(), /*value*/ 1);
+ expected_counters.insert(tags, /*value*/ 1);
}
assert_eq!(
(counters, durations),
diff --git a/codex-rs/exec-server/src/telemetry.rs b/codex-rs/exec-server/src/telemetry.rs
index 8f54473691..96c9a908d2 100644
--- a/codex-rs/exec-server/src/telemetry.rs
+++ b/codex-rs/exec-server/src/telemetry.rs
@@ -186,6 +186,7 @@ impl ExecServerTelemetry {
&self,
duration: Duration,
result: Result<(), &'static str>,
+ capture_tags: &[(&str, &str)],
) {
// Local execution has no exec-server telemetry owner. Use the host's
// configured metrics client while preserving an explicit server client.
@@ -199,10 +200,11 @@ impl ExecServerTelemetry {
};
let success = if result.is_ok() { "true" } else { "false" };
let mut tags = vec![("version", "v2"), ("success", success)];
- let _ = metrics.record_duration("codex.shell_snapshot.duration_ms", duration, &tags);
+ tags.extend_from_slice(capture_tags);
if let Err(failure_reason) = result {
tags.push(("failure_reason", failure_reason));
}
+ let _ = metrics.record_duration("codex.shell_snapshot.duration_ms", duration, &tags);
let _ = metrics.counter("codex.shell_snapshot", /*inc*/ 1, &tags);
}