Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Instrument exec-server shell snapshot capture (#40620)

jife16d098c00e4
diff --git a/codex-rs/core/src/shell_snapshot.rs b/codex-rs/core/src/shell_snapshot.rs
index b0782e5c5a..2e0d734a98 100644
--- a/codex-rs/core/src/shell_snapshot.rs
+++ b/codex-rs/core/src/shell_snapshot.rs
@@ -94,7 +94,7 @@ impl ShellSnapshot {
async {
let timer = config
.session_telemetry
- .start_timer("codex.shell_snapshot.duration_ms", &[]);
+ .start_timer("codex.shell_snapshot.duration_ms", &[("version", "v1")]);
let snapshot = ShellSnapshot::try_create(
&config.codex_home,
config.session_id,
@@ -105,7 +105,7 @@ impl ShellSnapshot {
.await;
let success_tag = if snapshot.is_ok() { "true" } else { "false" };
let _ = timer.map(|timer| timer.record(&[("success", success_tag)]));
- let mut counter_tags = vec![("success", success_tag)];
+ let mut counter_tags = vec![("version", "v1"), ("success", success_tag)];
if let Some(failure_reason) = snapshot.as_ref().err() {
counter_tags.push(("failure_reason", *failure_reason));
}
diff --git a/codex-rs/exec-server/src/local_process.rs b/codex-rs/exec-server/src/local_process.rs
index 927276f36e..27746255ce 100644
--- a/codex-rs/exec-server/src/local_process.rs
+++ b/codex-rs/exec-server/src/local_process.rs
@@ -355,7 +355,7 @@ impl LocalProcess {
#[cfg(unix)]
self.inner
.shell_snapshots
- .prepare(&params, &mut prepared)
+ .prepare(&params, &mut prepared, &self.inner.telemetry)
.await?;
if prepared.command.is_empty() {
return Err(invalid_params("argv must not be empty".to_string()));
diff --git a/codex-rs/exec-server/src/shell_snapshot.rs b/codex-rs/exec-server/src/shell_snapshot.rs
index e352d63641..324580a66f 100644
--- a/codex-rs/exec-server/src/shell_snapshot.rs
+++ b/codex-rs/exec-server/src/shell_snapshot.rs
@@ -26,6 +26,7 @@ use crate::protocol::ExecParams;
use crate::protocol::ShellSnapshotRequest;
use crate::rpc::internal_error;
use crate::rpc::invalid_params;
+use crate::telemetry::ExecServerTelemetry;
const MAX_CACHED_SNAPSHOTS: usize = 16;
const MAX_SNAPSHOT_BYTES: usize = 512 * 1024;
@@ -60,6 +61,7 @@ impl ShellSnapshotCache {
&self,
params: &ExecParams,
prepared: &mut PreparedExecRequest,
+ telemetry: &ExecServerTelemetry,
) -> Result<(), JSONRPCErrorError> {
let Some(request) = params.shell_snapshot.as_ref() else {
return Ok(());
@@ -134,12 +136,16 @@ impl ShellSnapshotCache {
};
let Ok(snapshot) = snapshot
.get_or_init(|| async {
- capture_snapshot(params, prepared, shell_type)
- .await
- .map_err(|err| {
- tracing::warn!("failed to capture shell snapshot: {err:?}");
- Instant::now() + SNAPSHOT_RETRY_BACKOFF
- })
+ let started_at = std::time::Instant::now();
+ let result = capture_snapshot(params, prepared, shell_type).await;
+ telemetry.shell_snapshot_captured(
+ started_at.elapsed(),
+ result.as_ref().map(|_| ()).map_err(|_| "capture_failed"),
+ );
+ result.map_err(|err| {
+ tracing::warn!("failed to capture shell snapshot: {err:?}");
+ Instant::now() + SNAPSHOT_RETRY_BACKOFF
+ })
})
.await
else {
diff --git a/codex-rs/exec-server/src/shell_snapshot_tests.rs b/codex-rs/exec-server/src/shell_snapshot_tests.rs
index 1f24ef5de8..f8a380a3f1 100644
--- a/codex-rs/exec-server/src/shell_snapshot_tests.rs
+++ b/codex-rs/exec-server/src/shell_snapshot_tests.rs
@@ -1,6 +1,12 @@
+use std::collections::BTreeMap;
use std::collections::HashMap;
+use codex_otel::MetricsClient;
+use codex_otel::MetricsConfig;
use codex_protocol::config_types::ShellEnvironmentPolicyInherit;
+use opentelemetry_sdk::metrics::InMemoryMetricExporter;
+use opentelemetry_sdk::metrics::data::AggregatedMetrics;
+use opentelemetry_sdk::metrics::data::MetricData;
use pretty_assertions::assert_eq;
use test_case::test_case;
@@ -14,7 +20,9 @@ use crate::protocol::ExecParams;
use crate::protocol::ProcessId;
use crate::protocol::ShellInfo;
use crate::protocol::ShellSnapshotRequest;
+use crate::telemetry::ExecServerTelemetry;
+#[test_case(1; "succeeds_on_first_attempt")]
#[test_case(2; "recovers_on_second_attempt")]
#[test_case(3; "recovers_on_last_attempt")]
#[test_case(4; "stops_after_three_failures")]
@@ -57,6 +65,16 @@ async fn snapshot_failure_retries_are_bounded_and_single_flight(
network_proxy: None,
};
let cache = ShellSnapshotCache::default();
+ let metrics = MetricsClient::new(
+ MetricsConfig::in_memory(
+ "test",
+ "codex-exec-server",
+ env!("CARGO_PKG_VERSION"),
+ InMemoryMetricExporter::default(),
+ )
+ .with_runtime_reader(),
+ )?;
+ let telemetry = ExecServerTelemetry::new(metrics.clone());
for attempt in 1..=5 {
if attempt == recovery_attempt {
@@ -84,8 +102,8 @@ async fn snapshot_failure_retries_are_bounded_and_single_flight(
.await
.expect("prepare concurrent capture");
let (first, second) = tokio::join!(
- cache.prepare(&params, &mut prepared),
- cache.prepare(&params, &mut concurrent),
+ cache.prepare(&params, &mut prepared, &telemetry),
+ cache.prepare(&params, &mut concurrent, &telemetry),
);
first.expect("capture failure must preserve command fallback");
second.expect("concurrent request must share the capture attempt");
@@ -97,7 +115,7 @@ async fn snapshot_failure_retries_are_bounded_and_single_flight(
tokio::time::pause();
if attempt < recovery_attempt || recovery_attempt > MAX_SNAPSHOT_ATTEMPTS {
cache
- .prepare(&params, &mut prepared)
+ .prepare(&params, &mut prepared, &telemetry)
.await
.expect("capture must stay cached during backoff");
assert_eq!(
@@ -114,6 +132,66 @@ async fn snapshot_failure_retries_are_bounded_and_single_flight(
tokio::time::advance(SNAPSHOT_RETRY_BACKOFF).await;
tokio::time::resume();
}
+
+ let snapshot = metrics.snapshot()?;
+ let mut counters = BTreeMap::new();
+ let mut durations = BTreeMap::new();
+ for metric in snapshot
+ .scope_metrics()
+ .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)
+ {
+ match metric.name() {
+ "codex.shell_snapshot" => {
+ let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
+ panic!("expected shell snapshot counter");
+ };
+ for point in sum.data_points() {
+ let tags = point
+ .attributes()
+ .map(|attribute| (attribute.key.to_string(), attribute.value.to_string()))
+ .collect::<BTreeMap<_, _>>();
+ counters.insert(tags, point.value());
+ }
+ }
+ "codex.shell_snapshot.duration_ms" => {
+ let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
+ panic!("expected shell snapshot duration histogram");
+ };
+ for point in histogram.data_points() {
+ let tags = point
+ .attributes()
+ .map(|attribute| (attribute.key.to_string(), attribute.value.to_string()))
+ .collect::<BTreeMap<_, _>>();
+ durations.insert(tags, point.count());
+ }
+ }
+ _ => {}
+ }
+ }
+ let mut expected_counters = BTreeMap::new();
+ let mut expected_durations = BTreeMap::new();
+ let failures = (recovery_attempt - 1).min(MAX_SNAPSHOT_ATTEMPTS) as u64;
+ for (success, count) in [
+ ("false", failures),
+ ("true", u64::from(recovery_attempt <= MAX_SNAPSHOT_ATTEMPTS)),
+ ] {
+ if count == 0 {
+ continue;
+ }
+ let mut tags = BTreeMap::from([
+ ("version".to_string(), "v2".to_string()),
+ ("success".to_string(), success.to_string()),
+ ]);
+ expected_durations.insert(tags.clone(), count);
+ if success == "false" {
+ tags.insert("failure_reason".to_string(), "capture_failed".to_string());
+ }
+ expected_counters.insert(tags, count);
+ }
+ assert_eq!(
+ (counters, durations),
+ (expected_counters, expected_durations)
+ );
Ok(())
}
diff --git a/codex-rs/exec-server/src/telemetry.rs b/codex-rs/exec-server/src/telemetry.rs
index e78d2ec455..13dbbb8a4f 100644
--- a/codex-rs/exec-server/src/telemetry.rs
+++ b/codex-rs/exec-server/src/telemetry.rs
@@ -159,6 +159,31 @@ impl ExecServerTelemetry {
});
}
+ #[cfg(unix)]
+ pub(crate) fn shell_snapshot_captured(
+ &self,
+ duration: Duration,
+ result: Result<(), &'static str>,
+ ) {
+ // Local execution has no exec-server telemetry owner. Use the host's
+ // configured metrics client while preserving an explicit server client.
+ let Some(metrics) = self
+ .inner
+ .as_ref()
+ .map(|inner| inner.metrics.clone())
+ .or_else(codex_otel::global)
+ else {
+ return;
+ };
+ let success = if result.is_ok() { "true" } else { "false" };
+ let mut tags = vec![("version", "v2"), ("success", success)];
+ let _ = metrics.record_duration("codex.shell_snapshot.duration_ms", duration, &tags);
+ if let Err(failure_reason) = result {
+ tags.push(("failure_reason", failure_reason));
+ }
+ let _ = metrics.counter("codex.shell_snapshot", /*inc*/ 1, &tags);
+ }
+
pub(crate) fn remote_registration_completed(&self, result: &'static str, duration: Duration) {
self.record_operation(REMOTE_REGISTRATION_METRICS, result, duration);
}