Lightweight coding agent that runs in your terminal
Rust
Add multi-agent spawn latency and failure metrics (#47984)
Owen Lin9ef08dcf2bda
diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rsindex 719336828f..705d48c1d5 100644--- a/codex-rs/core/src/agent/control.rs+++ b/codex-rs/core/src/agent/control.rs@@ -77,6 +77,7 @@ mod sender_context;mod service_tier;mod spawn;mod spawn_guard;+mod spawn_telemetry;mod target;mod user_authorization;mod watch;diff --git a/codex-rs/core/src/agent/control/spawn.rs b/codex-rs/core/src/agent/control/spawn.rsindex 7fe302678d..9b642981a3 100644--- a/codex-rs/core/src/agent/control/spawn.rs+++ b/codex-rs/core/src/agent/control/spawn.rs@@ -1,5 +1,7 @@use super::residency::is_v2_resident_session_source;use super::spawn_guard::PendingSpawn;+use super::spawn_telemetry::SpawnMeasurements;+use super::spawn_telemetry::record_spawn_success;use super::*;use crate::agent::child_config::build_agent_resume_config;use crate::agent::role::apply_role_to_config;@@ -31,6 +33,8 @@ use codex_thread_store::PersistContext;use codex_utils_path_uri::PathUri;use futures::StreamExt;use futures::stream;+use std::time::Duration;+use std::time::Instant;const AGENT_NAMES: &str = include_str!("../../../assets/agent/agent_names.txt");@@ -39,6 +43,12 @@ struct SpawnAgentThreadInheritance {exec_policy: Option<Arc<crate::exec_policy::ExecPolicyManager>>,}+struct SpawnedThreadResult {+ new_thread: crate::thread_manager::NewThread,+ fork_context: Option<Duration>,+ child_create: Duration,+}+/// Initial input delivered after a spawned agent acquires execution capacity.////// V2 communication spawns keep the communication and its context paired so centralized@@ -624,6 +634,7 @@ impl LocalAgentControl {session_source: Option<SessionSource>,options: SpawnAgentOptions,) -> CodexResult<(LiveAgent, ThreadConfigSnapshot)> {+ let spawn_started_at = Instant::now();let state = self.runtime.upgrade()?;let multi_agent_version = state.effective_multi_agent_version_for_spawn(@@ -634,6 +645,7 @@ impl LocalAgentControl {&config,).await;+ let product_sku = config.apps_mcp_product_sku.clone();if let Some(session_source) = session_source.as_ref() {self.ensure_execution_capacity(multi_agent_version, session_source)?;}@@ -642,13 +654,17 @@ impl LocalAgentControl {&& session_source.as_ref().is_some_and(is_v2_resident_session_source);- let residency_slot = if spawn_uses_v2_residency {- Some(- self.reserve_v2_residency_slot(&state, &config, /*protected_thread_id*/ None)- .await?,+ let (residency_slot, residency_reservation) = if spawn_uses_v2_residency {+ let residency_reservation_started_at = Instant::now();+ let residency_slot = self+ .reserve_v2_residency_slot(&state, &config, /*protected_thread_id*/ None)+ .await?;+ (+ Some(residency_slot),+ Some(residency_reservation_started_at.elapsed()),)} else {- None+ (None, None)};let reservation_max_threads = if spawn_uses_v2_residency {None@@ -691,7 +707,11 @@ impl LocalAgentControl {let notification_source = session_source.clone();// The same `LocalAgentControl` is sent to spawn the thread.- let new_thread = match (session_source, options.fork_mode.as_ref(), inheritance) {+ let SpawnedThreadResult {+ new_thread,+ fork_context,+ child_create,+ } = match (session_source, options.fork_mode.as_ref(), inheritance) {(Some(session_source), Some(_), inheritance) => {Box::pin(self.spawn_forked_thread(&state,@@ -715,7 +735,8 @@ impl LocalAgentControl {} else {None};- Box::pin(state.spawn_new_thread_with_source(+ let child_create_started_at = Instant::now();+ let new_thread = Box::pin(state.spawn_new_thread_with_source(config.clone(),self.clone(),session_source,@@ -728,9 +749,23 @@ impl LocalAgentControl {inheritance.exec_policy,options.environments.clone(),))- .await?+ .await?;+ SpawnedThreadResult {+ new_thread,+ fork_context: None,+ child_create: child_create_started_at.elapsed(),+ }+ }+ (None, _, _) => {+ let child_create_started_at = Instant::now();+ let new_thread =+ Box::pin(state.spawn_new_thread(config.clone(), self.clone())).await?;+ SpawnedThreadResult {+ new_thread,+ fork_context: None,+ child_create: child_create_started_at.elapsed(),+ }}- (None, _, _) => Box::pin(state.spawn_new_thread(config.clone(), self.clone())).await?,};agent_metadata.agent_id = Some(new_thread.thread_id);let mut pending_spawn = PendingSpawn::new(Arc::clone(&state), new_thread.thread_id);@@ -781,6 +816,7 @@ impl LocalAgentControl {).await;}));+ let durability_wait_started_at = Instant::now();if options.fork_mode.is_some() {tokio::join!(new_thread@@ -792,6 +828,7 @@ impl LocalAgentControl {} else {pending_spawn.wait_for_edge().await;}+ let durability_wait = durability_wait_started_at.elapsed();let start_options = TurnStartOptions {parent_turn_id: options.parent_turn_id,@@ -800,6 +837,7 @@ impl LocalAgentControl {cyber_access_program: options.cyber_access_program,..Default::default()};+ let input_admission_started_at = Instant::now();match initial_input {SpawnInitialInput::UserInput(input) => {self.send_input(new_thread.thread_id, input, start_options)@@ -816,6 +854,7 @@ impl LocalAgentControl {.await?;}}+ let input_admission = input_admission_started_at.elapsed();reservation.commit(agent_metadata.clone());if let Some(residency_slot) = residency_slot {residency_slot.commit(new_thread.thread_id);@@ -846,6 +885,24 @@ impl LocalAgentControl {status: self.get_status(new_thread.thread_id).await,};let config = new_thread.thread.config_snapshot().await;+ let session_telemetry = new_thread+ .thread+ .session_telemetry()+ .with_product_sku(product_sku.as_deref());+ record_spawn_success(+ &session_telemetry,+ options.fork_mode.as_ref(),+ multi_agent_version,+ SpawnMeasurements {+ history_mode: config.history_mode,+ residency_reservation,+ fork_context,+ child_create,+ durability_wait,+ input_admission,+ total: spawn_started_at.elapsed(),+ },+ );Ok((agent, config))}@@ -857,7 +914,7 @@ impl LocalAgentControl {options: &SpawnAgentOptions,inheritance: SpawnAgentThreadInheritance,multi_agent_version: MultiAgentVersion,- ) -> CodexResult<crate::thread_manager::NewThread> {+ ) -> CodexResult<SpawnedThreadResult> {let SpawnAgentThreadInheritance {environments: inherited_environments,exec_policy: inherited_exec_policy,@@ -881,6 +938,7 @@ impl LocalAgentControl {));};+ let fork_context_started_at = Instant::now();let parent_thread_id = *parent_thread_id;let parent_thread = state.get_thread(parent_thread_id).await?;let (subagent_developer_instructions, parent_developer_instructions) = match (@@ -1157,7 +1215,9 @@ impl LocalAgentControl {let mut thread_extension_init = ExtensionDataInit::new();thread_extension_init.insert(selected_capability_roots);- state+ let fork_context = fork_context_started_at.elapsed();+ let child_create_started_at = Instant::now();+ let new_thread = state.fork_thread_with_source(config.clone(),InitialHistory::Forked(forked_rollout_items),@@ -1172,7 +1232,13 @@ impl LocalAgentControl {options.environments.clone(),thread_extension_init,)- .await+ .await?;+ let child_create = child_create_started_at.elapsed();+ Ok(SpawnedThreadResult {+ new_thread,+ fork_context: Some(fork_context),+ child_create,+ })}/// Resume an existing agent thread from a recorded rollout file.diff --git a/codex-rs/core/src/agent/control/spawn_telemetry.rs b/codex-rs/core/src/agent/control/spawn_telemetry.rsnew file mode 100644index 0000000000..7b681ce9c7--- /dev/null+++ b/codex-rs/core/src/agent/control/spawn_telemetry.rs@@ -0,0 +1,65 @@+//! Successful multi-agent spawn latency metrics.+//!+//! Runtime values are collapsed to bounded labels before emission. Failed spawns do not emit+//! partial phase timings.++use crate::agent::types::SpawnAgentForkMode;+use codex_otel::SessionTelemetry;+use codex_protocol::protocol::MultiAgentVersion;+use codex_protocol::protocol::ThreadHistoryMode;+use std::time::Duration;++pub(super) struct SpawnMeasurements {+ pub(super) history_mode: ThreadHistoryMode,+ pub(super) residency_reservation: Option<Duration>,+ pub(super) fork_context: Option<Duration>,+ pub(super) child_create: Duration,+ pub(super) durability_wait: Duration,+ pub(super) input_admission: Duration,+ pub(super) total: Duration,+}++pub(super) fn record_spawn_success(+ telemetry: &SessionTelemetry,+ fork_mode: Option<&SpawnAgentForkMode>,+ multi_agent_version: MultiAgentVersion,+ measurements: SpawnMeasurements,+) {+ let fork_mode = match fork_mode {+ None => "none",+ Some(SpawnAgentForkMode::FullHistory) => "all",+ Some(SpawnAgentForkMode::LastNTurns(_)) => "last_n",+ };+ let history_mode = match measurements.history_mode {+ ThreadHistoryMode::Legacy => "legacy",+ ThreadHistoryMode::Paginated => "paginated",+ };+ let multi_agent_version = match multi_agent_version {+ MultiAgentVersion::Disabled => "disabled",+ MultiAgentVersion::V1 => "v1",+ MultiAgentVersion::V2 => "v2",+ };+ let record_phase = |phase, duration| {+ telemetry.record_multi_agent_spawn_phase(+ phase,+ duration,+ fork_mode,+ history_mode,+ multi_agent_version,+ );+ };+ if let Some(duration) = measurements.residency_reservation {+ record_phase("residency_reservation", duration);+ }+ if let Some(duration) = measurements.fork_context {+ record_phase("fork_context", duration);+ }+ record_phase("child_create", measurements.child_create);+ record_phase("durability_wait", measurements.durability_wait);+ record_phase("input_admission", measurements.input_admission);+ record_phase("total", measurements.total);+}++#[cfg(test)]+#[path = "spawn_telemetry_tests.rs"]+mod tests;diff --git a/codex-rs/core/src/agent/control/spawn_telemetry_tests.rs b/codex-rs/core/src/agent/control/spawn_telemetry_tests.rsnew file mode 100644index 0000000000..410ace28d9--- /dev/null+++ b/codex-rs/core/src/agent/control/spawn_telemetry_tests.rs@@ -0,0 +1,95 @@+use super::*;+use codex_otel::MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC;+use codex_otel::MetricsClient;+use codex_otel::MetricsConfig;+use codex_protocol::ThreadId;+use codex_protocol::protocol::SessionSource;+use opentelemetry_sdk::metrics::InMemoryMetricExporter;+use opentelemetry_sdk::metrics::data::AggregatedMetrics;+use opentelemetry_sdk::metrics::data::MetricData;+use pretty_assertions::assert_eq;+use std::collections::BTreeMap;++#[test]+fn records_bounded_spawn_phase_metrics() {+ let metrics = MetricsClient::new(+ MetricsConfig::in_memory(+ "test",+ "codex-core",+ env!("CARGO_PKG_VERSION"),+ InMemoryMetricExporter::default(),+ )+ .with_runtime_reader(),+ )+ .expect("create in-memory metrics client");+ let telemetry = SessionTelemetry::new(+ ThreadId::new(),+ "test-model",+ "test-model",+ /*account_id*/ None,+ /*account_email*/ None,+ /*auth_mode*/ None,+ "test-originator".to_string(),+ /*log_user_prompts*/ false,+ "test-terminal".to_string(),+ SessionSource::Cli,+ )+ .with_product_sku(Some("codex"))+ .with_metrics_without_metadata_tags(metrics.clone());++ record_spawn_success(+ &telemetry,+ Some(&SpawnAgentForkMode::LastNTurns(3)),+ MultiAgentVersion::V2,+ SpawnMeasurements {+ history_mode: ThreadHistoryMode::Paginated,+ residency_reservation: Some(Duration::from_millis(11)),+ fork_context: Some(Duration::from_millis(13)),+ child_create: Duration::from_millis(17),+ durability_wait: Duration::from_millis(19),+ input_admission: Duration::from_millis(23),+ total: Duration::from_millis(29),+ },+ );++ let snapshot = metrics.snapshot().expect("snapshot spawn metrics");+ let metric = snapshot+ .scope_metrics()+ .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)+ .find(|metric| metric.name() == MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC)+ .expect("spawn phase metric");+ let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {+ panic!("expected spawn phase histogram");+ };+ let phases = histogram+ .data_points()+ .map(|point| {+ let attributes = point+ .attributes()+ .map(|attribute| {+ (+ attribute.key.as_str().to_string(),+ attribute.value.as_str().to_string(),+ )+ })+ .collect::<BTreeMap<_, _>>();+ assert_eq!(attributes.len(), 5);+ assert_eq!(attributes["fork_mode"], "last_n");+ assert_eq!(attributes["history_mode"], "paginated");+ assert_eq!(attributes["multi_agent_version"], "v2");+ assert_eq!(attributes["product_sku"], "codex");+ (attributes["phase"].clone(), (point.count(), point.sum()))+ })+ .collect::<BTreeMap<_, _>>();+ assert_eq!(+ phases,+ BTreeMap::from([+ ("child_create".to_string(), (1, 17.0)),+ ("durability_wait".to_string(), (1, 19.0)),+ ("fork_context".to_string(), (1, 13.0)),+ ("input_admission".to_string(), (1, 23.0)),+ ("residency_reservation".to_string(), (1, 11.0)),+ ("total".to_string(), (1, 29.0)),+ ])+ );+}diff --git a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rsindex 0961db39f8..9bfc5571c4 100644--- a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs+++ b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs@@ -12,6 +12,7 @@ use crate::agent::types::SpawnAgentForkMode;use crate::agent::types::SpawnAgentOptions;use crate::tools::handlers::multi_agents_spec::SpawnAgentToolOptions;use crate::tools::handlers::multi_agents_spec::create_spawn_agent_tool_v1;+use codex_protocol::protocol::MultiAgentVersion;use codex_tools::ToolSpec;#[derive(Default)]@@ -108,6 +109,7 @@ async fn handle_spawn_agent(.await.map_err(FunctionCallError::RespondToModel)?;let config = prepared.config;+ let fork_mode = args.fork_context.then_some(SpawnAgentForkMode::FullHistory);let result = session.services.agent_control@@ -123,8 +125,8 @@ async fn handle_spawn_agent(/*task_name*/ None,)?,options: SpawnAgentOptions {- fork_parent_spawn_call_id: args.fork_context.then(|| call_id.clone()),- fork_mode: args.fork_context.then_some(SpawnAgentForkMode::FullHistory),+ fork_parent_spawn_call_id: fork_mode.as_ref().map(|_| call_id.clone()),+ fork_mode: fork_mode.clone(),parent_thread_id: Some(session.thread_id),parent_turn_id: Some(turn.sub_id.clone()),root_turn_id: turn.turn_metadata_state.root_turn_id(),@@ -135,7 +137,16 @@ async fn handle_spawn_agent(},}).await- .map_err(collab_spawn_error);+ .map_err(|err| {+ record_collab_spawn_failure(+ &turn.session_telemetry,+ turn.config.apps_mcp_product_sku.as_deref(),+ &err,+ fork_mode.as_ref(),+ MultiAgentVersion::V1,+ );+ collab_spawn_error(err)+ });let (new_thread_id, status) = match &result {Ok((spawned_agent, _)) => (Some(spawned_agent.thread_id), spawned_agent.status.clone()),Err(_) => (None, AgentStatus::NotFound),diff --git a/codex-rs/core/src/tools/handlers/multi_agents_common.rs b/codex-rs/core/src/tools/handlers/multi_agents_common.rsindex 4f8f9072d0..be17be61b3 100644--- a/codex-rs/core/src/tools/handlers/multi_agents_common.rs+++ b/codex-rs/core/src/tools/handlers/multi_agents_common.rs@@ -1,14 +1,17 @@+use crate::agent::types::SpawnAgentForkMode;use crate::config::DEFAULT_MULTI_AGENT_V2_MIN_WAIT_TIMEOUT_MS;use crate::config::HARD_MAX_MULTI_AGENT_V2_TIMEOUT_MS;use crate::function_tool::FunctionCallError;use crate::tools::context::FunctionToolOutput;use crate::tools::context::ToolOutput;use crate::tools::context::ToolPayload;+use codex_otel::SessionTelemetry;use codex_protocol::AgentPath;use codex_protocol::ThreadId;use codex_protocol::error::CodexErr;use codex_protocol::error::CodexErrorDetails;use codex_protocol::models::ResponseInputItem;+use codex_protocol::protocol::MultiAgentVersion;use codex_protocol::protocol::SessionSource;use codex_protocol::protocol::SubAgentSource;use codex_protocol::user_input::UserInput;@@ -72,6 +75,36 @@ pub(crate) fn collab_spawn_error(err: CodexErr) -> FunctionCallError {}}+pub(crate) fn record_collab_spawn_failure(+ telemetry: &SessionTelemetry,+ product_sku: Option<&str>,+ err: &CodexErr,+ fork_mode: Option<&SpawnAgentForkMode>,+ multi_agent_version: MultiAgentVersion,+) {+ let reason = match err.details() {+ CodexErrorDetails::AgentLimitReached { .. } => "limit_reached",+ CodexErrorDetails::InvalidRequest(_) => "invalid_request",+ CodexErrorDetails::ThreadNotFound(_) => "thread_not_found",+ CodexErrorDetails::UnsupportedOperation(_) => "unsupported_operation",+ _ => "internal",+ };+ let fork_mode = match fork_mode {+ None => "none",+ Some(SpawnAgentForkMode::FullHistory) => "all",+ Some(SpawnAgentForkMode::LastNTurns(_)) => "last_n",+ };+ let multi_agent_version = match multi_agent_version {+ MultiAgentVersion::Disabled => "disabled",+ MultiAgentVersion::V1 => "v1",+ MultiAgentVersion::V2 => "v2",+ };+ telemetry+ .clone()+ .with_product_sku(product_sku)+ .record_multi_agent_spawn_failure(reason, fork_mode, multi_agent_version);+}+pub(crate) fn collab_agent_error(agent_id: ThreadId, err: CodexErr) -> FunctionCallError {match err.details() {CodexErrorDetails::ThreadNotFound(id) => {diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rsindex 98360657ed..9c8268c542 100644--- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs+++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs@@ -36,6 +36,9 @@ use codex_login::CodexAuth;use codex_model_provider::create_model_provider;use codex_model_provider_info::built_in_model_providers;use codex_models_manager::manager::StaticModelsManager;+use codex_otel::MULTI_AGENT_SPAWN_FAILURE_METRIC;+use codex_otel::MetricsClient;+use codex_otel::MetricsConfig;use codex_protocol::AgentPath;use codex_protocol::ThreadId;use codex_protocol::config_types::ApprovalsReviewer;@@ -73,9 +76,13 @@ use codex_protocol::protocol::TurnCompleteEvent;use codex_protocol::user_input::UserInput;use codex_state::DirectionalThreadSpawnEdgeStatus;use core_test_support::TempDirExt;+use opentelemetry_sdk::metrics::InMemoryMetricExporter;+use opentelemetry_sdk::metrics::data::AggregatedMetrics;+use opentelemetry_sdk::metrics::data::MetricData;use pretty_assertions::assert_eq;use serde::Deserialize;use serde_json::json;+use std::collections::BTreeMap;use std::collections::HashMap;use std::path::PathBuf;use std::sync::Arc;@@ -294,6 +301,74 @@ async fn spawn_agent_rejects_when_message_and_items_are_both_set() {);}+#[tokio::test]+async fn spawn_agent_limit_failure_emits_bounded_metric() {+ let metrics = MetricsClient::new(+ MetricsConfig::in_memory(+ "test",+ "codex-core",+ env!("CARGO_PKG_VERSION"),+ InMemoryMetricExporter::default(),+ )+ .with_runtime_reader(),+ )+ .expect("create in-memory metrics client");+ let (mut session, mut turn) = make_session_and_context().await;+ turn.session_telemetry = turn.session_telemetry.clone().with_metrics(metrics.clone());+ let mut config = (*turn.config).clone();+ config.agent_max_threads = Some(0);+ config.apps_mcp_product_sku = Some("codex".to_string());+ turn.config = Arc::new(config);+ let manager = thread_manager();+ set_agent_control(&mut session, manager.agent_control());++ let Err(err) = SpawnAgentHandler::default()+ .handle(invocation(+ Arc::new(session),+ Arc::new(turn),+ "spawn_agent",+ function_payload(json!({"message": "inspect this repo"})),+ ))+ .await+ else {+ panic!("spawn_agent should respect the thread limit");+ };+ assert_eq!(+ err,+ FunctionCallError::RespondToModel(+ "collab spawn failed: agent thread limit reached".to_string()+ )+ );++ let snapshot = metrics.snapshot().expect("snapshot spawn metrics");+ let failure_metric = snapshot+ .scope_metrics()+ .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)+ .find(|metric| metric.name() == MULTI_AGENT_SPAWN_FAILURE_METRIC)+ .expect("spawn failure metric");+ let AggregatedMetrics::U64(MetricData::Sum(sum)) = failure_metric.data() else {+ panic!("expected spawn failure counter");+ };+ let points = sum.data_points().collect::<Vec<_>>();+ assert_eq!(points.len(), 1);+ assert_eq!(points[0].value(), 1);+ assert_eq!(+ points[0]+ .attributes()+ .map(|attribute| (+ attribute.key.as_str().to_string(),+ attribute.value.as_str().to_string(),+ ))+ .collect::<BTreeMap<_, _>>(),+ BTreeMap::from([+ ("fork_mode".to_string(), "none".to_string()),+ ("multi_agent_version".to_string(), "v1".to_string()),+ ("product_sku".to_string(), "codex".to_string()),+ ("reason".to_string(), "limit_reached".to_string()),+ ])+ );+}+#[tokio::test]async fn spawn_agent_uses_explorer_role_and_preserves_approval_policy() {#[derive(Debug, Deserialize)]diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rsindex 9f4c0c24ff..ff00a422e6 100644--- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs+++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs@@ -194,7 +194,7 @@ async fn handle_spawn_agent(source: spawn_source,options: SpawnAgentOptions {fork_parent_spawn_call_id: fork_mode.as_ref().map(|_| call_id.clone()),- fork_mode,+ fork_mode: fork_mode.clone(),parent_thread_id: Some(session.thread_id),parent_turn_id: Some(turn.sub_id.clone()),root_turn_id: turn.turn_metadata_state.root_turn_id(),@@ -205,7 +205,16 @@ async fn handle_spawn_agent(},}).await- .map_err(collab_spawn_error)?;+ .map_err(|err| {+ record_collab_spawn_failure(+ &turn.session_telemetry,+ turn.config.apps_mcp_product_sku.as_deref(),+ &err,+ fork_mode.as_ref(),+ MultiAgentVersion::V2,+ );+ collab_spawn_error(err)+ })?;let new_thread_id = spawned_agent.thread_id;let agent_status = spawned_agent.status;let nickname = agent_snapshotdiff --git a/codex-rs/otel/src/events/session_telemetry.rs b/codex-rs/otel/src/events/session_telemetry.rsindex 01789e3c3e..33f2c82037 100644--- a/codex-rs/otel/src/events/session_telemetry.rs+++ b/codex-rs/otel/src/events/session_telemetry.rs@@ -5,6 +5,8 @@ use crate::events::shared::log_event;use crate::events::shared::trace_event;use crate::metrics::API_CALL_COUNT_METRIC;use crate::metrics::API_CALL_DURATION_METRIC;+use crate::metrics::MULTI_AGENT_SPAWN_FAILURE_METRIC;+use crate::metrics::MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC;use crate::metrics::MetricsClient;use crate::metrics::MetricsConfig;use crate::metrics::MetricsError;@@ -152,7 +154,7 @@ impl SessionTelemetry {self}- /// Attributes tool telemetry without turning arbitrary configuration into metric labels.+ /// Attributes bounded telemetry without turning arbitrary configuration into metric labels.pub fn with_product_sku(mut self, product_sku: Option<&str>) -> Self {const KNOWN_PRODUCT_SKUS: &[&str] = &["codex"];@@ -261,6 +263,53 @@ impl SessionTelemetry {}}+ /// Records one successful multi-agent spawn phase with bounded spawn dimensions.+ pub fn record_multi_agent_spawn_phase(+ &self,+ phase: &'static str,+ duration: Duration,+ fork_mode: &'static str,+ history_mode: &'static str,+ multi_agent_version: &'static str,+ ) {+ let Some(metrics) = &self.metrics else {+ return;+ };+ let mut tags = self.multi_agent_spawn_tags(fork_mode, multi_agent_version);+ tags.push(("history_mode", history_mode));+ tags.push(("phase", phase));+ let _ = metrics.record_duration(MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC, duration, &tags);+ }++ /// Records one failed multi-agent spawn after request validation has completed.+ pub fn record_multi_agent_spawn_failure(+ &self,+ reason: &'static str,+ fork_mode: &'static str,+ multi_agent_version: &'static str,+ ) {+ let Some(metrics) = &self.metrics else {+ return;+ };+ let mut tags = self.multi_agent_spawn_tags(fork_mode, multi_agent_version);+ tags.push(("reason", reason));+ let _ = metrics.counter(MULTI_AGENT_SPAWN_FAILURE_METRIC, /*inc*/ 1, &tags);+ }++ fn multi_agent_spawn_tags(+ &self,+ fork_mode: &'static str,+ multi_agent_version: &'static str,+ ) -> Vec<(&'static str, &'static str)> {+ let mut tags = Vec::with_capacity(5);+ tags.push(("fork_mode", fork_mode));+ tags.push(("multi_agent_version", multi_agent_version));+ if let Some(product_sku) = self.metadata.product_sku {+ tags.push(("product_sku", product_sku));+ }+ tags+ }+fn record_duration_ms_f64(&self, name: &str, duration_ms: f64, tags: &[(&str, &str)]) {let res: MetricsResult<()> = (|| {let Some(metrics) = &self.metrics else {diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rsindex f27d77a27b..fcc15025dd 100644--- a/codex-rs/otel/src/metrics/names.rs+++ b/codex-rs/otel/src/metrics/names.rs@@ -1,6 +1,9 @@pub const TOOL_CALL_COUNT_METRIC: &str = "codex.tool.call";pub const TOOL_CALL_DURATION_METRIC: &str = "codex.tool.call.duration_ms";pub const TOOL_CALL_UNIFIED_EXEC_METRIC: &str = "codex.tool.unified_exec";+pub const MULTI_AGENT_SPAWN_FAILURE_METRIC: &str = "codex.multi_agent.spawn.failure";+pub const MULTI_AGENT_SPAWN_PHASE_DURATION_METRIC: &str =+ "codex.multi_agent.spawn.phase.duration_ms";pub const ARTIFACT_OPERATION_STARTED_METRIC: &str = "codex.artifact.operation.started";pub const ARTIFACT_OPERATION_EXPECTED_OUTPUT_COUNT_METRIC: &str ="codex.artifact.operation.expected_output_count";