Lightweight coding agent that runs in your terminal
Rust
Record tool registration metrics for each sampling request (#51868)
Alex Kirafbe97c792908
diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rsindex 23e1e071e1..632bb1a294 100644--- a/codex-rs/core/src/session/turn.rs+++ b/codex-rs/core/src/session/turn.rs@@ -56,6 +56,7 @@ use crate::tools::parallel::ToolCallRuntime;use crate::tools::registry::ToolArgumentDiffConsumer;use crate::tools::router::ToolSuggestCandidates;use crate::tools::router::ToolSuggestPresentation;+use crate::tools::router::record_registration_metrics;use crate::tools::spec_plan::build_tool_router;use crate::tools::spec_plan::tool_suggest_enabled;use crate::turn_diff_tracker::TurnDiffTracker;@@ -1631,6 +1632,7 @@ async fn run_sampling_request(Arc::clone(&turn_diff_tracker),);let max_retries = turn_context.provider.info().stream_max_retries();+ record_registration_metrics(&step_context.tool_router, &step_context.session_telemetry);let mut retry_state = ResponsesStreamRetryState::default();let mut initial_input = Some(input);let mut original_input = None;diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rsindex baf39f082a..8f8635dcfa 100644--- a/codex-rs/core/src/tools/router.rs+++ b/codex-rs/core/src/tools/router.rs@@ -13,9 +13,13 @@ use crate::tools::handlers::ToolSearchHandlerCache;use crate::tools::registry::AnyToolResult;use crate::tools::registry::CoreToolRuntime;use crate::tools::registry::ToolArgumentDiffConsumer;+use crate::tools::registry::ToolExposure;use crate::tools::registry::ToolRegistry;#[cfg(test)]use crate::tools::spec_plan::finalize_tool_router;+use codex_otel::SessionTelemetry;+use codex_otel::TOOL_REGISTRATIONS_METRIC;+use codex_otel::TOOL_REGISTRATIONS_METRIC_BUCKETS;use codex_protocol::models::ResponseItem;use codex_protocol::models::SearchToolCallParams;#[cfg(test)]@@ -33,6 +37,13 @@ use tracing::instrument;pub use crate::tools::context::ToolCallSource;+const EXPOSURE_DIRECT: &str = "direct";+const EXPOSURE_DIRECT_MODEL_ONLY: &str = "direct_model_only";+const EXPOSURE_DEFERRED: &str = "deferred";+const EXPOSURE_DEFERRED_MODEL_ONLY: &str = "deferred_model_only";+const EXPOSURE_CODE_MODE_ONLY: &str = "code_mode_only";+const EXPOSURE_HIDDEN: &str = "hidden";+#[derive(Clone, Debug, PartialEq)]pub struct ToolCall {pub tool_name: ToolName,@@ -382,6 +393,48 @@ impl ToolRouter {}}+/// Records the registered inventory once per sampling request, before transport retries.+/// Registrations do not imply model visibility or reachability; hosted tools are excluded.+pub(crate) fn record_registration_metrics(router: &ToolRouter, telemetry: &SessionTelemetry) {+ let mut direct = 0usize;+ let mut direct_model_only = 0usize;+ let mut deferred = 0usize;+ let mut deferred_model_only = 0usize;+ let mut code_mode_only = 0usize;+ let mut hidden = 0usize;+ for tool in router.registry.entries() {+ match tool.exposure {+ ToolExposure::Direct => direct += 1,+ ToolExposure::DirectModelOnly => direct_model_only += 1,+ ToolExposure::Deferred => deferred += 1,+ ToolExposure::DeferredModelOnly => deferred_model_only += 1,+ ToolExposure::CodeModeOnly => code_mode_only += 1,+ ToolExposure::Hidden => hidden += 1,+ }+ }++ let tool_mode = match router.tool_mode {+ ToolMode::Direct => "direct",+ ToolMode::CodeMode => "code_mode",+ ToolMode::CodeModeOnly => "code_mode_only",+ };+ for (exposure, count) in [+ (EXPOSURE_DIRECT, direct),+ (EXPOSURE_DIRECT_MODEL_ONLY, direct_model_only),+ (EXPOSURE_DEFERRED, deferred),+ (EXPOSURE_DEFERRED_MODEL_ONLY, deferred_model_only),+ (EXPOSURE_CODE_MODE_ONLY, code_mode_only),+ (EXPOSURE_HIDDEN, hidden),+ ] {+ telemetry.histogram_with_boundaries(+ TOOL_REGISTRATIONS_METRIC,+ i64::try_from(count).unwrap_or(i64::MAX),+ TOOL_REGISTRATIONS_METRIC_BUCKETS,+ &[("exposure", exposure), ("tool_mode", tool_mode)],+ );+ }+}+#[cfg(test)]#[path = "router_tests.rs"]mod tests;diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rsindex 159e505be7..d64c3a3ed2 100644--- a/codex-rs/core/tests/suite/mod.rs+++ b/codex-rs/core/tests/suite/mod.rs@@ -223,6 +223,7 @@ mod token_usage_rollout;mod tool_harness;mod tool_lifecycle;mod tool_parallelism;+mod tool_registration_metrics;mod tools;mod truncation;#[path = "turn_error_details_tests.rs"]diff --git a/codex-rs/core/tests/suite/tool_registration_metrics.rs b/codex-rs/core/tests/suite/tool_registration_metrics.rsnew file mode 100644index 0000000000..a6751df289--- /dev/null+++ b/codex-rs/core/tests/suite/tool_registration_metrics.rs@@ -0,0 +1,202 @@+//! Verifies registration telemetry on real sampling requests, including stream retries.++use anyhow::Result;+use codex_core::StartThreadOptions;+use codex_core::TurnInputRequest;+use codex_features::Feature;+use codex_otel::MetricsClient;+use codex_otel::MetricsConfig;+use codex_otel::TOOL_REGISTRATIONS_METRIC;+use codex_protocol::config_types::WebSearchMode;+use codex_protocol::dynamic_tools::DynamicToolFunctionSpec;+use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec;+use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;+use codex_protocol::dynamic_tools::DynamicToolSpec;+use codex_protocol::openai_models::ToolMode;+use codex_protocol::protocol::EventMsg;+use codex_protocol::user_input::UserInput;+use core_test_support::responses;+use core_test_support::responses::namespace_child_tool;+use core_test_support::skip_if_no_network;+use core_test_support::test_codex::test_codex;+use core_test_support::wait_for_event;+use opentelemetry_sdk::metrics::InMemoryMetricExporter;+use opentelemetry_sdk::metrics::data::AggregatedMetrics;+use opentelemetry_sdk::metrics::data::MetricData;+use opentelemetry_sdk::metrics::data::ScopeMetrics;+use pretty_assertions::assert_eq;+use serde_json::Value;+use serde_json::json;+use std::collections::BTreeMap;++/// Transport retries must not inflate registrations or include hosted tools in the inventory.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn sampling_records_tool_registrations_once_across_stream_retry() -> Result<()> {+ skip_if_no_network!(Ok(()));+ const MODEL: &str = "tool-registrations-model";+ let server = responses::start_mock_server().await;+ let response = responses::mount_sse_sequence(+ &server,+ vec![+ responses::sse(vec![responses::ev_response_created("incomplete")]),+ responses::sse(vec![+ responses::ev_response_created("complete"),+ responses::ev_completed("complete"),+ ]),+ ],+ )+ .await;+ let metrics = MetricsClient::new(+ MetricsConfig::in_memory(+ "test",+ "tool-registrations",+ env!("CARGO_PKG_VERSION"),+ InMemoryMetricExporter::default(),+ )+ .with_runtime_reader(),+ )?;+ let test = test_codex()+ .with_model(MODEL)+ .with_model_info_override(MODEL, |model| {+ model.supports_search_tool = true;+ model.use_responses_lite = false;+ model.tool_mode = Some(ToolMode::Direct);+ model.experimental_supported_tools.clear();+ })+ .with_config(|config| {+ // Keep request_user_input as the only model-only registration.+ for feature in [+ Feature::Collab,+ Feature::SleepTool,+ Feature::TokenBudget,+ Feature::SendMessageToUserAsync,+ ] {+ config.features.disable(feature).expect("disable feature");+ }+ config.experimental_request_user_input_enabled = true;+ config+ .web_search_mode+ .set(WebSearchMode::Cached)+ .expect("enable hosted web search");+ config.model_provider.request_max_retries = Some(0);+ config.model_provider.stream_max_retries = Some(1);+ })+ .build_with_auto_env(&server)+ .await?;+ let input_schema = json!({+ "type": "object",+ "properties": {},+ "additionalProperties": false,+ });+ let mut options = StartThreadOptions {+ dynamic_tools: vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {+ name: "workspace_tools".to_string(),+ description: "Tools for this workspace.".to_string(),+ tools: [("status", false), ("search", true)]+ .into_iter()+ .map(|(name, defer_loading)| {+ DynamicToolNamespaceTool::Function(DynamicToolFunctionSpec {+ name: name.to_string(),+ description: format!("Workspace {name}."),+ input_schema: input_schema.clone(),+ defer_loading,+ })+ })+ .collect(),+ })],+ ..StartThreadOptions::new(test.config.clone())+ };+ options.thread_extension_init.insert(metrics.clone());+ let thread = test.thread_manager.start_thread(options).await?.thread;+ thread+ .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {+ text: "Describe the available workspace tools.".to_string(),+ text_elements: Vec::new(),+ }]))+ .await?;+ let EventMsg::TurnComplete(completed) =+ wait_for_event(&thread, |event| matches!(event, EventMsg::TurnComplete(_))).await+ else {+ unreachable!("event predicate guarantees turn completion");+ };+ assert_eq!(completed.error, None);++ let requests = response.requests();+ assert_eq!(requests.len(), 2, "incomplete stream must retry");+ let body = requests[0].body_json();+ assert_eq!(requests[1].body_json()["tools"], body["tools"]);+ assert!(namespace_child_tool(&body, "workspace_tools", "status").is_some());+ assert_eq!(+ namespace_child_tool(&body, "workspace_tools", "search"),+ None+ );+ let tools = body["tools"].as_array().expect("request tools");+ assert!(tools.iter().any(|tool| tool["type"] == "web_search"));+ assert!(+ tools+ .iter()+ .any(|tool| tool["name"] == "request_user_input")+ );+ // The direct-mode request exposes every direct registration, plus the one+ // model-only registration and hosted web search. Namespace children count separately.+ let direct_count = tools+ .iter()+ .filter(|tool| tool["type"] != "web_search")+ .map(|tool| {+ tool.get("tools")+ .and_then(Value::as_array)+ .map_or(1, Vec::len)+ })+ .sum::<usize>()+ - 1;++ let snapshot = metrics.snapshot()?;+ let mut samples = BTreeMap::new();+ for metric in snapshot.scope_metrics().flat_map(ScopeMetrics::metrics) {+ if metric.name() != TOOL_REGISTRATIONS_METRIC {+ continue;+ }+ let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {+ panic!("tool registration metrics must be histograms");+ };+ for point in histogram.data_points() {+ let attributes = point+ .attributes()+ .map(|attribute| (attribute.key.as_str(), attribute.value.as_str().to_string()))+ .collect::<BTreeMap<_, _>>();+ assert_eq!(attributes.get("model").map(String::as_str), Some(MODEL));+ assert_eq!(attributes["tool_mode"], "direct");+ assert!(+ attributes.keys().all(|key| matches!(+ *key,+ "auth_mode"+ | "session_source"+ | "originator"+ | "service_name"+ | "model"+ | "app.version"+ | "exposure"+ | "tool_mode"+ )),+ "registration telemetry must not label tool names or descriptions"+ );+ assert_eq!(+ samples.insert(attributes["exposure"].clone(), (point.count(), point.sum()),),+ None,+ );+ }+ }+ assert_eq!(+ samples,+ BTreeMap::from([+ ("direct".to_string(), (1, direct_count as f64)),+ ("direct_model_only".to_string(), (1, 1.0)),+ ("deferred".to_string(), (1, 1.0)),+ ("deferred_model_only".to_string(), (1, 0.0)),+ ("code_mode_only".to_string(), (1, 0.0)),+ ("hidden".to_string(), (1, 0.0)),+ ])+ );+ metrics.shutdown()?;+ Ok(())+}diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rsindex e93d19ed85..d80948bb1b 100644--- a/codex-rs/otel/src/metrics/names.rs+++ b/codex-rs/otel/src/metrics/names.rs@@ -77,6 +77,13 @@ pub const THREAD_SKILLS_TRUNCATED_METRIC: &str = "codex.thread.skills.truncated"pub const THREAD_TOOLS_NAMESPACES_TOTAL_METRIC: &str = "codex.thread.tools.namespaces_total";pub const THREAD_TOOLS_FRAGMENT_BYTES_METRIC: &str = "codex.thread.tools.fragment_bytes";+/// Finalized tool registrations per sampling request, tagged by exposure and tool_mode.+pub const TOOL_REGISTRATIONS_METRIC: &str = "codex.tools.registered";+pub const TOOL_REGISTRATIONS_METRIC_BUCKETS: &[f64] = &[+ 0.0, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 50.0, 75.0, 100.0, 150.0, 200.0, 300.0, 500.0, 750.0,+ 1_000.0, 2_000.0, 5_000.0, 10_000.0, 20_000.0,+];+/// Logarithmic boundaries up to 32,768 for tools bytes and namespace counts.pub static THREAD_TOOLS_METRIC_BUCKETS: LazyLock<[f64; 511]> =LazyLock::new(|| context_log_buckets(/*max_exponent*/ 15.0));