Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Record tool registration metrics for each sampling request (#51868)

Alex Kirafbe97c792908
diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs
index 23e1e071e1..632bb1a294 100644
--- a/codex-rs/core/src/session/turn.rs
+++ b/codex-rs/core/src/session/turn.rs
@@ -56,6 +56,7 @@ use crate::tools::parallel::ToolCallRuntime;
use crate::tools::registry::ToolArgumentDiffConsumer;
use crate::tools::router::ToolSuggestCandidates;
use crate::tools::router::ToolSuggestPresentation;
+use crate::tools::router::record_registration_metrics;
use crate::tools::spec_plan::build_tool_router;
use crate::tools::spec_plan::tool_suggest_enabled;
use crate::turn_diff_tracker::TurnDiffTracker;
@@ -1631,6 +1632,7 @@ async fn run_sampling_request(
Arc::clone(&turn_diff_tracker),
);
let max_retries = turn_context.provider.info().stream_max_retries();
+ record_registration_metrics(&step_context.tool_router, &step_context.session_telemetry);
let mut retry_state = ResponsesStreamRetryState::default();
let mut initial_input = Some(input);
let mut original_input = None;
diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs
index baf39f082a..8f8635dcfa 100644
--- a/codex-rs/core/src/tools/router.rs
+++ b/codex-rs/core/src/tools/router.rs
@@ -13,9 +13,13 @@ use crate::tools::handlers::ToolSearchHandlerCache;
use crate::tools::registry::AnyToolResult;
use crate::tools::registry::CoreToolRuntime;
use crate::tools::registry::ToolArgumentDiffConsumer;
+use crate::tools::registry::ToolExposure;
use crate::tools::registry::ToolRegistry;
#[cfg(test)]
use crate::tools::spec_plan::finalize_tool_router;
+use codex_otel::SessionTelemetry;
+use codex_otel::TOOL_REGISTRATIONS_METRIC;
+use codex_otel::TOOL_REGISTRATIONS_METRIC_BUCKETS;
use codex_protocol::models::ResponseItem;
use codex_protocol::models::SearchToolCallParams;
#[cfg(test)]
@@ -33,6 +37,13 @@ use tracing::instrument;
pub use crate::tools::context::ToolCallSource;
+const EXPOSURE_DIRECT: &str = "direct";
+const EXPOSURE_DIRECT_MODEL_ONLY: &str = "direct_model_only";
+const EXPOSURE_DEFERRED: &str = "deferred";
+const EXPOSURE_DEFERRED_MODEL_ONLY: &str = "deferred_model_only";
+const EXPOSURE_CODE_MODE_ONLY: &str = "code_mode_only";
+const EXPOSURE_HIDDEN: &str = "hidden";
+
#[derive(Clone, Debug, PartialEq)]
pub struct ToolCall {
pub tool_name: ToolName,
@@ -382,6 +393,48 @@ impl ToolRouter {
}
}
+/// Records the registered inventory once per sampling request, before transport retries.
+/// Registrations do not imply model visibility or reachability; hosted tools are excluded.
+pub(crate) fn record_registration_metrics(router: &ToolRouter, telemetry: &SessionTelemetry) {
+ let mut direct = 0usize;
+ let mut direct_model_only = 0usize;
+ let mut deferred = 0usize;
+ let mut deferred_model_only = 0usize;
+ let mut code_mode_only = 0usize;
+ let mut hidden = 0usize;
+ for tool in router.registry.entries() {
+ match tool.exposure {
+ ToolExposure::Direct => direct += 1,
+ ToolExposure::DirectModelOnly => direct_model_only += 1,
+ ToolExposure::Deferred => deferred += 1,
+ ToolExposure::DeferredModelOnly => deferred_model_only += 1,
+ ToolExposure::CodeModeOnly => code_mode_only += 1,
+ ToolExposure::Hidden => hidden += 1,
+ }
+ }
+
+ let tool_mode = match router.tool_mode {
+ ToolMode::Direct => "direct",
+ ToolMode::CodeMode => "code_mode",
+ ToolMode::CodeModeOnly => "code_mode_only",
+ };
+ for (exposure, count) in [
+ (EXPOSURE_DIRECT, direct),
+ (EXPOSURE_DIRECT_MODEL_ONLY, direct_model_only),
+ (EXPOSURE_DEFERRED, deferred),
+ (EXPOSURE_DEFERRED_MODEL_ONLY, deferred_model_only),
+ (EXPOSURE_CODE_MODE_ONLY, code_mode_only),
+ (EXPOSURE_HIDDEN, hidden),
+ ] {
+ telemetry.histogram_with_boundaries(
+ TOOL_REGISTRATIONS_METRIC,
+ i64::try_from(count).unwrap_or(i64::MAX),
+ TOOL_REGISTRATIONS_METRIC_BUCKETS,
+ &[("exposure", exposure), ("tool_mode", tool_mode)],
+ );
+ }
+}
+
#[cfg(test)]
#[path = "router_tests.rs"]
mod tests;
diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs
index 159e505be7..d64c3a3ed2 100644
--- a/codex-rs/core/tests/suite/mod.rs
+++ b/codex-rs/core/tests/suite/mod.rs
@@ -223,6 +223,7 @@ mod token_usage_rollout;
mod tool_harness;
mod tool_lifecycle;
mod tool_parallelism;
+mod tool_registration_metrics;
mod tools;
mod truncation;
#[path = "turn_error_details_tests.rs"]
diff --git a/codex-rs/core/tests/suite/tool_registration_metrics.rs b/codex-rs/core/tests/suite/tool_registration_metrics.rs
new file mode 100644
index 0000000000..a6751df289
--- /dev/null
+++ b/codex-rs/core/tests/suite/tool_registration_metrics.rs
@@ -0,0 +1,202 @@
+//! Verifies registration telemetry on real sampling requests, including stream retries.
+
+use anyhow::Result;
+use codex_core::StartThreadOptions;
+use codex_core::TurnInputRequest;
+use codex_features::Feature;
+use codex_otel::MetricsClient;
+use codex_otel::MetricsConfig;
+use codex_otel::TOOL_REGISTRATIONS_METRIC;
+use codex_protocol::config_types::WebSearchMode;
+use codex_protocol::dynamic_tools::DynamicToolFunctionSpec;
+use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec;
+use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;
+use codex_protocol::dynamic_tools::DynamicToolSpec;
+use codex_protocol::openai_models::ToolMode;
+use codex_protocol::protocol::EventMsg;
+use codex_protocol::user_input::UserInput;
+use core_test_support::responses;
+use core_test_support::responses::namespace_child_tool;
+use core_test_support::skip_if_no_network;
+use core_test_support::test_codex::test_codex;
+use core_test_support::wait_for_event;
+use opentelemetry_sdk::metrics::InMemoryMetricExporter;
+use opentelemetry_sdk::metrics::data::AggregatedMetrics;
+use opentelemetry_sdk::metrics::data::MetricData;
+use opentelemetry_sdk::metrics::data::ScopeMetrics;
+use pretty_assertions::assert_eq;
+use serde_json::Value;
+use serde_json::json;
+use std::collections::BTreeMap;
+
+/// Transport retries must not inflate registrations or include hosted tools in the inventory.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn sampling_records_tool_registrations_once_across_stream_retry() -> Result<()> {
+ skip_if_no_network!(Ok(()));
+ const MODEL: &str = "tool-registrations-model";
+ let server = responses::start_mock_server().await;
+ let response = responses::mount_sse_sequence(
+ &server,
+ vec![
+ responses::sse(vec![responses::ev_response_created("incomplete")]),
+ responses::sse(vec![
+ responses::ev_response_created("complete"),
+ responses::ev_completed("complete"),
+ ]),
+ ],
+ )
+ .await;
+ let metrics = MetricsClient::new(
+ MetricsConfig::in_memory(
+ "test",
+ "tool-registrations",
+ env!("CARGO_PKG_VERSION"),
+ InMemoryMetricExporter::default(),
+ )
+ .with_runtime_reader(),
+ )?;
+ let test = test_codex()
+ .with_model(MODEL)
+ .with_model_info_override(MODEL, |model| {
+ model.supports_search_tool = true;
+ model.use_responses_lite = false;
+ model.tool_mode = Some(ToolMode::Direct);
+ model.experimental_supported_tools.clear();
+ })
+ .with_config(|config| {
+ // Keep request_user_input as the only model-only registration.
+ for feature in [
+ Feature::Collab,
+ Feature::SleepTool,
+ Feature::TokenBudget,
+ Feature::SendMessageToUserAsync,
+ ] {
+ config.features.disable(feature).expect("disable feature");
+ }
+ config.experimental_request_user_input_enabled = true;
+ config
+ .web_search_mode
+ .set(WebSearchMode::Cached)
+ .expect("enable hosted web search");
+ config.model_provider.request_max_retries = Some(0);
+ config.model_provider.stream_max_retries = Some(1);
+ })
+ .build_with_auto_env(&server)
+ .await?;
+ let input_schema = json!({
+ "type": "object",
+ "properties": {},
+ "additionalProperties": false,
+ });
+ let mut options = StartThreadOptions {
+ dynamic_tools: vec![DynamicToolSpec::Namespace(DynamicToolNamespaceSpec {
+ name: "workspace_tools".to_string(),
+ description: "Tools for this workspace.".to_string(),
+ tools: [("status", false), ("search", true)]
+ .into_iter()
+ .map(|(name, defer_loading)| {
+ DynamicToolNamespaceTool::Function(DynamicToolFunctionSpec {
+ name: name.to_string(),
+ description: format!("Workspace {name}."),
+ input_schema: input_schema.clone(),
+ defer_loading,
+ })
+ })
+ .collect(),
+ })],
+ ..StartThreadOptions::new(test.config.clone())
+ };
+ options.thread_extension_init.insert(metrics.clone());
+ let thread = test.thread_manager.start_thread(options).await?.thread;
+ thread
+ .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
+ text: "Describe the available workspace tools.".to_string(),
+ text_elements: Vec::new(),
+ }]))
+ .await?;
+ let EventMsg::TurnComplete(completed) =
+ wait_for_event(&thread, |event| matches!(event, EventMsg::TurnComplete(_))).await
+ else {
+ unreachable!("event predicate guarantees turn completion");
+ };
+ assert_eq!(completed.error, None);
+
+ let requests = response.requests();
+ assert_eq!(requests.len(), 2, "incomplete stream must retry");
+ let body = requests[0].body_json();
+ assert_eq!(requests[1].body_json()["tools"], body["tools"]);
+ assert!(namespace_child_tool(&body, "workspace_tools", "status").is_some());
+ assert_eq!(
+ namespace_child_tool(&body, "workspace_tools", "search"),
+ None
+ );
+ let tools = body["tools"].as_array().expect("request tools");
+ assert!(tools.iter().any(|tool| tool["type"] == "web_search"));
+ assert!(
+ tools
+ .iter()
+ .any(|tool| tool["name"] == "request_user_input")
+ );
+ // The direct-mode request exposes every direct registration, plus the one
+ // model-only registration and hosted web search. Namespace children count separately.
+ let direct_count = tools
+ .iter()
+ .filter(|tool| tool["type"] != "web_search")
+ .map(|tool| {
+ tool.get("tools")
+ .and_then(Value::as_array)
+ .map_or(1, Vec::len)
+ })
+ .sum::<usize>()
+ - 1;
+
+ let snapshot = metrics.snapshot()?;
+ let mut samples = BTreeMap::new();
+ for metric in snapshot.scope_metrics().flat_map(ScopeMetrics::metrics) {
+ if metric.name() != TOOL_REGISTRATIONS_METRIC {
+ continue;
+ }
+ let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
+ panic!("tool registration metrics must be histograms");
+ };
+ for point in histogram.data_points() {
+ let attributes = point
+ .attributes()
+ .map(|attribute| (attribute.key.as_str(), attribute.value.as_str().to_string()))
+ .collect::<BTreeMap<_, _>>();
+ assert_eq!(attributes.get("model").map(String::as_str), Some(MODEL));
+ assert_eq!(attributes["tool_mode"], "direct");
+ assert!(
+ attributes.keys().all(|key| matches!(
+ *key,
+ "auth_mode"
+ | "session_source"
+ | "originator"
+ | "service_name"
+ | "model"
+ | "app.version"
+ | "exposure"
+ | "tool_mode"
+ )),
+ "registration telemetry must not label tool names or descriptions"
+ );
+ assert_eq!(
+ samples.insert(attributes["exposure"].clone(), (point.count(), point.sum()),),
+ None,
+ );
+ }
+ }
+ assert_eq!(
+ samples,
+ BTreeMap::from([
+ ("direct".to_string(), (1, direct_count as f64)),
+ ("direct_model_only".to_string(), (1, 1.0)),
+ ("deferred".to_string(), (1, 1.0)),
+ ("deferred_model_only".to_string(), (1, 0.0)),
+ ("code_mode_only".to_string(), (1, 0.0)),
+ ("hidden".to_string(), (1, 0.0)),
+ ])
+ );
+ metrics.shutdown()?;
+ Ok(())
+}
diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rs
index e93d19ed85..d80948bb1b 100644
--- a/codex-rs/otel/src/metrics/names.rs
+++ b/codex-rs/otel/src/metrics/names.rs
@@ -77,6 +77,13 @@ pub const THREAD_SKILLS_TRUNCATED_METRIC: &str = "codex.thread.skills.truncated"
pub const THREAD_TOOLS_NAMESPACES_TOTAL_METRIC: &str = "codex.thread.tools.namespaces_total";
pub const THREAD_TOOLS_FRAGMENT_BYTES_METRIC: &str = "codex.thread.tools.fragment_bytes";
+/// Finalized tool registrations per sampling request, tagged by exposure and tool_mode.
+pub const TOOL_REGISTRATIONS_METRIC: &str = "codex.tools.registered";
+pub const TOOL_REGISTRATIONS_METRIC_BUCKETS: &[f64] = &[
+ 0.0, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 50.0, 75.0, 100.0, 150.0, 200.0, 300.0, 500.0, 750.0,
+ 1_000.0, 2_000.0, 5_000.0, 10_000.0, 20_000.0,
+];
+
/// Logarithmic boundaries up to 32,768 for tools bytes and namespace counts.
pub static THREAD_TOOLS_METRIC_BUCKETS: LazyLock<[f64; 511]> =
LazyLock::new(|| context_log_buckets(/*max_exponent*/ 15.0));