Lightweight coding agent that runs in your terminal
Rust
Record metrics for incremental tool updates (#51893)
Alex Kira70efa82b09ed
diff --git a/codex-rs/core/src/context/world_state/top_level_tools.rs b/codex-rs/core/src/context/world_state/top_level_tools.rsindex 5fc9b5b6f3..cc07cef8f5 100644--- a/codex-rs/core/src/context/world_state/top_level_tools.rs+++ b/codex-rs/core/src/context/world_state/top_level_tools.rs@@ -2,6 +2,7 @@//! Missing state starts a fresh catalog; this section does not migrate legacy history.//! Incremental hints decorate emitted declarations only, leaving catalog hashes unchanged.//! Whole namespace removals subsume their members in the removal notice.+//! Metrics count rendered tool redefinitions, excluding namespace headers and initial catalogs.use super::PreviousSectionState;use super::SectionTransition;@@ -10,6 +11,8 @@ use super::WorldStateSection;use super::WorldStateUpdate;use crate::context::ContextualUserFragment;use crate::context::DeveloperInstructions;+use codex_extension_api::ExtensionMetrics;+use codex_otel::TOOL_INCREMENTAL_UPDATES_METRIC;use codex_protocol::error::CodexErrorDetails;use codex_protocol::error::Result;use codex_protocol::models::ContentItemKind;@@ -17,6 +20,11 @@ use codex_protocol::models::ResponseItem;use serde_json::Value;use std::collections::BTreeMap;use std::collections::BTreeSet;+use std::sync::Arc;++const ACTION_ADDED: &str = "added";+const ACTION_REMOVED: &str = "removed";+const ACTION_SCHEMA_CHANGED: &str = "schema_changed";const NAMESPACE_UPDATE_HINT: &str = "This is an incremental namespace update. Previously declared tools remain available for direct calls unless explicitly marked unavailable. If a tool is redefined here, its latest definition replaces the earlier one.";const REMOVED_TOOLS_HEADER: &str = "The following tools are no longer available. Do not call them:";@@ -26,6 +34,7 @@ const REMOVED_NAMESPACES_HEADER: &str = "The following namespaces are no longerpub(crate) struct TopLevelToolsState {definitions: Vec<Value>,hashes: BTreeMap<String, WorldStateHash>,+ metrics: Option<Arc<dyn ExtensionMetrics>>,}impl TopLevelToolsState {@@ -33,7 +42,10 @@ impl TopLevelToolsState {clippy::expect_used,reason = "the Responses Lite serializer emits objects with array-valued namespace members")]- pub(crate) fn new(definitions: Vec<Value>) -> Result<Self> {+ pub(crate) fn new(+ definitions: Vec<Value>,+ metrics: Option<Arc<dyn ExtensionMetrics>>,+ ) -> Result<Self> {let mut hashes = BTreeMap::new();let mut insert_hash = |name: String, definition: &Value| -> Result<()> {let hash = WorldStateHash::from_json(definition);@@ -59,6 +71,7 @@ impl TopLevelToolsState {Ok(Self {definitions,hashes,+ metrics,})}}@@ -75,6 +88,50 @@ fn definition_name(definition: &Value) -> &str {.unwrap_or_else(|| definition["type"].as_str().expect("serialized tool type"))}+/// Accumulates callable changes during rendering without retaining tool definitions.+#[derive(Default)]+struct ToolUpdateMetrics {+ count_added: usize,+ count_removed: usize,+ count_schema_changed: usize,+}++impl ToolUpdateMetrics {+ fn count_changed_tool(+ &mut self,+ name: &str,+ previous: Option<&BTreeMap<String, WorldStateHash>>,+ ) {+ if let Some(previous) = previous {+ if previous.contains_key(name) {+ // The existing hash covers the entire definition, including descriptions.+ self.count_schema_changed += 1;+ } else {+ self.count_added += 1;+ }+ }+ }++ fn record(self, metrics: Option<&dyn ExtensionMetrics>) {+ let Some(metrics) = metrics else {+ return;+ };+ for (action, count) in [+ (ACTION_ADDED, self.count_added),+ (ACTION_REMOVED, self.count_removed),+ (ACTION_SCHEMA_CHANGED, self.count_schema_changed),+ ] {+ if count > 0 {+ metrics.counter(+ TOOL_INCREMENTAL_UPDATES_METRIC,+ i64::try_from(count).unwrap_or(i64::MAX),+ &[("action", action)],+ );+ }+ }+ }+}+impl WorldStateSection for TopLevelToolsState {const ID: &'static str = "top_level_tools";type Snapshot = BTreeMap<String, WorldStateHash>;@@ -89,24 +146,29 @@ impl WorldStateSection for TopLevelToolsState {};let changed =|name: &str| previous.and_then(|previous| previous.get(name)) != self.hashes.get(name);+ let mut counts = ToolUpdateMetrics::default();let mut namespace_updates = Vec::new();let mut tools = Vec::new();for definition in &self.definitions {let name = definition_name(definition);let Some(members) = definition["tools"].as_array() else {if changed(name) {+ counts.count_changed_tool(name, previous);tools.push(definition.clone());}continue;};- let members = members- .iter()- .filter(|tool| changed(&format!("{name}.{}", definition_name(tool))))- .cloned()- .collect::<Vec<_>>();- if !members.is_empty() {+ let mut changed_members = Vec::new();+ for tool in members {+ let tool_name = format!("{name}.{}", definition_name(tool));+ if changed(&tool_name) {+ counts.count_changed_tool(&tool_name, previous);+ changed_members.push(tool.clone());+ }+ }+ if !changed_members.is_empty() {let mut namespace = definition.clone();- namespace["tools"] = Value::Array(members);+ namespace["tools"] = Value::Array(changed_members);if previous.is_some_and(|previous| previous.contains_key(name)) {let description = definition["description"].as_str().unwrap_or_default();namespace["description"] = Value::String(if description.is_empty() {@@ -153,11 +215,14 @@ impl WorldStateSection for TopLevelToolsState {let tools = previous.keys().filter(|key| !self.hashes.contains_key(*key))+ .filter(|key| !namespaces.contains(key.as_str()))+ .inspect(|_| {+ // Count members even when the notice collapses their removed namespace.+ counts.count_removed += 1;+ }).filter(|key| {- !namespaces.contains(key.as_str())- && !key- .split_once('.')- .is_some_and(|(namespace, _)| namespaces.contains(namespace))+ !key.split_once('.')+ .is_some_and(|(namespace, _)| namespaces.contains(namespace))}).cloned().collect::<Vec<_>>();@@ -171,6 +236,7 @@ impl WorldStateSection for TopLevelToolsState {);}}+ counts.record(self.metrics.as_deref());// Persist the empty map too: it is a known empty catalog, not missing state.(Some(self.hashes.clone()), updates)}diff --git a/codex-rs/core/src/context/world_state/top_level_tools_tests.rs b/codex-rs/core/src/context/world_state/top_level_tools_tests.rsindex dd15408619..a072b039af 100644--- a/codex-rs/core/src/context/world_state/top_level_tools_tests.rs+++ b/codex-rs/core/src/context/world_state/top_level_tools_tests.rs@@ -3,6 +3,16 @@use super::*;use crate::context::world_state::test_support::render_section_cases;use crate::context_manager::updates::merge_world_state_updates;+use crate::session::extension_metrics::from_session_telemetry;+use codex_otel::MetricsClient;+use codex_otel::MetricsConfig;+use codex_otel::SessionTelemetry;+use codex_protocol::ThreadId;+use codex_protocol::protocol::SessionSource;+use opentelemetry_sdk::metrics::InMemoryMetricExporter;+use opentelemetry_sdk::metrics::data::AggregatedMetrics;+use opentelemetry_sdk::metrics::data::MetricData;+use opentelemetry_sdk::metrics::data::ScopeMetrics;use pretty_assertions::assert_eq;use serde_json::json;@@ -32,12 +42,12 @@ fn snapshots_catalog_transitions() {"description": "Search tools.","parameters": {"type": "object", "properties": {}}});let web = json!({"type": "web_search", "external_web_access": true});- let original = TopLevelToolsState::new(vec![search.clone()]).unwrap();+ let original = TopLevelToolsState::new(vec![search.clone()], /*metrics*/ None).unwrap();let mut changed = search;changed["parameters"]["properties"] = json!({"query": {"type": "string"}});- let updated = TopLevelToolsState::new(vec![changed, web.clone()]).unwrap();- let only_web = TopLevelToolsState::new(vec![web]).unwrap();- let empty = TopLevelToolsState::new(Vec::new()).unwrap();+ let updated = TopLevelToolsState::new(vec![changed, web.clone()], /*metrics*/ None).unwrap();+ let only_web = TopLevelToolsState::new(vec![web], /*metrics*/ None).unwrap();+ let empty = TopLevelToolsState::new(Vec::new(), /*metrics*/ None).unwrap();insta::assert_snapshot!(render_section_cases(&[(Absent, Known(&original)),@@ -49,43 +59,82 @@ fn snapshots_catalog_transitions() {]));}-#[test]-fn snapshots_namespace_transitions() {+#[tokio::test]+async fn snapshots_namespace_transitions() {use PreviousSectionState::Absent;use PreviousSectionState::Known;- let original = TopLevelToolsState::new(vec![- namespace(- "functions",- vec![- declaration("unchanged"),- declaration("removed"),- declaration("changed"),- ],- ),- namespace("other", vec![declaration("lookup")]),- ])+ let metrics = MetricsClient::new(+ MetricsConfig::in_memory(+ "test",+ "tool-updates",+ env!("CARGO_PKG_VERSION"),+ InMemoryMetricExporter::default(),+ )+ .with_runtime_reader(),+ )+ .unwrap();+ let telemetry = from_session_telemetry(+ SessionTelemetry::new(+ ThreadId::new(),+ "test-model",+ "test-model",+ /*account_id*/ None,+ /*account_email*/ None,+ /*auth_mode*/ None,+ "test".to_string(),+ /*log_user_prompts*/ false,+ "test".to_string(),+ SessionSource::Cli,+ )+ .with_metrics_without_metadata_tags(metrics.clone()),+ );+ let original = TopLevelToolsState::new(+ vec![+ namespace(+ "functions",+ vec![+ declaration("unchanged"),+ declaration("removed"),+ declaration("changed"),+ ],+ ),+ namespace("other", vec![declaration("lookup")]),+ ],+ Some(Arc::clone(&telemetry)),+ ).unwrap();let mut changed = declaration("changed");changed["parameters"]["properties"] = json!({"query": {"type": "string"}});- let updated = TopLevelToolsState::new(vec![- namespace(- "functions",- vec![declaration("unchanged"), changed, declaration("added")],- ),- namespace("other", vec![declaration("lookup")]),- ])+ let updated = TopLevelToolsState::new(+ vec![+ namespace(+ "functions",+ vec![declaration("unchanged"), changed, declaration("added")],+ ),+ namespace("other", vec![declaration("lookup")]),+ ],+ Some(Arc::clone(&telemetry)),+ )+ .unwrap();+ let only_other = TopLevelToolsState::new(+ vec![namespace("other", vec![declaration("lookup")])],+ Some(Arc::clone(&telemetry)),+ )+ .unwrap();+ let only_functions = TopLevelToolsState::new(+ vec![namespace("functions", vec![declaration("unchanged")])],+ Some(Arc::clone(&telemetry)),+ ).unwrap();- let only_other =- TopLevelToolsState::new(vec![namespace("other", vec![declaration("lookup")])]).unwrap();- let only_functions =- TopLevelToolsState::new(vec![namespace("functions", vec![declaration("unchanged")])])- .unwrap();- let empty = TopLevelToolsState::new(Vec::new()).unwrap();- let restored = TopLevelToolsState::new(vec![namespace(- "functions",- vec![declaration("unchanged"), declaration("restored")],- )])+ let empty = TopLevelToolsState::new(Vec::new(), Some(Arc::clone(&telemetry))).unwrap();+ let restored = TopLevelToolsState::new(+ vec![namespace(+ "functions",+ vec![declaration("unchanged"), declaration("restored")],+ )],+ Some(telemetry),+ ).unwrap();insta::assert_snapshot!(render_section_cases(&[@@ -102,6 +151,38 @@ fn snapshots_namespace_transitions() {// Restoration declares both an old member and one that has never been available.(Known(&empty), Known(&restored)),]));++ let snapshot = metrics.snapshot().unwrap();+ let counter = snapshot+ .scope_metrics()+ .flat_map(ScopeMetrics::metrics)+ .find(|metric| metric.name() == TOOL_INCREMENTAL_UPDATES_METRIC)+ .expect("tool update counter");+ let AggregatedMetrics::U64(MetricData::Sum(sum)) = counter.data() else {+ panic!("tool updates must be a counter");+ };+ let counts = sum+ .data_points()+ .map(|point| {+ let tags = point+ .attributes()+ .map(|attribute| (attribute.key.as_str(), attribute.value.as_str().to_string()))+ .collect::<BTreeMap<_, _>>();+ assert_eq!(tags.len(), 1);+ (tags["action"].clone(), point.value())+ })+ .collect::<BTreeMap<_, _>>();+ // Initial/full snapshots and unchanged catalogs contribute nothing; namespace+ // headers do not count, and removing a whole namespace counts each member.+ assert_eq!(+ counts,+ BTreeMap::from([+ ("added".to_string(), 6),+ ("removed".to_string(), 11),+ ("schema_changed".to_string(), 1),+ ])+ );+ metrics.shutdown().unwrap();}#[test]@@ -110,7 +191,7 @@ fn initial_catalog_preserves_order_and_unchanged_catalog_emits_nothing() {namespace("zebra", vec![declaration("lookup")]),namespace("alpha", vec![declaration("lookup")]),];- let state = TopLevelToolsState::new(definitions.clone()).unwrap();+ let state = TopLevelToolsState::new(definitions.clone(), /*metrics*/ None).unwrap();let (snapshot, updates) = state.render_diff(PreviousSectionState::Absent);assert_eq!(merge_world_state_updates(updates),@@ -131,12 +212,12 @@ fn initial_catalog_preserves_order_and_unchanged_catalog_emits_nothing() {#[test]fn empty_catalog_is_persisted_and_can_add_tools_again() {let definitions = vec![namespace("functions", vec![declaration("lookup")])];- let original = TopLevelToolsState::new(definitions.clone()).unwrap();+ let original = TopLevelToolsState::new(definitions.clone(), /*metrics*/ None).unwrap();let previous = original.render_diff(PreviousSectionState::Absent).0.unwrap();- let empty = TopLevelToolsState::new(Vec::new()).unwrap();+ let empty = TopLevelToolsState::new(Vec::new(), /*metrics*/ None).unwrap();let (snapshot, updates) = empty.render_diff(PreviousSectionState::Known(&previous));assert_eq!(snapshot, Some(BTreeMap::new()));assert_eq!(@@ -152,17 +233,20 @@ fn empty_catalog_is_persisted_and_can_add_tools_again() {#[test]fn namespace_diff_contains_only_changed_and_added_tools() {- let original = TopLevelToolsState::new(vec![- namespace(- "functions",- vec![- declaration("changed"),- declaration("unchanged"),- declaration("removed"),- ],- ),- namespace("other", vec![declaration("changed")]),- ])+ let original = TopLevelToolsState::new(+ vec![+ namespace(+ "functions",+ vec![+ declaration("changed"),+ declaration("unchanged"),+ declaration("removed"),+ ],+ ),+ namespace("other", vec![declaration("changed")]),+ ],+ /*metrics*/ None,+ ).unwrap();let snapshot = original.render_diff(PreviousSectionState::Absent)@@ -171,13 +255,16 @@ fn namespace_diff_contains_only_changed_and_added_tools() {let mut changed = declaration("changed");changed["parameters"]["properties"] = json!({"query": {"type": "string"}});let added = declaration("added");- let current = TopLevelToolsState::new(vec![- namespace(- "functions",- vec![changed.clone(), declaration("unchanged"), added.clone()],- ),- namespace("other", vec![declaration("changed")]),- ])+ let current = TopLevelToolsState::new(+ vec![+ namespace(+ "functions",+ vec![changed.clone(), declaration("unchanged"), added.clone()],+ ),+ namespace("other", vec![declaration("changed")]),+ ],+ /*metrics*/ None,+ ).unwrap();let (snapshot, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));let mut expected = namespace("functions", vec![changed, added]);@@ -202,15 +289,18 @@ fn namespace_diff_contains_only_changed_and_added_tools() {#[test]fn namespace_description_change_does_not_repeat_unchanged_tools() {- let original =- TopLevelToolsState::new(vec![namespace("functions", vec![declaration("lookup")])]).unwrap();+ let original = TopLevelToolsState::new(+ vec![namespace("functions", vec![declaration("lookup")])],+ /*metrics*/ None,+ )+ .unwrap();let snapshot = original.render_diff(PreviousSectionState::Absent).0.unwrap();let mut updated = namespace("functions", vec![declaration("lookup")]);updated["description"] = json!("Updated namespace guidance.");- let current = TopLevelToolsState::new(vec![updated]).unwrap();+ let current = TopLevelToolsState::new(vec![updated], /*metrics*/ None).unwrap();let (snapshot, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));assert_eq!(merge_world_state_updates(updates),@@ -228,10 +318,13 @@ fn namespace_description_change_does_not_repeat_unchanged_tools() {#[test]fn tool_type_change_keeps_the_callable_name_available() {- let original = TopLevelToolsState::new(vec![namespace(- "functions",- vec![declaration("lookup"), declaration("removed")],- )])+ let original = TopLevelToolsState::new(+ vec![namespace(+ "functions",+ vec![declaration("lookup"), declaration("removed")],+ )],+ /*metrics*/ None,+ ).unwrap();let snapshot = original.render_diff(PreviousSectionState::Absent)@@ -240,7 +333,7 @@ fn tool_type_change_keeps_the_callable_name_available() {let custom = json!({"type": "custom", "name": "lookup", "description": "Look up a value.","format": {"type": "text"}});let mut updated = namespace("functions", vec![custom]);- let current = TopLevelToolsState::new(vec![updated.clone()]).unwrap();+ let current = TopLevelToolsState::new(vec![updated.clone()], /*metrics*/ None).unwrap();let (_, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));updated["description"] = json!(format!("Tools.\n{NAMESPACE_UPDATE_HINT}"));assert_eq!(@@ -257,26 +350,32 @@ fn tool_type_change_keeps_the_callable_name_available() {#[test]fn namespace_removals_emit_one_notice() {- let original = TopLevelToolsState::new(vec![- namespace(- "functions",- vec![- declaration("lookup"),- declaration("removed"),- declaration("also_removed"),- ],- ),- namespace("function", vec![declaration("lookup")]),- namespace("other", vec![declaration("lookup")]),- json!({"type": "web_search"}),- ])+ let original = TopLevelToolsState::new(+ vec![+ namespace(+ "functions",+ vec![+ declaration("lookup"),+ declaration("removed"),+ declaration("also_removed"),+ ],+ ),+ namespace("function", vec![declaration("lookup")]),+ namespace("other", vec![declaration("lookup")]),+ json!({"type": "web_search"}),+ ],+ /*metrics*/ None,+ ).unwrap();let snapshot = original.render_diff(PreviousSectionState::Absent).0.unwrap();- let current =- TopLevelToolsState::new(vec![namespace("functions", vec![declaration("lookup")])]).unwrap();+ let current = TopLevelToolsState::new(+ vec![namespace("functions", vec![declaration("lookup")])],+ /*metrics*/ None,+ )+ .unwrap();let (_, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));assert_eq!(merge_world_state_updates(updates),@@ -293,15 +392,17 @@ fn namespace_removals_emit_one_notice() {#[test]fn namespace_replaced_by_builtin_keeps_the_replacement_available() {- let original =- TopLevelToolsState::new(vec![namespace("web_search", vec![declaration("lookup")])])- .unwrap();+ let original = TopLevelToolsState::new(+ vec![namespace("web_search", vec![declaration("lookup")])],+ /*metrics*/ None,+ )+ .unwrap();let snapshot = original.render_diff(PreviousSectionState::Absent).0.unwrap();let replacement = json!({"type": "web_search"});- let current = TopLevelToolsState::new(vec![replacement.clone()]).unwrap();+ let current = TopLevelToolsState::new(vec![replacement.clone()], /*metrics*/ None).unwrap();let (_, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));assert_eq!(merge_world_state_updates(updates),diff --git a/codex-rs/core/src/session/world_state.rs b/codex-rs/core/src/session/world_state.rsindex a050fccf13..c487edda28 100644--- a/codex-rs/core/src/session/world_state.rs+++ b/codex-rs/core/src/session/world_state.rs@@ -140,10 +140,16 @@ impl Session {String::new()};let mut world_state = WorldState::default();+ let extension_metrics = super::extension_metrics::from_session_telemetry(+ step_context.session_telemetry.clone(),+ );if incremental_tools {let specs = step_context.tool_router.model_visible_specs();let definitions = codex_tools::create_tools_json_for_responses_lite(&specs)?;- world_state.add_section(TopLevelToolsState::new(definitions)?);+ world_state.add_section(TopLevelToolsState::new(+ definitions,+ Some(Arc::clone(&extension_metrics)),+ )?);world_state.add_section(BaseInstructionsState(base_instructions));}world_state.add_section(ModelInstructionsState::new(@@ -305,9 +311,6 @@ impl Session {world_state.add_section(PluginsInstructionsState::new(plugins_usage_instructions_available,));- let extension_metrics = super::extension_metrics::from_session_telemetry(- step_context.session_telemetry.clone(),- );if turn_context.config.featuresdiff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rsindex d80948bb1b..d4aa1a6e69 100644--- a/codex-rs/otel/src/metrics/names.rs+++ b/codex-rs/otel/src/metrics/names.rs@@ -88,6 +88,10 @@ pub const TOOL_REGISTRATIONS_METRIC_BUCKETS: &[f64] = &[pub static THREAD_TOOLS_METRIC_BUCKETS: LazyLock<[f64; 511]> =LazyLock::new(|| context_log_buckets(/*max_exponent*/ 15.0));+/// Rendered incremental tool deltas, tagged by action; excludes initial catalogs.+/// schema_changed includes any definition change, such as description edits.+pub const TOOL_INCREMENTAL_UPDATES_METRIC: &str = "codex.tools.incremental_updates";+pub static THREAD_SKILLS_COUNT_METRIC_BUCKETS: LazyLock<[f64; 513]> =LazyLock::new(|| std::array::from_fn(|index| index as f64));