Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Record metrics for incremental tool updates (#51893)

Alex Kira70efa82b09ed
diff --git a/codex-rs/core/src/context/world_state/top_level_tools.rs b/codex-rs/core/src/context/world_state/top_level_tools.rs
index 5fc9b5b6f3..cc07cef8f5 100644
--- a/codex-rs/core/src/context/world_state/top_level_tools.rs
+++ b/codex-rs/core/src/context/world_state/top_level_tools.rs
@@ -2,6 +2,7 @@
//! Missing state starts a fresh catalog; this section does not migrate legacy history.
//! Incremental hints decorate emitted declarations only, leaving catalog hashes unchanged.
//! Whole namespace removals subsume their members in the removal notice.
+//! Metrics count rendered tool redefinitions, excluding namespace headers and initial catalogs.
use super::PreviousSectionState;
use super::SectionTransition;
@@ -10,6 +11,8 @@ use super::WorldStateSection;
use super::WorldStateUpdate;
use crate::context::ContextualUserFragment;
use crate::context::DeveloperInstructions;
+use codex_extension_api::ExtensionMetrics;
+use codex_otel::TOOL_INCREMENTAL_UPDATES_METRIC;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::error::Result;
use codex_protocol::models::ContentItemKind;
@@ -17,6 +20,11 @@ use codex_protocol::models::ResponseItem;
use serde_json::Value;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
+use std::sync::Arc;
+
+const ACTION_ADDED: &str = "added";
+const ACTION_REMOVED: &str = "removed";
+const ACTION_SCHEMA_CHANGED: &str = "schema_changed";
const NAMESPACE_UPDATE_HINT: &str = "This is an incremental namespace update. Previously declared tools remain available for direct calls unless explicitly marked unavailable. If a tool is redefined here, its latest definition replaces the earlier one.";
const REMOVED_TOOLS_HEADER: &str = "The following tools are no longer available. Do not call them:";
@@ -26,6 +34,7 @@ const REMOVED_NAMESPACES_HEADER: &str = "The following namespaces are no longer
pub(crate) struct TopLevelToolsState {
definitions: Vec<Value>,
hashes: BTreeMap<String, WorldStateHash>,
+ metrics: Option<Arc<dyn ExtensionMetrics>>,
}
impl TopLevelToolsState {
@@ -33,7 +42,10 @@ impl TopLevelToolsState {
clippy::expect_used,
reason = "the Responses Lite serializer emits objects with array-valued namespace members"
)]
- pub(crate) fn new(definitions: Vec<Value>) -> Result<Self> {
+ pub(crate) fn new(
+ definitions: Vec<Value>,
+ metrics: Option<Arc<dyn ExtensionMetrics>>,
+ ) -> Result<Self> {
let mut hashes = BTreeMap::new();
let mut insert_hash = |name: String, definition: &Value| -> Result<()> {
let hash = WorldStateHash::from_json(definition);
@@ -59,6 +71,7 @@ impl TopLevelToolsState {
Ok(Self {
definitions,
hashes,
+ metrics,
})
}
}
@@ -75,6 +88,50 @@ fn definition_name(definition: &Value) -> &str {
.unwrap_or_else(|| definition["type"].as_str().expect("serialized tool type"))
}
+/// Accumulates callable changes during rendering without retaining tool definitions.
+#[derive(Default)]
+struct ToolUpdateMetrics {
+ count_added: usize,
+ count_removed: usize,
+ count_schema_changed: usize,
+}
+
+impl ToolUpdateMetrics {
+ fn count_changed_tool(
+ &mut self,
+ name: &str,
+ previous: Option<&BTreeMap<String, WorldStateHash>>,
+ ) {
+ if let Some(previous) = previous {
+ if previous.contains_key(name) {
+ // The existing hash covers the entire definition, including descriptions.
+ self.count_schema_changed += 1;
+ } else {
+ self.count_added += 1;
+ }
+ }
+ }
+
+ fn record(self, metrics: Option<&dyn ExtensionMetrics>) {
+ let Some(metrics) = metrics else {
+ return;
+ };
+ for (action, count) in [
+ (ACTION_ADDED, self.count_added),
+ (ACTION_REMOVED, self.count_removed),
+ (ACTION_SCHEMA_CHANGED, self.count_schema_changed),
+ ] {
+ if count > 0 {
+ metrics.counter(
+ TOOL_INCREMENTAL_UPDATES_METRIC,
+ i64::try_from(count).unwrap_or(i64::MAX),
+ &[("action", action)],
+ );
+ }
+ }
+ }
+}
+
impl WorldStateSection for TopLevelToolsState {
const ID: &'static str = "top_level_tools";
type Snapshot = BTreeMap<String, WorldStateHash>;
@@ -89,24 +146,29 @@ impl WorldStateSection for TopLevelToolsState {
};
let changed =
|name: &str| previous.and_then(|previous| previous.get(name)) != self.hashes.get(name);
+ let mut counts = ToolUpdateMetrics::default();
let mut namespace_updates = Vec::new();
let mut tools = Vec::new();
for definition in &self.definitions {
let name = definition_name(definition);
let Some(members) = definition["tools"].as_array() else {
if changed(name) {
+ counts.count_changed_tool(name, previous);
tools.push(definition.clone());
}
continue;
};
- let members = members
- .iter()
- .filter(|tool| changed(&format!("{name}.{}", definition_name(tool))))
- .cloned()
- .collect::<Vec<_>>();
- if !members.is_empty() {
+ let mut changed_members = Vec::new();
+ for tool in members {
+ let tool_name = format!("{name}.{}", definition_name(tool));
+ if changed(&tool_name) {
+ counts.count_changed_tool(&tool_name, previous);
+ changed_members.push(tool.clone());
+ }
+ }
+ if !changed_members.is_empty() {
let mut namespace = definition.clone();
- namespace["tools"] = Value::Array(members);
+ namespace["tools"] = Value::Array(changed_members);
if previous.is_some_and(|previous| previous.contains_key(name)) {
let description = definition["description"].as_str().unwrap_or_default();
namespace["description"] = Value::String(if description.is_empty() {
@@ -153,11 +215,14 @@ impl WorldStateSection for TopLevelToolsState {
let tools = previous
.keys()
.filter(|key| !self.hashes.contains_key(*key))
+ .filter(|key| !namespaces.contains(key.as_str()))
+ .inspect(|_| {
+ // Count members even when the notice collapses their removed namespace.
+ counts.count_removed += 1;
+ })
.filter(|key| {
- !namespaces.contains(key.as_str())
- && !key
- .split_once('.')
- .is_some_and(|(namespace, _)| namespaces.contains(namespace))
+ !key.split_once('.')
+ .is_some_and(|(namespace, _)| namespaces.contains(namespace))
})
.cloned()
.collect::<Vec<_>>();
@@ -171,6 +236,7 @@ impl WorldStateSection for TopLevelToolsState {
);
}
}
+ counts.record(self.metrics.as_deref());
// Persist the empty map too: it is a known empty catalog, not missing state.
(Some(self.hashes.clone()), updates)
}
diff --git a/codex-rs/core/src/context/world_state/top_level_tools_tests.rs b/codex-rs/core/src/context/world_state/top_level_tools_tests.rs
index dd15408619..a072b039af 100644
--- a/codex-rs/core/src/context/world_state/top_level_tools_tests.rs
+++ b/codex-rs/core/src/context/world_state/top_level_tools_tests.rs
@@ -3,6 +3,16 @@
use super::*;
use crate::context::world_state::test_support::render_section_cases;
use crate::context_manager::updates::merge_world_state_updates;
+use crate::session::extension_metrics::from_session_telemetry;
+use codex_otel::MetricsClient;
+use codex_otel::MetricsConfig;
+use codex_otel::SessionTelemetry;
+use codex_protocol::ThreadId;
+use codex_protocol::protocol::SessionSource;
+use opentelemetry_sdk::metrics::InMemoryMetricExporter;
+use opentelemetry_sdk::metrics::data::AggregatedMetrics;
+use opentelemetry_sdk::metrics::data::MetricData;
+use opentelemetry_sdk::metrics::data::ScopeMetrics;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -32,12 +42,12 @@ fn snapshots_catalog_transitions() {
"description": "Search tools.",
"parameters": {"type": "object", "properties": {}}});
let web = json!({"type": "web_search", "external_web_access": true});
- let original = TopLevelToolsState::new(vec![search.clone()]).unwrap();
+ let original = TopLevelToolsState::new(vec![search.clone()], /*metrics*/ None).unwrap();
let mut changed = search;
changed["parameters"]["properties"] = json!({"query": {"type": "string"}});
- let updated = TopLevelToolsState::new(vec![changed, web.clone()]).unwrap();
- let only_web = TopLevelToolsState::new(vec![web]).unwrap();
- let empty = TopLevelToolsState::new(Vec::new()).unwrap();
+ let updated = TopLevelToolsState::new(vec![changed, web.clone()], /*metrics*/ None).unwrap();
+ let only_web = TopLevelToolsState::new(vec![web], /*metrics*/ None).unwrap();
+ let empty = TopLevelToolsState::new(Vec::new(), /*metrics*/ None).unwrap();
insta::assert_snapshot!(render_section_cases(&[
(Absent, Known(&original)),
@@ -49,43 +59,82 @@ fn snapshots_catalog_transitions() {
]));
}
-#[test]
-fn snapshots_namespace_transitions() {
+#[tokio::test]
+async fn snapshots_namespace_transitions() {
use PreviousSectionState::Absent;
use PreviousSectionState::Known;
- let original = TopLevelToolsState::new(vec![
- namespace(
- "functions",
- vec![
- declaration("unchanged"),
- declaration("removed"),
- declaration("changed"),
- ],
- ),
- namespace("other", vec![declaration("lookup")]),
- ])
+ let metrics = MetricsClient::new(
+ MetricsConfig::in_memory(
+ "test",
+ "tool-updates",
+ env!("CARGO_PKG_VERSION"),
+ InMemoryMetricExporter::default(),
+ )
+ .with_runtime_reader(),
+ )
+ .unwrap();
+ let telemetry = from_session_telemetry(
+ SessionTelemetry::new(
+ ThreadId::new(),
+ "test-model",
+ "test-model",
+ /*account_id*/ None,
+ /*account_email*/ None,
+ /*auth_mode*/ None,
+ "test".to_string(),
+ /*log_user_prompts*/ false,
+ "test".to_string(),
+ SessionSource::Cli,
+ )
+ .with_metrics_without_metadata_tags(metrics.clone()),
+ );
+ let original = TopLevelToolsState::new(
+ vec![
+ namespace(
+ "functions",
+ vec![
+ declaration("unchanged"),
+ declaration("removed"),
+ declaration("changed"),
+ ],
+ ),
+ namespace("other", vec![declaration("lookup")]),
+ ],
+ Some(Arc::clone(&telemetry)),
+ )
.unwrap();
let mut changed = declaration("changed");
changed["parameters"]["properties"] = json!({"query": {"type": "string"}});
- let updated = TopLevelToolsState::new(vec![
- namespace(
- "functions",
- vec![declaration("unchanged"), changed, declaration("added")],
- ),
- namespace("other", vec![declaration("lookup")]),
- ])
+ let updated = TopLevelToolsState::new(
+ vec![
+ namespace(
+ "functions",
+ vec![declaration("unchanged"), changed, declaration("added")],
+ ),
+ namespace("other", vec![declaration("lookup")]),
+ ],
+ Some(Arc::clone(&telemetry)),
+ )
+ .unwrap();
+ let only_other = TopLevelToolsState::new(
+ vec![namespace("other", vec![declaration("lookup")])],
+ Some(Arc::clone(&telemetry)),
+ )
+ .unwrap();
+ let only_functions = TopLevelToolsState::new(
+ vec![namespace("functions", vec![declaration("unchanged")])],
+ Some(Arc::clone(&telemetry)),
+ )
.unwrap();
- let only_other =
- TopLevelToolsState::new(vec![namespace("other", vec![declaration("lookup")])]).unwrap();
- let only_functions =
- TopLevelToolsState::new(vec![namespace("functions", vec![declaration("unchanged")])])
- .unwrap();
- let empty = TopLevelToolsState::new(Vec::new()).unwrap();
- let restored = TopLevelToolsState::new(vec![namespace(
- "functions",
- vec![declaration("unchanged"), declaration("restored")],
- )])
+ let empty = TopLevelToolsState::new(Vec::new(), Some(Arc::clone(&telemetry))).unwrap();
+ let restored = TopLevelToolsState::new(
+ vec![namespace(
+ "functions",
+ vec![declaration("unchanged"), declaration("restored")],
+ )],
+ Some(telemetry),
+ )
.unwrap();
insta::assert_snapshot!(render_section_cases(&[
@@ -102,6 +151,38 @@ fn snapshots_namespace_transitions() {
// Restoration declares both an old member and one that has never been available.
(Known(&empty), Known(&restored)),
]));
+
+ let snapshot = metrics.snapshot().unwrap();
+ let counter = snapshot
+ .scope_metrics()
+ .flat_map(ScopeMetrics::metrics)
+ .find(|metric| metric.name() == TOOL_INCREMENTAL_UPDATES_METRIC)
+ .expect("tool update counter");
+ let AggregatedMetrics::U64(MetricData::Sum(sum)) = counter.data() else {
+ panic!("tool updates must be a counter");
+ };
+ let counts = sum
+ .data_points()
+ .map(|point| {
+ let tags = point
+ .attributes()
+ .map(|attribute| (attribute.key.as_str(), attribute.value.as_str().to_string()))
+ .collect::<BTreeMap<_, _>>();
+ assert_eq!(tags.len(), 1);
+ (tags["action"].clone(), point.value())
+ })
+ .collect::<BTreeMap<_, _>>();
+ // Initial/full snapshots and unchanged catalogs contribute nothing; namespace
+ // headers do not count, and removing a whole namespace counts each member.
+ assert_eq!(
+ counts,
+ BTreeMap::from([
+ ("added".to_string(), 6),
+ ("removed".to_string(), 11),
+ ("schema_changed".to_string(), 1),
+ ])
+ );
+ metrics.shutdown().unwrap();
}
#[test]
@@ -110,7 +191,7 @@ fn initial_catalog_preserves_order_and_unchanged_catalog_emits_nothing() {
namespace("zebra", vec![declaration("lookup")]),
namespace("alpha", vec![declaration("lookup")]),
];
- let state = TopLevelToolsState::new(definitions.clone()).unwrap();
+ let state = TopLevelToolsState::new(definitions.clone(), /*metrics*/ None).unwrap();
let (snapshot, updates) = state.render_diff(PreviousSectionState::Absent);
assert_eq!(
merge_world_state_updates(updates),
@@ -131,12 +212,12 @@ fn initial_catalog_preserves_order_and_unchanged_catalog_emits_nothing() {
#[test]
fn empty_catalog_is_persisted_and_can_add_tools_again() {
let definitions = vec![namespace("functions", vec![declaration("lookup")])];
- let original = TopLevelToolsState::new(definitions.clone()).unwrap();
+ let original = TopLevelToolsState::new(definitions.clone(), /*metrics*/ None).unwrap();
let previous = original
.render_diff(PreviousSectionState::Absent)
.0
.unwrap();
- let empty = TopLevelToolsState::new(Vec::new()).unwrap();
+ let empty = TopLevelToolsState::new(Vec::new(), /*metrics*/ None).unwrap();
let (snapshot, updates) = empty.render_diff(PreviousSectionState::Known(&previous));
assert_eq!(snapshot, Some(BTreeMap::new()));
assert_eq!(
@@ -152,17 +233,20 @@ fn empty_catalog_is_persisted_and_can_add_tools_again() {
#[test]
fn namespace_diff_contains_only_changed_and_added_tools() {
- let original = TopLevelToolsState::new(vec![
- namespace(
- "functions",
- vec![
- declaration("changed"),
- declaration("unchanged"),
- declaration("removed"),
- ],
- ),
- namespace("other", vec![declaration("changed")]),
- ])
+ let original = TopLevelToolsState::new(
+ vec![
+ namespace(
+ "functions",
+ vec![
+ declaration("changed"),
+ declaration("unchanged"),
+ declaration("removed"),
+ ],
+ ),
+ namespace("other", vec![declaration("changed")]),
+ ],
+ /*metrics*/ None,
+ )
.unwrap();
let snapshot = original
.render_diff(PreviousSectionState::Absent)
@@ -171,13 +255,16 @@ fn namespace_diff_contains_only_changed_and_added_tools() {
let mut changed = declaration("changed");
changed["parameters"]["properties"] = json!({"query": {"type": "string"}});
let added = declaration("added");
- let current = TopLevelToolsState::new(vec![
- namespace(
- "functions",
- vec![changed.clone(), declaration("unchanged"), added.clone()],
- ),
- namespace("other", vec![declaration("changed")]),
- ])
+ let current = TopLevelToolsState::new(
+ vec![
+ namespace(
+ "functions",
+ vec![changed.clone(), declaration("unchanged"), added.clone()],
+ ),
+ namespace("other", vec![declaration("changed")]),
+ ],
+ /*metrics*/ None,
+ )
.unwrap();
let (snapshot, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));
let mut expected = namespace("functions", vec![changed, added]);
@@ -202,15 +289,18 @@ fn namespace_diff_contains_only_changed_and_added_tools() {
#[test]
fn namespace_description_change_does_not_repeat_unchanged_tools() {
- let original =
- TopLevelToolsState::new(vec![namespace("functions", vec![declaration("lookup")])]).unwrap();
+ let original = TopLevelToolsState::new(
+ vec![namespace("functions", vec![declaration("lookup")])],
+ /*metrics*/ None,
+ )
+ .unwrap();
let snapshot = original
.render_diff(PreviousSectionState::Absent)
.0
.unwrap();
let mut updated = namespace("functions", vec![declaration("lookup")]);
updated["description"] = json!("Updated namespace guidance.");
- let current = TopLevelToolsState::new(vec![updated]).unwrap();
+ let current = TopLevelToolsState::new(vec![updated], /*metrics*/ None).unwrap();
let (snapshot, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));
assert_eq!(
merge_world_state_updates(updates),
@@ -228,10 +318,13 @@ fn namespace_description_change_does_not_repeat_unchanged_tools() {
#[test]
fn tool_type_change_keeps_the_callable_name_available() {
- let original = TopLevelToolsState::new(vec![namespace(
- "functions",
- vec![declaration("lookup"), declaration("removed")],
- )])
+ let original = TopLevelToolsState::new(
+ vec![namespace(
+ "functions",
+ vec![declaration("lookup"), declaration("removed")],
+ )],
+ /*metrics*/ None,
+ )
.unwrap();
let snapshot = original
.render_diff(PreviousSectionState::Absent)
@@ -240,7 +333,7 @@ fn tool_type_change_keeps_the_callable_name_available() {
let custom = json!({"type": "custom", "name": "lookup", "description": "Look up a value.",
"format": {"type": "text"}});
let mut updated = namespace("functions", vec![custom]);
- let current = TopLevelToolsState::new(vec![updated.clone()]).unwrap();
+ let current = TopLevelToolsState::new(vec![updated.clone()], /*metrics*/ None).unwrap();
let (_, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));
updated["description"] = json!(format!("Tools.\n{NAMESPACE_UPDATE_HINT}"));
assert_eq!(
@@ -257,26 +350,32 @@ fn tool_type_change_keeps_the_callable_name_available() {
#[test]
fn namespace_removals_emit_one_notice() {
- let original = TopLevelToolsState::new(vec![
- namespace(
- "functions",
- vec![
- declaration("lookup"),
- declaration("removed"),
- declaration("also_removed"),
- ],
- ),
- namespace("function", vec![declaration("lookup")]),
- namespace("other", vec![declaration("lookup")]),
- json!({"type": "web_search"}),
- ])
+ let original = TopLevelToolsState::new(
+ vec![
+ namespace(
+ "functions",
+ vec![
+ declaration("lookup"),
+ declaration("removed"),
+ declaration("also_removed"),
+ ],
+ ),
+ namespace("function", vec![declaration("lookup")]),
+ namespace("other", vec![declaration("lookup")]),
+ json!({"type": "web_search"}),
+ ],
+ /*metrics*/ None,
+ )
.unwrap();
let snapshot = original
.render_diff(PreviousSectionState::Absent)
.0
.unwrap();
- let current =
- TopLevelToolsState::new(vec![namespace("functions", vec![declaration("lookup")])]).unwrap();
+ let current = TopLevelToolsState::new(
+ vec![namespace("functions", vec![declaration("lookup")])],
+ /*metrics*/ None,
+ )
+ .unwrap();
let (_, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));
assert_eq!(
merge_world_state_updates(updates),
@@ -293,15 +392,17 @@ fn namespace_removals_emit_one_notice() {
#[test]
fn namespace_replaced_by_builtin_keeps_the_replacement_available() {
- let original =
- TopLevelToolsState::new(vec![namespace("web_search", vec![declaration("lookup")])])
- .unwrap();
+ let original = TopLevelToolsState::new(
+ vec![namespace("web_search", vec![declaration("lookup")])],
+ /*metrics*/ None,
+ )
+ .unwrap();
let snapshot = original
.render_diff(PreviousSectionState::Absent)
.0
.unwrap();
let replacement = json!({"type": "web_search"});
- let current = TopLevelToolsState::new(vec![replacement.clone()]).unwrap();
+ let current = TopLevelToolsState::new(vec![replacement.clone()], /*metrics*/ None).unwrap();
let (_, updates) = current.render_diff(PreviousSectionState::Known(&snapshot));
assert_eq!(
merge_world_state_updates(updates),
diff --git a/codex-rs/core/src/session/world_state.rs b/codex-rs/core/src/session/world_state.rs
index a050fccf13..c487edda28 100644
--- a/codex-rs/core/src/session/world_state.rs
+++ b/codex-rs/core/src/session/world_state.rs
@@ -140,10 +140,16 @@ impl Session {
String::new()
};
let mut world_state = WorldState::default();
+ let extension_metrics = super::extension_metrics::from_session_telemetry(
+ step_context.session_telemetry.clone(),
+ );
if incremental_tools {
let specs = step_context.tool_router.model_visible_specs();
let definitions = codex_tools::create_tools_json_for_responses_lite(&specs)?;
- world_state.add_section(TopLevelToolsState::new(definitions)?);
+ world_state.add_section(TopLevelToolsState::new(
+ definitions,
+ Some(Arc::clone(&extension_metrics)),
+ )?);
world_state.add_section(BaseInstructionsState(base_instructions));
}
world_state.add_section(ModelInstructionsState::new(
@@ -305,9 +311,6 @@ impl Session {
world_state.add_section(PluginsInstructionsState::new(
plugins_usage_instructions_available,
));
- let extension_metrics = super::extension_metrics::from_session_telemetry(
- step_context.session_telemetry.clone(),
- );
if turn_context
.config
.features
diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rs
index d80948bb1b..d4aa1a6e69 100644
--- a/codex-rs/otel/src/metrics/names.rs
+++ b/codex-rs/otel/src/metrics/names.rs
@@ -88,6 +88,10 @@ pub const TOOL_REGISTRATIONS_METRIC_BUCKETS: &[f64] = &[
pub static THREAD_TOOLS_METRIC_BUCKETS: LazyLock<[f64; 511]> =
LazyLock::new(|| context_log_buckets(/*max_exponent*/ 15.0));
+/// Rendered incremental tool deltas, tagged by action; excludes initial catalogs.
+/// schema_changed includes any definition change, such as description edits.
+pub const TOOL_INCREMENTAL_UPDATES_METRIC: &str = "codex.tools.incremental_updates";
+
pub static THREAD_SKILLS_COUNT_METRIC_BUCKETS: LazyLock<[f64; 513]> =
LazyLock::new(|| std::array::from_fn(|index| index as f64));