Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Classify tool analytics events by call origin (#45535)

Eddie Chen12b0164a484f
diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs
index c150995f6d..1c6c94097d 100644
--- a/codex-rs/analytics/src/analytics_client_tests.rs
+++ b/codex-rs/analytics/src/analytics_client_tests.rs
@@ -36,6 +36,7 @@ use crate::events::ReviewTrigger;
use crate::events::Reviewer;
use crate::events::ThreadInitializedEvent;
use crate::events::ThreadInitializedEventParams;
+use crate::events::ToolEventType;
use crate::events::ToolItemTerminalStatus;
use crate::events::TrackEventRequest;
use crate::events::codex_app_metadata;
@@ -1823,6 +1824,7 @@ fn command_execution_event_serializes_expected_shape() {
subagent_source: None,
parent_thread_id: None,
tool_name: "shell".to_string(),
+ tool_event_type: Some(ToolEventType::ModelToolCall),
started_at_ms: 123_000,
completed_at_ms: 125_000,
duration_ms: Some(2000),
@@ -1849,63 +1851,63 @@ fn command_execution_event_serializes_expected_shape() {
});
let payload = serde_json::to_value(&event).expect("serialize command execution event");
- assert_eq!(
- payload,
- json!({
- "event_type": "codex_command_execution_event",
- "event_params": {
- "model_slug": null,
- "reasoning_effort": null,
- "thread_id": "thread-1",
- "session_id": "session-thread-1",
- "turn_id": "turn-1",
- "root_turn_id": "root-turn",
- "item_id": "item-1",
- "cell_id": null,
- "parent_call_id": null,
- "originating_response_id": null,
- "subsequent_response_id": null,
- "app_server_client": {
- "product_client_id": "codex_tui",
- "client_name": "codex-tui",
- "client_version": "1.2.3",
- "rpc_transport": "websocket",
- "experimental_api_enabled": true
- },
- "runtime": {
- "codex_rs_version": "0.99.0",
- "runtime_os": "macos",
- "runtime_os_version": "15.3.1",
- "runtime_arch": "aarch64"
- },
- "thread_source": "user",
- "subagent_source": null,
- "parent_thread_id": null,
- "tool_name": "shell",
- "started_at_ms": 123000,
- "completed_at_ms": 125000,
- "duration_ms": 2000,
- "execution_duration_ms": 1900,
- "review_count": 0,
- "guardian_review_count": 0,
- "user_review_count": 0,
- "final_approval_outcome": "not_needed",
- "terminal_status": "completed",
- "failure_kind": null,
- "requested_additional_permissions": false,
- "requested_network_access": false,
- "plugin_id": "sample@openai-curated",
- "script_path": "scripts/run.py",
- "command_execution_source": "agent",
- "exit_code": 0,
- "command_total_action_count": 4,
- "command_read_action_count": 1,
- "command_list_files_action_count": 1,
- "command_search_action_count": 1,
- "command_unknown_action_count": 1
- }
- })
- );
+ let mut expected = json!({
+ "event_type": "codex_command_execution_event",
+ "event_params": {
+ "model_slug": null,
+ "reasoning_effort": null,
+ "thread_id": "thread-1",
+ "session_id": "session-thread-1",
+ "turn_id": "turn-1",
+ "root_turn_id": "root-turn",
+ "item_id": "item-1",
+ "cell_id": null,
+ "parent_call_id": null,
+ "originating_response_id": null,
+ "subsequent_response_id": null,
+ "app_server_client": {
+ "product_client_id": "codex_tui",
+ "client_name": "codex-tui",
+ "client_version": "1.2.3",
+ "rpc_transport": "websocket",
+ "experimental_api_enabled": true
+ },
+ "runtime": {
+ "codex_rs_version": "0.99.0",
+ "runtime_os": "macos",
+ "runtime_os_version": "15.3.1",
+ "runtime_arch": "aarch64"
+ },
+ "thread_source": "user",
+ "subagent_source": null,
+ "parent_thread_id": null,
+ "tool_name": "shell",
+ "started_at_ms": 123000,
+ "completed_at_ms": 125000,
+ "duration_ms": 2000,
+ "execution_duration_ms": 1900,
+ "review_count": 0,
+ "guardian_review_count": 0,
+ "user_review_count": 0,
+ "final_approval_outcome": "not_needed",
+ "terminal_status": "completed",
+ "failure_kind": null,
+ "requested_additional_permissions": false,
+ "requested_network_access": false,
+ "plugin_id": "sample@openai-curated",
+ "script_path": "scripts/run.py",
+ "command_execution_source": "agent",
+ "exit_code": 0,
+ "command_total_action_count": 4,
+ "command_read_action_count": 1,
+ "command_list_files_action_count": 1,
+ "command_search_action_count": 1,
+ "command_unknown_action_count": 1
+ }
+ });
+ // Keep this field separate to stay within json!'s macro recursion limit.
+ expected["event_params"]["tool_event_type"] = json!("model_tool_call");
+ assert_eq!(payload, expected);
}
#[test]
@@ -3182,6 +3184,7 @@ async fn collaborator_tool_events_keep_response_ids_when_completion_races_sampli
"origin": params["originating_response_id"],
"subsequent": params["subsequent_response_id"],
"status": params["terminal_status"],
+ "tool_event_type": params["tool_event_type"],
}),
json!({
"type": "codex_collab_agent_tool_call_event",
@@ -3189,6 +3192,7 @@ async fn collaborator_tool_events_keep_response_ids_when_completion_races_sampli
"origin": "response-1",
"subsequent": "response-2",
"status": "failed",
+ "tool_event_type": "model_tool_call",
}),
);
}
@@ -3266,6 +3270,7 @@ async fn code_mode_exec_wait_and_child_events_share_cell_and_response_ids() {
let event = serde_json::to_value(event).expect("serialize tool event");
serde_json::json!({
"item": event["event_params"]["item_id"],
+ "tool_event_type": event["event_params"]["tool_event_type"],
"root": event["event_params"]["root_turn_id"],
"cell": event["event_params"]["cell_id"],
"parent": event["event_params"]["parent_call_id"],
@@ -3277,13 +3282,134 @@ async fn code_mode_exec_wait_and_child_events_share_cell_and_response_ids() {
assert_eq!(
actual,
vec![
- serde_json::json!({"item":"exec-1","root":"root-a","cell":"cell-1","parent":null,"origin":"resp-a","subsequent":"resp-b"}),
- serde_json::json!({"item":"child-1","root":"root-a","cell":"cell-1","parent":"exec-1","origin":"resp-a","subsequent":"resp-b"}),
- serde_json::json!({"item":"wait-1","root":"root-b","cell":"cell-1","parent":"exec-1","origin":"resp-c","subsequent":"resp-d"}),
+ serde_json::json!({"tool_event_type":"model_tool_call","item":"exec-1","root":"root-a","cell":"cell-1","parent":null,"origin":"resp-a","subsequent":"resp-b"}),
+ serde_json::json!({"tool_event_type":"inner_tool_call","item":"child-1","root":"root-a","cell":"cell-1","parent":"exec-1","origin":"resp-a","subsequent":"resp-b"}),
+ serde_json::json!({"tool_event_type":"model_tool_call","item":"wait-1","root":"root-b","cell":"cell-1","parent":"exec-1","origin":"resp-c","subsequent":"resp-d"}),
]
);
}
+#[tokio::test]
+async fn tool_event_types_require_exact_unambiguous_call_origin() {
+ for (sampled_ids, child_ids, expected) in [
+ (
+ vec!["command-1", "control-1"],
+ vec![],
+ json!("model_tool_call"),
+ ),
+ (
+ vec![],
+ vec!["command-1", "control-1"],
+ json!("inner_tool_call"),
+ ),
+ (vec![], vec![], json!(null)),
+ (
+ vec!["command-1", "control-1"],
+ vec!["command-1", "control-1"],
+ json!(null),
+ ),
+ (vec!["other-call"], vec!["other-child"], json!(null)),
+ ] {
+ let mut reducer = AnalyticsReducer::default();
+ let mut events = Vec::new();
+ ingest_review_prerequisites(&mut reducer, &mut events).await;
+ ingest_code_mode_facts(
+ &mut reducer,
+ &mut events,
+ [
+ sampling_response("turn-1", "response-1", &sampled_ids),
+ CodeModeToolCallFact::CellStarted {
+ thread_id: "thread-1".into(),
+ turn_id: "turn-1".into(),
+ call_id: "exec-1".into(),
+ cell_id: "cell-1".into(),
+ },
+ ],
+ )
+ .await;
+ ingest_code_mode_facts(
+ &mut reducer,
+ &mut events,
+ child_ids
+ .into_iter()
+ .map(|call_id| CodeModeToolCallFact::ChildStarted {
+ thread_id: "thread-1".into(),
+ turn_id: "turn-1".into(),
+ call_id: call_id.into(),
+ cell_id: "cell-1".into(),
+ }),
+ )
+ .await;
+ ingest_completed_command_execution_item(&mut reducer, &mut events, "thread-1", "command-1")
+ .await;
+ reducer
+ .ingest(
+ AnalyticsFact::Custom(CustomAnalyticsFact::ControlToolCall(ControlToolCallFact {
+ thread_id: "thread-1".into(),
+ turn_id: "turn-1".into(),
+ turn_metadata: test_turn_metadata(/*root_turn_id*/ None),
+ call_id: "control-1".into(),
+ cell_id: Some("cell-1".into()),
+ tool_name: "view_image".into(),
+ started_at_ms: 1_000,
+ completed_at_ms: 1_042,
+ status: ControlToolCallStatus::Completed,
+ })),
+ &mut events,
+ )
+ .await;
+ reducer.flush(&mut events);
+ let payload = serde_json::to_value(&events).expect("serialize tool events");
+ assert_eq!(
+ payload
+ .as_array()
+ .expect("events")
+ .iter()
+ .map(|event| json!({
+ "event": event["event_type"],
+ "type": event["event_params"]["tool_event_type"],
+ "status": event["event_params"]["terminal_status"],
+ "duration": event["event_params"]["duration_ms"],
+ }))
+ .collect::<Vec<_>>(),
+ vec![
+ json!({"event": "codex_command_execution_event", "type": expected, "status": "completed", "duration": 42}),
+ json!({"event": "codex_control_tool_call_event", "type": expected, "status": "completed", "duration": 42}),
+ ],
+ );
+ }
+}
+
+#[tokio::test]
+async fn tool_event_emitted_before_sampling_evidence_keeps_unknown_origin() {
+ let mut reducer = AnalyticsReducer::default();
+ let mut events = Vec::new();
+ ingest_review_prerequisites(&mut reducer, &mut events).await;
+ ingest_completed_command_execution_item(&mut reducer, &mut events, "thread-1", "call-1").await;
+ assert_eq!(events.len(), 1);
+ ingest_code_mode_facts(
+ &mut reducer,
+ &mut events,
+ [sampling_response("turn-1", "response-1", &["call-1"])],
+ )
+ .await;
+ reducer.flush(&mut events);
+ let payload = serde_json::to_value(&events).expect("serialize tool events");
+ assert_eq!(
+ payload
+ .as_array()
+ .expect("events")
+ .iter()
+ .map(|event| json!({
+ "item": event["event_params"]["item_id"],
+ "type": event["event_params"]["tool_event_type"],
+ "origin": event["event_params"]["originating_response_id"],
+ }))
+ .collect::<Vec<_>>(),
+ vec![json!({"item": "call-1", "type": null, "origin": null})],
+ );
+}
+
#[tokio::test]
async fn permissions_reviews_emit_events_without_denormalizing_onto_tool_items() {
let mut reducer = AnalyticsReducer::default();
@@ -4030,6 +4156,17 @@ async fn subagent_tool_items_inherit_parent_connection_metadata() {
)
.await;
+ ingest_code_mode_facts(
+ &mut reducer,
+ &mut events,
+ [CodeModeToolCallFact::SamplingResponseCompleted {
+ thread_id: "thread-subagent".into(),
+ turn_id: "turn-subagent".into(),
+ response_id: "response-subagent".into(),
+ tool_call_ids: vec!["item-1".into(), "exec-1".into()],
+ }],
+ )
+ .await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
@@ -4093,11 +4230,12 @@ async fn subagent_tool_items_inherit_parent_connection_metadata() {
.map(|event| json!({
"turn_id": event["event_params"]["turn_id"],
"root_turn_id": event["event_params"]["root_turn_id"],
+ "tool_event_type": event["event_params"]["tool_event_type"],
}))
.collect::<Vec<_>>(),
vec![
- json!({"turn_id": "turn-subagent", "root_turn_id": "child-causal-root"}),
- json!({"turn_id": "turn-subagent", "root_turn_id": "child-causal-root"}),
+ json!({"turn_id": "turn-subagent", "root_turn_id": "child-causal-root", "tool_event_type": "model_tool_call"}),
+ json!({"turn_id": "turn-subagent", "root_turn_id": "child-causal-root", "tool_event_type": "model_tool_call"}),
]
);
assert_eq!(payload[0]["event_type"], "codex_command_execution_event");
@@ -5544,8 +5682,9 @@ async fn turn_event_counts_completed_tool_items() {
mcp_tool_call_item(McpToolCallStatus::Completed, Some(2)),
ThreadItem::DynamicToolCall {
id: "dynamic-1".to_string(),
- namespace: None,
- tool: "render".to_string(),
+ // A name collision with code-mode exec must not change the event boundary.
+ namespace: Some("custom".to_string()),
+ tool: "exec".to_string(),
arguments: json!({}),
status: DynamicToolCallStatus::Completed,
content_items: None,
@@ -5661,22 +5800,31 @@ async fn turn_event_counts_completed_tool_items() {
event["event_params"]["root_turn_id"]
.as_str()
.expect("tool item event root turn ID"),
+ event["event_params"]["tool_event_type"].as_str(),
)
})
.collect::<Vec<_>>();
assert_eq!(
emitted_tool_events,
[
- "codex_command_execution_event",
- "codex_file_change_event",
- "codex_mcp_tool_call_event",
- "codex_dynamic_tool_call_event",
- "codex_web_search_event",
- "codex_image_generation_event",
- "codex_collab_agent_tool_call_event",
- "codex_control_tool_call_event",
+ ("codex_command_execution_event", None),
+ ("codex_file_change_event", None),
+ ("codex_mcp_tool_call_event", None),
+ ("codex_dynamic_tool_call_event", None),
+ ("codex_web_search_event", None),
+ ("codex_image_generation_event", None),
+ ("codex_collab_agent_tool_call_event", None),
+ ("codex_control_tool_call_event", None),
]
- .map(|event_type| (event_type, "session-thread-2", "turn-2", "root-ancestor"))
+ .map(|(event_type, tool_event_type)| {
+ (
+ event_type,
+ "session-thread-2",
+ "turn-2",
+ "root-ancestor",
+ tool_event_type,
+ )
+ })
.to_vec()
);
diff --git a/codex-rs/analytics/src/client_tests.rs b/codex-rs/analytics/src/client_tests.rs
index c54a97687d..b16700c8a4 100644
--- a/codex-rs/analytics/src/client_tests.rs
+++ b/codex-rs/analytics/src/client_tests.rs
@@ -216,6 +216,7 @@ fn sample_mcp_tool_call_event(thread_id: &str, plugin_id: Option<&str>) -> Track
subagent_source: None,
parent_thread_id: None,
tool_name: "search".to_string(),
+ tool_event_type: None,
started_at_ms: 1,
completed_at_ms: 2,
duration_ms: Some(1),
diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs
index d40651be98..b53453a10f 100644
--- a/codex-rs/analytics/src/events.rs
+++ b/codex-rs/analytics/src/events.rs
@@ -654,6 +654,16 @@ pub(crate) enum ToolItemFailureKind {
PolicyForbidden,
}
+/// The immediate initiator of the tool request associated with an analytics event.
+#[derive(Clone, Copy, Debug, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum ToolEventType {
+ /// The event item ID matches a tool call emitted in a sampled model response.
+ ModelToolCall,
+ /// The event item ID matches a child call dispatched by an executing tool.
+ InnerToolCall,
+}
+
#[derive(Serialize)]
pub(crate) struct CodexToolItemEventBase {
pub(crate) thread_id: String,
@@ -673,6 +683,8 @@ pub(crate) struct CodexToolItemEventBase {
pub(crate) subagent_source: Option<String>,
pub(crate) parent_thread_id: Option<String>,
pub(crate) tool_name: String,
+ /// Absent when origin evidence is unavailable or conflicting at emission time.
+ pub(crate) tool_event_type: Option<ToolEventType>,
pub(crate) started_at_ms: u64,
pub(crate) completed_at_ms: u64,
// Observed item lifecycle duration. This may undercount end-to-end execution
diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs
index a5e0892c6e..974d5d948b 100644
--- a/codex-rs/analytics/src/reducer.rs
+++ b/codex-rs/analytics/src/reducer.rs
@@ -60,6 +60,7 @@ use crate::events::ThreadArchiveEvent;
use crate::events::ThreadArchiveEventParams;
use crate::events::ThreadInitializedEvent;
use crate::events::ThreadInitializedEventParams;
+use crate::events::ToolEventType;
use crate::events::ToolItemFailureKind;
use crate::events::ToolItemTerminalStatus;
use crate::events::TrackEventRequest;
@@ -2569,6 +2570,16 @@ fn enrich_tool_response_event(
let Some(base) = tool_event_base_mut(event) else {
return;
};
+ // A cell association can also describe a separately sampled wait call. Classify
+ // only from evidence about this exact call ID, not its parent or response lineage.
+ base.tool_event_type = match (
+ state.response_ids_by_call_id.contains_key(&base.item_id),
+ state.cell_ids_by_child_call_id.contains_key(&base.item_id),
+ ) {
+ (true, false) => Some(ToolEventType::ModelToolCall),
+ (false, true) => Some(ToolEventType::InnerToolCall),
+ (false, false) | (true, true) => None,
+ };
if base.cell_id.is_none() {
base.cell_id = state.cell_ids_by_child_call_id.get(&base.item_id).cloned();
}
@@ -3034,6 +3045,7 @@ fn tool_item_base(
subagent_source: thread_metadata.subagent_source.clone(),
parent_thread_id: thread_metadata.parent_thread_id.clone(),
tool_name,
+ tool_event_type: None,
started_at_ms: context.started_at_ms,
completed_at_ms: context.completed_at_ms,
// duration_ms reflects item lifecycle observed by app-server. For web