Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Track built-in control tool calls in analytics (#39510)

jif992f5c681f0d
diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs
index 6791aee084..00ec2d8d4c 100644
--- a/codex-rs/analytics/src/client.rs
+++ b/codex-rs/analytics/src/client.rs
@@ -336,6 +336,12 @@ impl AnalyticsEventsClient {
));
}
+ pub fn track_control_tool_call(&self, input: crate::facts::ControlToolCallFact) {
+ self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::ControlToolCall(
+ input,
+ )));
+ }
+
pub fn track_guardian_review(
&self,
tracking: &GuardianReviewTrackContext,
diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs
index af431721eb..88b6870619 100644
--- a/codex-rs/analytics/src/events.rs
+++ b/codex-rs/analytics/src/events.rs
@@ -82,6 +82,7 @@ pub(crate) enum TrackEventRequest {
FileChange(CodexFileChangeEventRequest),
McpToolCall(CodexMcpToolCallEventRequest),
DynamicToolCall(CodexDynamicToolCallEventRequest),
+ ControlToolCall(CodexControlToolCallEventRequest),
CollabAgentToolCall(CodexCollabAgentToolCallEventRequest),
WebSearch(CodexWebSearchEventRequest),
ImageGeneration(CodexImageGenerationEventRequest),
@@ -827,6 +828,19 @@ pub(crate) struct CodexDynamicToolCallEventRequest {
pub(crate) event_params: CodexDynamicToolCallEventParams,
}
+#[derive(Serialize)]
+pub(crate) struct CodexControlToolCallEventParams {
+ #[serde(flatten)]
+ pub(crate) base: CodexToolItemEventBase,
+ pub(crate) success: bool,
+}
+
+#[derive(Serialize)]
+pub(crate) struct CodexControlToolCallEventRequest {
+ pub(crate) event_type: &'static str,
+ pub(crate) event_params: CodexControlToolCallEventParams,
+}
+
#[derive(Serialize)]
pub(crate) struct CodexCollabAgentToolCallEventParams {
#[serde(flatten)]
diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs
index f43acd9b06..3ca5b3327c 100644
--- a/codex-rs/analytics/src/facts.rs
+++ b/codex-rs/analytics/src/facts.rs
@@ -107,6 +107,26 @@ pub enum CodeModeToolCallStatus {
Interrupted,
}
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ControlToolCallFact {
+ pub thread_id: String,
+ pub turn_id: String,
+ pub call_id: String,
+ pub cell_id: Option<String>,
+ pub tool_name: String,
+ pub started_at_ms: u64,
+ pub completed_at_ms: u64,
+ pub status: ControlToolCallStatus,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ControlToolCallStatus {
+ Completed,
+ Failed,
+ Rejected,
+ Interrupted,
+}
+
pub fn build_track_events_context(
model_slug: String,
thread_id: String,
@@ -521,6 +541,7 @@ pub(crate) enum AnalyticsFact {
pub(crate) enum CustomAnalyticsFact {
ArtifactOperation(ArtifactOperationInput),
CodeModeToolCall(CodeModeToolCallFact),
+ ControlToolCall(ControlToolCallFact),
SubAgentThreadStarted(SubAgentThreadStartedInput),
Compaction(Box<CodexCompactionEvent>),
Goal(Box<CodexGoalEvent>),
diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs
index 7e7142abd3..942bfd2b95 100644
--- a/codex-rs/analytics/src/lib.rs
+++ b/codex-rs/analytics/src/lib.rs
@@ -38,6 +38,8 @@ pub use facts::CompactionReason;
pub use facts::CompactionStatus;
pub use facts::CompactionStrategy;
pub use facts::CompactionTrigger;
+pub use facts::ControlToolCallFact;
+pub use facts::ControlToolCallStatus;
pub use facts::ExternalAgentConfigImportCompletedInput;
pub use facts::ExternalAgentConfigImportFailureInput;
pub use facts::GoalEventKind;
diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs
index c7be5aab88..974658ec20 100644
--- a/codex-rs/analytics/src/reducer.rs
+++ b/codex-rs/analytics/src/reducer.rs
@@ -11,6 +11,8 @@ use crate::events::CodexCollabAgentToolCallEventRequest;
use crate::events::CodexCommandExecutionEventParams;
use crate::events::CodexCommandExecutionEventRequest;
use crate::events::CodexCompactionEventRequest;
+use crate::events::CodexControlToolCallEventParams;
+use crate::events::CodexControlToolCallEventRequest;
use crate::events::CodexDynamicToolCallEventParams;
use crate::events::CodexDynamicToolCallEventRequest;
use crate::events::CodexFileChangeEventParams;
@@ -82,6 +84,8 @@ use crate::facts::CodeModeToolCallFact;
use crate::facts::CodeModeToolCallStatus;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexGoalEvent;
+use crate::facts::ControlToolCallFact;
+use crate::facts::ControlToolCallStatus;
use crate::facts::CustomAnalyticsFact;
use crate::facts::ExternalAgentConfigImportCompletedInput;
use crate::facts::ExternalAgentConfigImportFailureInput;
@@ -589,6 +593,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::CodeModeToolCall(input) => {
self.ingest_code_mode_tool_call(input, out);
}
+ CustomAnalyticsFact::ControlToolCall(input) => {
+ self.ingest_control_tool_call(input, out);
+ }
CustomAnalyticsFact::SubAgentThreadStarted(input) => {
self.ingest_subagent_thread_started(input, out);
}
@@ -833,6 +840,83 @@ impl AnalyticsReducer {
}
}
+ fn ingest_control_tool_call(
+ &mut self,
+ input: ControlToolCallFact,
+ out: &mut Vec<TrackEventRequest>,
+ ) {
+ let ControlToolCallFact {
+ thread_id,
+ turn_id,
+ call_id,
+ cell_id,
+ tool_name,
+ started_at_ms,
+ completed_at_ms,
+ status,
+ } = input;
+ let drop_site = AnalyticsDropSite {
+ event_name: "control tool",
+ thread_id: &thread_id,
+ turn_id: Some(&turn_id),
+ review_id: None,
+ item_id: Some(&call_id),
+ };
+ let Some((connection_state, thread_state, thread_metadata)) =
+ self.thread_context_or_warn(drop_site)
+ else {
+ return;
+ };
+ let mut base = tool_item_base(
+ &thread_id,
+ &turn_id,
+ call_id,
+ tool_name,
+ ToolItemOutcome {
+ terminal_status: match status {
+ ControlToolCallStatus::Completed => ToolItemTerminalStatus::Completed,
+ ControlToolCallStatus::Failed => ToolItemTerminalStatus::Failed,
+ ControlToolCallStatus::Rejected => ToolItemTerminalStatus::Rejected,
+ ControlToolCallStatus::Interrupted => ToolItemTerminalStatus::Interrupted,
+ },
+ failure_kind: match status {
+ ControlToolCallStatus::Failed => Some(ToolItemFailureKind::ToolError),
+ ControlToolCallStatus::Rejected => Some(ToolItemFailureKind::PolicyForbidden),
+ ControlToolCallStatus::Completed | ControlToolCallStatus::Interrupted => None,
+ },
+ execution_duration_ms: observed_duration_ms(started_at_ms, completed_at_ms),
+ },
+ ToolItemContext {
+ started_at_ms,
+ completed_at_ms,
+ connection_state,
+ thread_state,
+ thread_metadata,
+ review_summary: None,
+ },
+ );
+ base.cell_id = cell_id;
+ let event = TrackEventRequest::ControlToolCall(CodexControlToolCallEventRequest {
+ event_type: "codex_control_tool_call_event",
+ event_params: CodexControlToolCallEventParams {
+ base,
+ success: status == ControlToolCallStatus::Completed,
+ },
+ });
+ self.turns
+ .entry(turn_id.clone())
+ .or_default()
+ .tool_counts
+ .total += 1;
+ self.record_tool_event(
+ &thread_id,
+ &turn_id,
+ event,
+ ToolEventEmission::AwaitResponse,
+ out,
+ );
+ }
+
fn ingest_sampling_response_completed(
&mut self,
thread_id: String,
@@ -2307,6 +2391,7 @@ fn tool_event_base_mut(event: &mut TrackEventRequest) -> Option<&mut CodexToolIt
TrackEventRequest::FileChange(event) => Some(&mut event.event_params.base),
TrackEventRequest::McpToolCall(event) => Some(&mut event.event_params.base),
TrackEventRequest::DynamicToolCall(event) => Some(&mut event.event_params.base),
+ TrackEventRequest::ControlToolCall(event) => Some(&mut event.event_params.base),
TrackEventRequest::CollabAgentToolCall(event) => Some(&mut event.event_params.base),
TrackEventRequest::WebSearch(event) => Some(&mut event.event_params.base),
TrackEventRequest::ImageGeneration(event) => Some(&mut event.event_params.base),
diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs
index 0e6f464228..cd107bf668 100644
--- a/codex-rs/app-server/tests/suite/v2/turn_start.rs
+++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs
@@ -95,6 +95,7 @@ use wiremock::ResponseTemplate;
use super::analytics::mount_analytics_capture;
use super::analytics::wait_for_analytics_event;
+use super::analytics::wait_for_matching_analytics_event;
#[cfg(windows)]
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25);
@@ -1109,18 +1110,50 @@ async fn code_mode_exec_emits_correlated_production_analytics() -> Result<()> {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<()> {
- let responses = vec![
+ let control_tools = [
+ ("call1", "request_user_input", json!({})),
+ (
+ "plan-call",
+ "update_plan",
+ json!({"plan": [{"step": "PRIVATE_PLAN", "status": "in_progress"}]}),
+ ),
+ (
+ "image-call",
+ "view_image",
+ json!({"path": "PRIVATE_IMAGE_PATH.png"}),
+ ),
+ ("get-goal-call", "get_goal", json!({})),
+ (
+ "create-goal-call",
+ "create_goal",
+ json!({"objective": "PRIVATE_GOAL"}),
+ ),
+ (
+ "update-goal-call",
+ "update_goal",
+ json!({"status": "complete"}),
+ ),
+ ];
+ let mut responses = vec![
create_request_user_input_sse_response("call1")?,
create_final_assistant_message_sse_response("Done")?,
];
+ responses.extend(control_tools.iter().skip(1).map(|(call_id, tool, args)| {
+ let response_id = format!("resp-{call_id}");
+ responses::sse(vec![
+ responses::ev_response_created(&response_id),
+ responses::ev_function_call(call_id, tool, &args.to_string()),
+ responses::ev_completed(&response_id),
+ ])
+ }));
+ responses.push(create_final_assistant_message_sse_response("Done")?);
let server = create_mock_responses_server_sequence(responses).await;
let codex_home = TempDir::new()?;
- write_mock_responses_config_toml_with_chatgpt_base_url(
- codex_home.path(),
- &server.uri(),
- &server.uri(),
- )?;
+ MockResponsesConfig::new(&server.uri())
+ .enable_feature(Feature::Goals)
+ .with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri()))
+ .write(codex_home.path())?;
mount_analytics_capture(&server, codex_home.path()).await?;
let mut mcp = TestAppServer::builder()
@@ -1193,16 +1226,90 @@ async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<()
.is_some_and(|duration| duration > 0),
"samplingRequestCount": params["sampling_request_count"],
"samplingRetryCount": params["sampling_retry_count"],
+ "totalToolCalls": params["total_tool_call_count"],
+ "dynamicToolCalls": params["dynamic_tool_call_count"],
"status": params["status"],
}),
json!({
"toolBlockingIsPositive": true,
"samplingRequestCount": 2,
"samplingRetryCount": 0,
+ "totalToolCalls": 1,
+ "dynamicToolCalls": 0,
"status": "completed",
})
);
+ let second_turn = mcp
+ .start_turn_and_wait_for_completion(TurnStartParams {
+ thread_id: thread.id.clone(),
+ input: vec![V2UserInput::Text {
+ text: "run the other control tools".to_string(),
+ text_elements: Vec::new(),
+ }],
+ collaboration_mode: Some(CollaborationMode {
+ mode: ModeKind::Default,
+ settings: Settings {
+ model: "mock-model".to_string(),
+ reasoning_effort: Some(ReasoningEffort::Medium),
+ developer_instructions: None,
+ },
+ }),
+ ..Default::default()
+ })
+ .await?;
+
+ for (call_id, tool_name, _) in control_tools {
+ let event = wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| {
+ event["event_type"] == "codex_control_tool_call_event"
+ && event["event_params"]["item_id"] == call_id
+ })
+ .await?;
+ let success = tool_name != "view_image";
+ assert_eq!(
+ json!({
+ "tool": event["event_params"]["tool_name"],
+ "success": event["event_params"]["success"],
+ "status": event["event_params"]["terminal_status"],
+ "hasOrigin": event["event_params"]["originating_response_id"]
+ .as_str()
+ .is_some(),
+ }),
+ json!({
+ "tool": tool_name,
+ "success": success,
+ "status": if success { "completed" } else { "failed" },
+ "hasOrigin": true,
+ })
+ );
+ let serialized = event.to_string();
+ assert!(
+ ![
+ "PRIVATE_PLAN",
+ "PRIVATE_IMAGE_PATH",
+ "PRIVATE_GOAL",
+ "Proceed with the plan?"
+ ]
+ .iter()
+ .any(|private_input| serialized.contains(private_input)),
+ "tool-call analytics must not contain private tool arguments: {serialized}"
+ );
+ }
+
+ let second_turn_event =
+ wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| {
+ event["event_type"] == "codex_turn_event"
+ && event["event_params"]["turn_id"] == second_turn.turn.id
+ })
+ .await?;
+ assert_eq!(
+ json!({
+ "total": second_turn_event["event_params"]["total_tool_call_count"],
+ "dynamic": second_turn_event["event_params"]["dynamic_tool_call_count"],
+ }),
+ json!({"total": 5, "dynamic": 0})
+ );
+
Ok(())
}
diff --git a/codex-rs/core/src/tools/control_tool_analytics.rs b/codex-rs/core/src/tools/control_tool_analytics.rs
new file mode 100644
index 0000000000..46719201e6
--- /dev/null
+++ b/codex-rs/core/src/tools/control_tool_analytics.rs
@@ -0,0 +1,52 @@
+use crate::tools::context::ToolCallSource;
+use crate::tools::context::ToolInvocation;
+use codex_analytics::ControlToolCallFact;
+use codex_analytics::ControlToolCallStatus;
+
+pub(crate) struct ControlToolCallGuard<'a> {
+ invocation: &'a ToolInvocation,
+ started_at_ms: u64,
+ completed_at_ms: Option<u64>,
+ status: ControlToolCallStatus,
+}
+
+impl<'a> ControlToolCallGuard<'a> {
+ pub(crate) fn new(invocation: &'a ToolInvocation) -> Self {
+ Self {
+ invocation,
+ started_at_ms: codex_analytics::now_unix_millis(),
+ completed_at_ms: None,
+ status: ControlToolCallStatus::Interrupted,
+ }
+ }
+
+ pub(crate) fn finish(&mut self, status: ControlToolCallStatus) {
+ self.completed_at_ms = Some(codex_analytics::now_unix_millis());
+ self.status = status;
+ }
+}
+
+impl Drop for ControlToolCallGuard<'_> {
+ fn drop(&mut self) {
+ let invocation = self.invocation;
+ invocation
+ .session
+ .services
+ .analytics_events_client
+ .track_control_tool_call(ControlToolCallFact {
+ thread_id: invocation.session.thread_id.to_string(),
+ turn_id: invocation.turn.sub_id.clone(),
+ call_id: invocation.call_id.clone(),
+ cell_id: match &invocation.source {
+ ToolCallSource::CodeMode { cell_id, .. } => Some(cell_id.clone()),
+ ToolCallSource::Direct | ToolCallSource::DirectPlaintextMessage => None,
+ },
+ tool_name: invocation.tool_name.name.clone(),
+ started_at_ms: self.started_at_ms,
+ completed_at_ms: self
+ .completed_at_ms
+ .unwrap_or_else(codex_analytics::now_unix_millis),
+ status: self.status,
+ });
+ }
+}
diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs
index 87dc39563d..57b6e51e95 100644
--- a/codex-rs/core/src/tools/handlers/extension_tools.rs
+++ b/codex-rs/core/src/tools/handlers/extension_tools.rs
@@ -61,6 +61,15 @@ impl ToolExecutor<ToolInvocation> for ExtensionToolAdapter {
}
impl CoreToolRuntime for ExtensionToolAdapter {
+ fn is_builtin_control_tool(&self) -> bool {
+ let tool_name = self.0.tool_name();
+ tool_name.is_default_namespace()
+ && matches!(
+ tool_name.name.as_str(),
+ "get_goal" | "create_goal" | "update_goal"
+ )
+ }
+
fn matches_kind(&self, payload: &ToolPayload) -> bool {
match payload {
ToolPayload::Function { .. } => true,
diff --git a/codex-rs/core/src/tools/handlers/plan.rs b/codex-rs/core/src/tools/handlers/plan.rs
index 33b91aa512..d962314752 100644
--- a/codex-rs/core/src/tools/handlers/plan.rs
+++ b/codex-rs/core/src/tools/handlers/plan.rs
@@ -96,7 +96,11 @@ impl PlanHandler {
}
}
-impl CoreToolRuntime for PlanHandler {}
+impl CoreToolRuntime for PlanHandler {
+ fn is_builtin_control_tool(&self) -> bool {
+ true
+ }
+}
fn parse_update_plan_arguments(arguments: &str) -> Result<UpdatePlanArgs, FunctionCallError> {
serde_json::from_str::<UpdatePlanArgs>(arguments).map_err(|e| {
diff --git a/codex-rs/core/src/tools/handlers/request_user_input.rs b/codex-rs/core/src/tools/handlers/request_user_input.rs
index 92e36fc668..f586f7afd4 100644
--- a/codex-rs/core/src/tools/handlers/request_user_input.rs
+++ b/codex-rs/core/src/tools/handlers/request_user_input.rs
@@ -98,7 +98,11 @@ impl RequestUserInputHandler {
}
}
-impl CoreToolRuntime for RequestUserInputHandler {}
+impl CoreToolRuntime for RequestUserInputHandler {
+ fn is_builtin_control_tool(&self) -> bool {
+ true
+ }
+}
#[cfg(test)]
#[path = "request_user_input_tests.rs"]
diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs
index 5e6bf3b0b4..ee8684350e 100644
--- a/codex-rs/core/src/tools/handlers/view_image.rs
+++ b/codex-rs/core/src/tools/handlers/view_image.rs
@@ -208,7 +208,11 @@ impl ViewImageHandler {
}
}
-impl CoreToolRuntime for ViewImageHandler {}
+impl CoreToolRuntime for ViewImageHandler {
+ fn is_builtin_control_tool(&self) -> bool {
+ true
+ }
+}
pub struct ViewImageOutput {
image_url: String,
diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs
index 3ca459a83e..5c29628079 100644
--- a/codex-rs/core/src/tools/mod.rs
+++ b/codex-rs/core/src/tools/mod.rs
@@ -1,6 +1,7 @@
mod approvals;
pub(crate) mod code_mode;
pub(crate) mod context;
+mod control_tool_analytics;
pub(crate) mod events;
mod executed_tool_calls;
pub(crate) mod handlers;
diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs
index c02f3dec74..e7c416d077 100644
--- a/codex-rs/core/src/tools/registry.rs
+++ b/codex-rs/core/src/tools/registry.rs
@@ -19,6 +19,7 @@ use crate::tools::context::FunctionToolOutput;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolOutput;
use crate::tools::context::ToolPayload;
+use crate::tools::control_tool_analytics::ControlToolCallGuard;
use crate::tools::flat_tool_name;
use crate::tools::handlers::multi_agents_spec::MULTI_AGENT_V1_NAMESPACE;
use crate::tools::hook_names::HookToolName;
@@ -27,6 +28,7 @@ use crate::tools::lifecycle::notify_tool_start;
use crate::tools::router::tool_log_payload;
use crate::tools::tool_dispatch_trace::ToolDispatchTrace;
use crate::util::error_or_panic;
+use codex_analytics::ControlToolCallStatus;
use codex_extension_api::ToolCallOutcome;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
@@ -51,6 +53,11 @@ pub use codex_tools::ToolExposure;
/// Implementers provide the shared `ToolExecutor` behavior plus optional
/// core-owned metadata for hooks, telemetry, tool search, and argument diffs.
pub(crate) trait CoreToolRuntime: ToolExecutor<ToolInvocation> {
+ /// Whether this built-in control tool needs a structured tool-call event.
+ fn is_builtin_control_tool(&self) -> bool {
+ false
+ }
+
/// Returns a shared spec when both the spec and search metadata are immutable.
fn immutable_spec(&self) -> Option<&Arc<ToolSpec>> {
None
@@ -577,6 +584,10 @@ impl ToolRegistry {
.await
{
PreToolUseHookResult::Blocked(message) => {
+ if tool.is_builtin_control_tool() {
+ let mut analytics = ControlToolCallGuard::new(&invocation);
+ analytics.finish(ControlToolCallStatus::Rejected);
+ }
let err = FunctionCallError::RespondToModel(message);
dispatch_trace.record_failed(&err);
notify_tool_finish_if_unclaimed(
@@ -594,6 +605,10 @@ impl ToolRegistry {
invocation = updated_invocation;
}
Err(err) => {
+ if tool.is_builtin_control_tool() {
+ let mut analytics = ControlToolCallGuard::new(&invocation);
+ analytics.finish(ControlToolCallStatus::Failed);
+ }
dispatch_trace.record_failed(&err);
notify_tool_finish_if_unclaimed(
&invocation,
@@ -613,6 +628,9 @@ impl ToolRegistry {
}
notify_tool_start(&invocation).await;
+ let mut control_tool_analytics = tool
+ .is_builtin_control_tool()
+ .then(|| ControlToolCallGuard::new(&invocation));
if let Some(command) = shell_script_for_invocation(&invocation) {
let parsed = parse_shell_script(&command);
@@ -663,6 +681,13 @@ impl ToolRegistry {
Ok((_, success)) => *success,
Err(_) => false,
};
+ if let Some(analytics) = control_tool_analytics.as_mut() {
+ analytics.finish(if success {
+ ControlToolCallStatus::Completed
+ } else {
+ ControlToolCallStatus::Failed
+ });
+ }
emit_metric_for_tool_read(&invocation, success);
let post_tool_use_payload = if success {
let guard = response_cell.lock().await;