Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add a critical-path timing recorder for direct tool calls (#56008)

taohong-oai082900149cc7
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index 40612050e7..d980227377 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -3166,6 +3166,14 @@ dependencies = [
"pretty_assertions",
]
+[[package]]
+name = "codex-direct-tool-timing"
+version = "0.0.0"
+dependencies = [
+ "codex-utils-critical-path",
+ "pretty_assertions",
+]
+
[[package]]
name = "codex-exec"
version = "0.0.0"
diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml
index d1cf32be45..aa7fa3ce89 100644
--- a/codex-rs/Cargo.toml
+++ b/codex-rs/Cargo.toml
@@ -48,6 +48,7 @@ members = [
"core-plugin-common",
"core-plugins",
"diagnostics",
+ "direct-tool-timing",
"guardian-context",
"hooks",
"history",
@@ -299,6 +300,7 @@ codex-utils-audio = { path = "utils/audio" }
codex-utils-cache = { path = "utils/cache" }
codex-utils-cargo-bin = { path = "utils/cargo-bin" }
codex-utils-cli = { path = "utils/cli" }
+codex-utils-critical-path = { path = "utils/critical-path" }
codex-utils-elapsed = { path = "utils/elapsed" }
codex-utils-fuzzy-match = { path = "utils/fuzzy-match" }
codex-utils-git-discovery = { path = "utils/git-discovery" }
diff --git a/codex-rs/direct-tool-timing/BUILD.bazel b/codex-rs/direct-tool-timing/BUILD.bazel
new file mode 100644
index 0000000000..5c5854e460
--- /dev/null
+++ b/codex-rs/direct-tool-timing/BUILD.bazel
@@ -0,0 +1,6 @@
+load("//:defs.bzl", "codex_rust_crate")
+
+codex_rust_crate(
+ name = "direct-tool-timing",
+ crate_name = "codex_direct_tool_timing",
+)
diff --git a/codex-rs/direct-tool-timing/Cargo.toml b/codex-rs/direct-tool-timing/Cargo.toml
new file mode 100644
index 0000000000..82a8aadac3
--- /dev/null
+++ b/codex-rs/direct-tool-timing/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "codex-direct-tool-timing"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[lints]
+workspace = true
+
+[lib]
+doctest = false
+
+[dependencies]
+codex-utils-critical-path = { workspace = true }
+
+[dev-dependencies]
+pretty_assertions = { workspace = true }
diff --git a/codex-rs/direct-tool-timing/src/direct_tool_timing_tests.rs b/codex-rs/direct-tool-timing/src/direct_tool_timing_tests.rs
new file mode 100644
index 0000000000..fb738af8c6
--- /dev/null
+++ b/codex-rs/direct-tool-timing/src/direct_tool_timing_tests.rs
@@ -0,0 +1,238 @@
+use super::*;
+use codex_utils_critical_path::PathDurations;
+use pretty_assertions::assert_eq;
+use std::time::Duration;
+
+#[test]
+fn parallel_winner_retains_its_review_and_shared_initial_path() {
+ let now = Instant::now();
+ let start = now - Duration::from_secs(10);
+ let shared_initial_path =
+ CriticalPath::default().work(start, start + Duration::from_secs(1), WorkKind::Inference);
+ let batch = ToolBatchTiming::default();
+ let timings = DirectToolTimings::default();
+ let losing = timings.start_direct_call(
+ "loser",
+ batch.clone(),
+ start + Duration::from_secs(1),
+ Some(shared_initial_path.clone()),
+ );
+ let winner = timings.start_direct_call(
+ "winner",
+ batch.clone(),
+ start + Duration::from_secs(1),
+ Some(shared_initial_path),
+ );
+ {
+ let mut state = timings.state();
+ let work = state.unique_work("winner").unwrap();
+ work.record_until(start + Duration::from_secs(2));
+ work.active_auto_reviews = 1;
+ work.record_until(start + Duration::from_secs(5));
+ work.active_auto_reviews = 0;
+ }
+ drop(losing);
+ assert!(batch.completed().is_none());
+ drop(winner);
+ let completed = batch.completed().unwrap();
+ assert_eq!(
+ completed.path.unwrap().durations(start, completed.ended_at),
+ PathDurations {
+ inference: Duration::from_secs(1),
+ auto_review: Duration::from_secs(3),
+ tools: completed.ended_at.duration_since(start) - Duration::from_secs(4),
+ ..PathDurations::default()
+ }
+ );
+}
+
+#[test]
+fn duplicate_ids_and_late_review_drop_do_not_complete_another_invocation() {
+ let timings = DirectToolTimings::default();
+ let batch = ToolBatchTiming::default();
+ let start = Instant::now();
+ let first =
+ timings.start_direct_call("same", batch.clone(), start, Some(CriticalPath::default()));
+ let review = first.start_review().unwrap();
+ let second =
+ timings.start_direct_call("same", batch.clone(), start, Some(CriticalPath::default()));
+
+ drop(first);
+ drop(review);
+ assert_eq!(
+ timings
+ .state()
+ .unique_work("same")
+ .unwrap()
+ .active_auto_reviews,
+ 0
+ );
+ assert!(batch.completed().is_none());
+ drop(second);
+ assert!(batch.completed().unwrap().path.is_none());
+ assert!(timings.state().calls.is_empty());
+}
+
+#[test]
+fn late_review_start_does_not_attach_to_reused_call_id() {
+ let timings = DirectToolTimings::default();
+ let batch = ToolBatchTiming::default();
+ let first = timings.start_direct_call(
+ "same",
+ batch.clone(),
+ Instant::now(),
+ Some(CriticalPath::default()),
+ );
+ let review = first.start_review().unwrap();
+ first.finish().unwrap();
+ let started_at = Instant::now();
+ let second = timings.start_direct_call(
+ "same",
+ batch.clone(),
+ started_at,
+ Some(CriticalPath::default()),
+ );
+ assert!(first.start_review().is_none());
+ drop(review);
+ drop(first);
+ assert!(batch.completed().is_none());
+ drop(second);
+ let completed = batch.completed().unwrap();
+ assert_eq!(
+ completed
+ .path
+ .unwrap()
+ .durations(started_at, completed.ended_at),
+ PathDurations {
+ tools: completed.ended_at.duration_since(started_at),
+ ..PathDurations::default()
+ }
+ );
+ assert!(timings.state().calls.is_empty());
+}
+
+#[test]
+fn cancellation_completes_once_and_review_can_outlive_call() {
+ let timings = DirectToolTimings::default();
+ let batch = ToolBatchTiming::default();
+ let work = timings.start_direct_call(
+ "cancelled",
+ batch.clone(),
+ Instant::now(),
+ Some(CriticalPath::default()),
+ );
+ let review = work.start_review().unwrap();
+ let first = work.finish().unwrap();
+ drop(work);
+ drop(review);
+ assert_eq!(batch.completed().unwrap().ended_at, first.ended_at);
+ assert!(timings.state().calls.is_empty());
+}
+
+#[test]
+fn missing_dependency_remains_unknown() {
+ let start = Instant::now() - Duration::from_secs(1);
+ let timings = DirectToolTimings::default();
+ let batch = ToolBatchTiming::default();
+ let work =
+ timings.start_direct_call("call", batch.clone(), start, Some(CriticalPath::default()));
+ timings.join(
+ "call",
+ start,
+ start + Duration::from_millis(500),
+ /*dependency*/ None,
+ );
+ drop(work);
+ let completed = batch.completed().unwrap();
+ assert_eq!(
+ completed
+ .path
+ .unwrap()
+ .durations(start, completed.ended_at)
+ .unknown,
+ Duration::from_millis(500)
+ );
+}
+
+#[test]
+fn human_wait_takes_precedence_over_reviews_and_overlapping_waits() {
+ let start = Instant::now();
+ let at = |seconds| start + Duration::from_secs(seconds);
+ let timings = DirectToolTimings::default();
+ let batch = ToolBatchTiming::default();
+ let _guard = timings.start_direct_call("call", batch, start, Some(CriticalPath::default()));
+ let mut state = timings.state();
+ let work = state.unique_work("call").unwrap();
+ work.record_until(at(1));
+ work.active_auto_reviews = 1;
+ work.record_until(at(2));
+ work.active_user_waits = 1;
+ work.record_until(at(3));
+ work.active_user_waits = 2;
+ work.record_until(at(4));
+ work.active_user_waits = 1;
+ work.record_until(at(5));
+ work.active_user_waits = 0;
+ work.record_until(at(6));
+ work.active_auto_reviews = 0;
+ work.record_until(at(7));
+ assert_eq!(
+ work.path.as_ref().unwrap().durations(start, at(7)),
+ PathDurations {
+ tools: Duration::from_secs(2),
+ auto_review: Duration::from_secs(2),
+ user_wait: Duration::from_secs(3),
+ ..PathDurations::default()
+ }
+ );
+}
+
+#[test]
+fn user_wait_guards_balance_and_do_not_follow_reused_call_ids() {
+ let timings = DirectToolTimings::default();
+ let batch = ToolBatchTiming::default();
+ let first = timings.start_direct_call(
+ "call",
+ batch.clone(),
+ Instant::now(),
+ Some(CriticalPath::default()),
+ );
+ let first_wait = first.start_user_wait().unwrap();
+ let nested_wait = first.start_user_wait().unwrap();
+ assert_eq!(
+ timings
+ .state()
+ .unique_work("call")
+ .unwrap()
+ .active_user_waits,
+ 2
+ );
+ drop(nested_wait);
+ assert_eq!(
+ timings
+ .state()
+ .unique_work("call")
+ .unwrap()
+ .active_user_waits,
+ 1
+ );
+ first.finish().unwrap();
+ let next = timings.start_direct_call(
+ "call",
+ batch.clone(),
+ Instant::now(),
+ Some(CriticalPath::default()),
+ );
+ assert!(first.start_user_wait().is_none());
+ drop(first_wait);
+ assert_eq!(
+ timings
+ .state()
+ .unique_work("call")
+ .unwrap()
+ .active_user_waits,
+ 0
+ );
+ drop(next);
+ assert!(batch.completed().is_some());
+}
diff --git a/codex-rs/direct-tool-timing/src/lib.rs b/codex-rs/direct-tool-timing/src/lib.rs
new file mode 100644
index 0000000000..4b007deefb
--- /dev/null
+++ b/codex-rs/direct-tool-timing/src/lib.rs
@@ -0,0 +1,7 @@
+mod recorder;
+
+pub use recorder::CompletedWork;
+pub use recorder::DirectToolTimings;
+pub use recorder::ToolBatchTiming;
+pub use recorder::ToolBlockingTimingGuard;
+pub use recorder::ToolTimingGuard;
diff --git a/codex-rs/direct-tool-timing/src/recorder.rs b/codex-rs/direct-tool-timing/src/recorder.rs
new file mode 100644
index 0000000000..5ae32e768a
--- /dev/null
+++ b/codex-rs/direct-tool-timing/src/recorder.rs
@@ -0,0 +1,286 @@
+//! Records direct-tool invocations, automated reviews, human waits, and batch completion.
+//!
+//! Prefixes and dependency paths can contain any work kind; this recorder adds tool and
+//! review and human-wait intervals, or Unknown for a dependency wait whose history is unavailable.
+//! Review callbacks must retain their invocation guard and bracket blocking automated
+//! reviews with the review guard's lifetime. Execution-lock admission timing is separate.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::Mutex;
+use std::time::Instant;
+
+use codex_utils_critical_path::CriticalPath;
+use codex_utils_critical_path::WorkKind;
+
+/// Completion time and dependency history for one direct invocation.
+#[derive(Clone, Debug)]
+pub struct CompletedWork {
+ pub ended_at: Instant,
+ pub path: Option<CriticalPath>,
+}
+
+/// Tracks the last-finishing invocation after every invocation in a batch completes.
+#[derive(Clone, Default)]
+pub struct ToolBatchTiming(Arc<Mutex<BatchState>>);
+
+#[derive(Default)]
+struct BatchState {
+ pending: usize,
+ latest: Option<CompletedWork>,
+}
+
+impl ToolBatchTiming {
+ /// Returns the batch result only when no invocation remains active.
+ pub fn completed(&self) -> Option<CompletedWork> {
+ let state = self
+ .0
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ (state.pending == 0).then(|| state.latest.clone()).flatten()
+ }
+
+ fn started(&self) {
+ self.0
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .pending += 1;
+ }
+
+ fn record(&self, work: CompletedWork) {
+ let mut state = self
+ .0
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ state.pending -= 1;
+ if state
+ .latest
+ .as_ref()
+ .is_none_or(|previous| previous.ended_at < work.ended_at)
+ {
+ state.latest = Some(work);
+ }
+ }
+}
+
+/// Owns direct invocations; duplicate external call IDs invalidate attribution.
+#[derive(Clone, Default)]
+pub struct DirectToolTimings(Arc<Mutex<State>>);
+
+#[derive(Default)]
+struct State {
+ calls: HashMap<String, Vec<DirectToolInvocation>>,
+}
+
+struct DirectToolInvocation {
+ identity: Arc<()>,
+ path: Option<CriticalPath>,
+ segment_started_at: Instant,
+ active_auto_reviews: usize,
+ active_user_waits: usize,
+ batch: ToolBatchTiming,
+}
+
+/// Finishes this invocation on drop; later blocking-guard drops cannot affect another call.
+pub struct ToolTimingGuard {
+ identity: Arc<()>,
+ timings: DirectToolTimings,
+ call_id: String,
+}
+
+#[derive(Clone, Copy)]
+enum BlockingKind {
+ AutoReview,
+ UserWait,
+}
+
+/// Records a blocking interval until dropped; user waits take precedence over reviews.
+pub struct ToolBlockingTimingGuard {
+ kind: BlockingKind,
+ identity: Arc<()>,
+ timings: DirectToolTimings,
+ call_id: String,
+}
+
+impl DirectToolInvocation {
+ fn record_until(&mut self, now: Instant) {
+ // A human answer can block an automated review. Attribute that overlap once,
+ // to the human wait; all remaining invocation time is elapsed tool work.
+ let kind = if self.active_user_waits > 0 {
+ WorkKind::UserWait
+ } else if self.active_auto_reviews > 0 {
+ WorkKind::AutoReview
+ } else {
+ WorkKind::Tools
+ };
+ self.path = self
+ .path
+ .take()
+ .map(|path| path.work(self.segment_started_at, now, kind));
+ self.segment_started_at = now;
+ }
+
+ fn join(&mut self, start: Instant, end: Instant, dependency: Option<CriticalPath>) {
+ self.record_until(start);
+ self.path = self.path.take().map(|path| match dependency {
+ Some(dependency) => path.join(start, end, dependency),
+ None => path.work(start, end, WorkKind::Unknown),
+ });
+ self.segment_started_at = end;
+ }
+}
+
+impl DirectToolTimings {
+ fn state(&self) -> std::sync::MutexGuard<'_, State> {
+ self.0
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ }
+
+ /// Starts an invocation with its causal prefix, or no attribution when unavailable.
+ pub fn start_direct_call(
+ &self,
+ call_id: &str,
+ batch: ToolBatchTiming,
+ started_at: Instant,
+ prefix: Option<CriticalPath>,
+ ) -> ToolTimingGuard {
+ batch.started();
+ let path = prefix.map(|path| {
+ let previous = path.end().unwrap_or(started_at);
+ path.work(previous, started_at, WorkKind::Tools)
+ });
+ let identity = Arc::new(());
+ let mut state = self.state();
+ let calls = state.calls.entry(call_id.to_string()).or_default();
+ let path = if calls.is_empty() {
+ path
+ } else {
+ // External call IDs are not guaranteed unique. Keep each guard's
+ // completion separate, but do not guess which invocation a lookup names.
+ for work in calls.iter_mut() {
+ work.path = None;
+ }
+ None
+ };
+ calls.push(DirectToolInvocation {
+ identity: Arc::clone(&identity),
+ path,
+ segment_started_at: started_at,
+ active_auto_reviews: 0,
+ active_user_waits: 0,
+ batch,
+ });
+ ToolTimingGuard {
+ identity,
+ timings: self.clone(),
+ call_id: call_id.to_string(),
+ }
+ }
+
+ /// Records a dependency wait for a unique call ID; missing history becomes unknown time.
+ pub fn join(
+ &self,
+ call_id: &str,
+ start: Instant,
+ end: Instant,
+ dependency: Option<CriticalPath>,
+ ) {
+ if let Some(work) = self.state().unique_work(call_id) {
+ work.join(start, end, dependency);
+ }
+ }
+
+ fn finish(&self, call_id: &str, identity: &Arc<()>) -> Option<CompletedWork> {
+ let mut state = self.state();
+ let calls = state.calls.get_mut(call_id)?;
+ let index = calls
+ .iter()
+ .position(|work| Arc::ptr_eq(&work.identity, identity))?;
+ let mut work = calls.swap_remove(index);
+ if calls.is_empty() {
+ state.calls.remove(call_id);
+ }
+ let ended_at = Instant::now();
+ work.record_until(ended_at);
+ let completed = CompletedWork {
+ ended_at,
+ path: work.path,
+ };
+ work.batch.record(completed.clone());
+ Some(completed)
+ }
+}
+
+impl State {
+ fn unique_work(&mut self, call_id: &str) -> Option<&mut DirectToolInvocation> {
+ let [work] = self.calls.get_mut(call_id)?.as_mut_slice() else {
+ return None;
+ };
+ Some(work)
+ }
+}
+
+impl ToolTimingGuard {
+ /// Starts a review only for this invocation, including after external call ID reuse.
+ pub fn start_review(&self) -> Option<ToolBlockingTimingGuard> {
+ self.start_blocking(BlockingKind::AutoReview)
+ }
+
+ /// Starts a human wait for this invocation, taking precedence over automated review.
+ pub fn start_user_wait(&self) -> Option<ToolBlockingTimingGuard> {
+ self.start_blocking(BlockingKind::UserWait)
+ }
+
+ fn start_blocking(&self, kind: BlockingKind) -> Option<ToolBlockingTimingGuard> {
+ let mut state = self.timings.state();
+ let work = state
+ .calls
+ .get_mut(&self.call_id)?
+ .iter_mut()
+ .find(|work| Arc::ptr_eq(&work.identity, &self.identity))?;
+ work.record_until(Instant::now());
+ match kind {
+ BlockingKind::AutoReview => work.active_auto_reviews += 1,
+ BlockingKind::UserWait => work.active_user_waits += 1,
+ }
+ Some(ToolBlockingTimingGuard {
+ kind,
+ identity: Arc::clone(&self.identity),
+ timings: self.timings.clone(),
+ call_id: self.call_id.clone(),
+ })
+ }
+
+ /// Completes this invocation once; subsequent calls and drop have no effect.
+ pub fn finish(&self) -> Option<CompletedWork> {
+ self.timings.finish(&self.call_id, &self.identity)
+ }
+}
+
+impl Drop for ToolTimingGuard {
+ fn drop(&mut self) {
+ self.finish();
+ }
+}
+
+impl Drop for ToolBlockingTimingGuard {
+ fn drop(&mut self) {
+ if let Some(calls) = self.timings.state().calls.get_mut(&self.call_id)
+ && let Some(work) = calls
+ .iter_mut()
+ .find(|work| Arc::ptr_eq(&work.identity, &self.identity))
+ {
+ work.record_until(Instant::now());
+ let counter = match self.kind {
+ BlockingKind::AutoReview => &mut work.active_auto_reviews,
+ BlockingKind::UserWait => &mut work.active_user_waits,
+ };
+ *counter = counter.saturating_sub(1);
+ }
+ }
+}
+
+#[cfg(test)]
+#[path = "direct_tool_timing_tests.rs"]
+mod tests;