Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): bound recording turn state capture and add start_recording budget options (#4215)

Francesco Bonacci5b3d48dfda23
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools-linux.mdx b/docs/content/docs/reference/cua-driver/mcp-tools-linux.mdx
index de38cd891f..51968955bd 100644
--- a/docs/content/docs/reference/cua-driver/mcp-tools-linux.mdx
+++ b/docs/content/docs/reference/cua-driver/mcp-tools-linux.mdx
@@ -573,6 +573,8 @@ Start trajectory recording. Every subsequent action-tool invocation (click, righ
- `action.json` — tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.
- `click.png` — for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.
+The per-turn accessibility walk is bounded like `get_window_state`: `state_timeout_ms` (default 1000) caps each before/after walk, a walk that runs out of budget records the PARTIAL tree, and `evidence.json` carries `truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `timeout_ms` for that phase. A provider that stops answering is abandoned after the budget plus a short grace and the phase is classified `state_capture_timeout`. Actions refused before dispatch (for example an unknown or expired `capture_id`) skip the state walk; their state is classified `not_applicable` / `action_refused_before_dispatch`. Pass `include_accessibility_tree: false` to record screenshots and actions without state.
+
Turn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started.
**Video is off by default.** Pass `record_video: true` to also capture the main display to `<output_dir>/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.
@@ -585,8 +587,10 @@ State persists for the life of the daemon; a restart resets to disabled with no
**Arguments:**
+- `include_accessibility_tree` (boolean, optional): Default true. Set false to skip the per-turn before/after accessibility walks entirely; screenshots, click markers and action.json are still recorded and state is classified `state_capture_disabled`. default: `true`
- `output_dir` (string, required): Absolute or ~-rooted directory where turn folders and (when enabled) the video file are written.
- `record_video` (boolean, optional): Capture the main display to &lt;output_dir&gt;/recording.mp4. Default: false. Set to true to also capture the main display to recording.mp4 (otherwise only the per-turn screenshots + JSON are recorded). On macOS this uses native ScreenCaptureKit (macOS 15.0+); macOS 26 can show a one-time direct screen-capture consent on first use (see `cua-driver permissions grant`). On Windows + Linux it requires ffmpeg on PATH.
+- `state_timeout_ms` (integer, optional): Wall-clock budget in milliseconds for EACH per-turn before/after accessibility walk (same default and bounds as get_window_state's timeout_ms). A walk that runs out records the partial tree and marks it truncated in evidence.json. default: `1000`; range: 100–120000
```json
{"output_dir":"~/cua-trajectories/demo1"}
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools-windows.mdx b/docs/content/docs/reference/cua-driver/mcp-tools-windows.mdx
index 2555ed3942..11de907666 100644
--- a/docs/content/docs/reference/cua-driver/mcp-tools-windows.mdx
+++ b/docs/content/docs/reference/cua-driver/mcp-tools-windows.mdx
@@ -665,6 +665,8 @@ Start trajectory recording. Every subsequent action-tool invocation (click, righ
- `action.json` — tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.
- `click.png` — for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.
+The per-turn accessibility walk is bounded like `get_window_state`: `state_timeout_ms` (default 1000) caps each before/after walk, a walk that runs out of budget records the PARTIAL tree, and `evidence.json` carries `truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `timeout_ms` for that phase. A provider that stops answering is abandoned after the budget plus a short grace and the phase is classified `state_capture_timeout`. Actions refused before dispatch (for example an unknown or expired `capture_id`) skip the state walk; their state is classified `not_applicable` / `action_refused_before_dispatch`. Pass `include_accessibility_tree: false` to record screenshots and actions without state.
+
Turn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started.
**Video is off by default.** Pass `record_video: true` to also capture the main display to `<output_dir>/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.
@@ -677,8 +679,10 @@ State persists for the life of the daemon; a restart resets to disabled with no
**Arguments:**
+- `include_accessibility_tree` (boolean, optional): Default true. Set false to skip the per-turn before/after accessibility walks entirely; screenshots, click markers and action.json are still recorded and state is classified `state_capture_disabled`. default: `true`
- `output_dir` (string, required): Absolute or ~-rooted directory where turn folders and (when enabled) the video file are written.
- `record_video` (boolean, optional): Capture the main display to &lt;output_dir&gt;/recording.mp4. Default: false. Set to true to also capture the main display to recording.mp4 (otherwise only the per-turn screenshots + JSON are recorded). On macOS this uses native ScreenCaptureKit (macOS 15.0+); macOS 26 can show a one-time direct screen-capture consent on first use (see `cua-driver permissions grant`). On Windows + Linux it requires ffmpeg on PATH.
+- `state_timeout_ms` (integer, optional): Wall-clock budget in milliseconds for EACH per-turn before/after accessibility walk (same default and bounds as get_window_state's timeout_ms). A walk that runs out records the partial tree and marks it truncated in evidence.json. default: `1000`; range: 100–120000
```json
{"output_dir":"~/cua-trajectories/demo1"}
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
index 0abb8ce7d9..b4f0dd02c3 100644
--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx
+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
@@ -623,6 +623,8 @@ Start trajectory recording. Every subsequent action-tool invocation (click, righ
- `action.json` — tool name, full input arguments, result summary, result-error flag, pid, click point (when applicable), ISO-8601 timestamp.
- `click.png` — for dispatched click-family actions only, `before.png` with a red marker at the click point. A call refused before target resolution is explicitly not applicable instead.
+The per-turn accessibility walk is bounded like `get_window_state`: `state_timeout_ms` (default 1000) caps each before/after walk, a walk that runs out of budget records the PARTIAL tree, and `evidence.json` carries `truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending` and `timeout_ms` for that phase. A provider that stops answering is abandoned after the budget plus a short grace and the phase is classified `state_capture_timeout`. Actions refused before dispatch (for example an unknown or expired `capture_id`) skip the state walk; their state is classified `not_applicable` / `action_refused_before_dispatch`. Pass `include_accessibility_tree: false` to record screenshots and actions without state.
+
Turn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started.
**Video is off by default.** Pass `record_video: true` to also capture the main display to `<output_dir>/recording.mp4` (H.264 / 30 fps) for the lifetime of the session. The recording is torn down automatically when the MCP client disconnects.
@@ -635,8 +637,10 @@ State persists for the life of the daemon; a restart resets to disabled with no
**Arguments:**
+- `include_accessibility_tree` (boolean, optional): Default true. Set false to skip the per-turn before/after accessibility walks entirely; screenshots, click markers and action.json are still recorded and state is classified `state_capture_disabled`. default: `true`
- `output_dir` (string, required): Absolute or ~-rooted directory where turn folders and (when enabled) the video file are written.
- `record_video` (boolean, optional): Capture the main display to &lt;output_dir&gt;/recording.mp4. Default: false. Set to true to also capture the main display to recording.mp4 (otherwise only the per-turn screenshots + JSON are recorded). On macOS this uses native ScreenCaptureKit (macOS 15.0+); macOS 26 can show a one-time direct screen-capture consent on first use (see `cua-driver permissions grant`). On Windows + Linux it requires ffmpeg on PATH.
+- `state_timeout_ms` (integer, optional): Wall-clock budget in milliseconds for EACH per-turn before/after accessibility walk (same default and bounds as get_window_state's timeout_ms). A walk that runs out records the partial tree and marks it truncated in evidence.json. default: `1000`; range: 100–120000
```json
{"output_dir":"~/cua-trajectories/demo1"}
diff --git a/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md b/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md
index 7941e0f70a..953e4b54ce 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md
@@ -54,7 +54,17 @@ Each action writes to `turn-NNNNN/` (five-digit zero-padded counter):
- `before_state.json` and `after_state.json` — application accessibility
state immediately before and after the action. They carry the same
- `tree_markdown` and `element_count` shape as `get_window_state`.
+ `tree_markdown` and `element_count` shape as `get_window_state`. Each walk
+ is bounded by `start_recording`'s `state_timeout_ms` (default 1000, like
+ `get_window_state`'s `timeout_ms`). A walk that runs out of time keeps the
+ partial tree, and `evidence.json` marks that phase `truncated` with
+ `truncation_reason`, `nodes_visited`, `nodes_pending`, and `timeout_ms`. If
+ an accessibility provider stops answering, the turn continues without
+ state and records `state_capture_timeout`. For an action refused before
+ dispatch, such as a click with an unknown or expired `capture_id`, the
+ state walk is skipped and the state is recorded as `not_applicable` with
+ the classification `action_refused_before_dispatch`. Pass
+ `include_accessibility_tree: false` to record without state.
- `before.png` and `after.png` — target-window images immediately before
and after the action. Window capture remains scoped to the target when
another window covers it.
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/capture_registry.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/capture_registry.rs
index ab14a00e41..f7e6be0d53 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/capture_registry.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/capture_registry.rs
@@ -1183,6 +1183,33 @@ impl CaptureService {
self.registry.admit_action(request)
}
+ /// Predict, without consuming anything, whether a capture-bound action
+ /// with these trusted runtime arguments will be refused at admission.
+ /// Returns the stable refusal wire code (`capture_id_invalid`,
+ /// `capture_binding_failed`, `capture_not_found`, `capture_expired`,
+ /// `capture_generation_mismatch`) or `None` when the capture is currently
+ /// admissible or `args` carries no `capture_id`.
+ ///
+ /// This is advisory: admission in the platform adapter remains the only
+ /// authority. Callers use it to avoid expensive work (for example
+ /// recording evidence) for calls that cannot dispatch. The refusals it
+ /// reports are monotonic: an unknown, expired, or superseded capture never
+ /// becomes admissible again.
+ pub fn predict_action_refusal(&self, args: &serde_json::Value) -> Option<&'static str> {
+ let capture_id = args.get("capture_id")?.as_str()?;
+ let id = match capture_id.parse::<CaptureId>() {
+ Ok(id) => id,
+ Err(error) => return Some(error.wire_code()),
+ };
+ let Ok(binding) = self.binding_from_args(args) else {
+ return Some("capture_binding_failed");
+ };
+ self.registry
+ .read_for_perception_bound(id, &binding)
+ .err()
+ .map(|error| error.wire_code())
+ }
+
pub fn retire_session(&self, binding: &CaptureBinding) -> usize {
let removed = self.registry.retire_session(binding);
if binding.runtime_generation == self.runtime_generation {
@@ -1966,4 +1993,78 @@ mod tests {
"capture_action_refused"
);
}
+
+ #[test]
+ fn refusal_prediction_matches_admission_without_consuming_the_capture() {
+ let clock = Arc::new(ManualClock::default());
+ let service = CaptureService::with_clock(config(4, 10_000), clock.clone()).unwrap();
+ let args = serde_json::json!({"_session_id": "predict"});
+ let binding = service.binding_from_args(&args).unwrap();
+ let publish = || {
+ service
+ .publish(CapturePublication {
+ png_bytes: png(2, 2, 1),
+ target: CaptureTarget::PrimaryDesktop,
+ encoded_dimensions: EncodedScreenshotDimensions::new(2, 2).unwrap(),
+ native_action_dimensions: NativeActionDimensions::new(2, 2).unwrap(),
+ screenshot_to_action: ScreenshotToActionTransform::identity(),
+ session_id: Arc::<str>::from("predict"),
+ session_generation: binding.session_generation(),
+ })
+ .unwrap()
+ };
+ let with_capture = |id: &str, session: Option<&str>| {
+ let mut value = serde_json::json!({"capture_id": id, "x": 0, "y": 0});
+ if let Some(session) = session {
+ value["_session_id"] = serde_json::json!(session);
+ }
+ value
+ };
+ let admit = |id: CaptureId| {
+ service.admit_action(CaptureActionRequest {
+ capture_id: id,
+ binding: binding.clone(),
+ target: CaptureTarget::PrimaryDesktop,
+ current_native_action_dimensions: NativeActionDimensions::new(2, 2).unwrap(),
+ screenshot_x: 0.0,
+ screenshot_y: 0.0,
+ })
+ };
+
+ assert_eq!(service.predict_action_refusal(&args), None, "no capture_id");
+ assert_eq!(
+ service.predict_action_refusal(&with_capture("not-a-capture", Some("predict"))),
+ Some(CAPTURE_ID_INVALID_CODE)
+ );
+
+ let live = publish();
+ let live_text = live.to_string();
+ assert_eq!(
+ service.predict_action_refusal(&with_capture(&live_text, None)),
+ Some("capture_binding_failed")
+ );
+ assert_eq!(
+ service.predict_action_refusal(&with_capture(&live_text, Some("other"))),
+ Some("capture_generation_mismatch")
+ );
+ // Predicting admissibility must not consume the one-shot capture.
+ assert_eq!(
+ service.predict_action_refusal(&with_capture(&live_text, Some("predict"))),
+ None
+ );
+ admit(live).expect("prediction left the capture admissible");
+ assert_eq!(
+ service.predict_action_refusal(&with_capture(&live_text, Some("predict"))),
+ Some("capture_not_found")
+ );
+ assert_eq!(admit(live).unwrap_err().wire_code(), "capture_not_found");
+
+ let expiring = publish();
+ clock.advance(Duration::from_secs(11));
+ assert_eq!(
+ service.predict_action_refusal(&with_capture(&expiring.to_string(), Some("predict"))),
+ Some("capture_expired")
+ );
+ assert_eq!(admit(expiring).unwrap_err().wire_code(), "capture_expired");
+ }
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
index d3db85fc21..674d53d494 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
@@ -9,9 +9,10 @@
//! { tool, arguments, result_summary, result_error, timestamp,
//! t_ms_from_session_start, t_start_ms_from_session_start }
+use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
use std::time::Instant;
@@ -96,19 +97,241 @@ pub fn set_click_marker_fn(f: impl Fn(&[u8], f64, f64) -> Option<Vec<u8>> + Send
// ── Platform AX-snapshot callback ────────────────────────────────────────────
//
-// Takes (window_id, pid) and returns JSON bytes for the phase's application
-// state. The post-action bytes are also kept as legacy `app_state.json`.
+// Takes (window_id, pid, budget) and returns JSON bytes for the phase's
+// application state. The post-action bytes are also kept as legacy
+// `app_state.json`.
+//
+// Every platform walks its own accessibility API, but the per-turn contract is
+// shared: the walk honours `budget.timeout_ms` through the common
+// [`crate::walk_budget::WalkBudget`] and reports the shared walk fields
+// (`truncated`, `truncation_reason`, `nodes_visited`, `nodes_pending`,
+// `walk_elapsed_ms`, `timeout_ms`) in its payload. The recorder additionally
+// runs the hook under a hard backstop so a provider that ignores the budget
+// cannot hold the action for longer than `timeout_ms` plus a fixed grace.
+
+/// Default wall-clock budget for one per-turn application-state walk. Matches
+/// the default `timeout_ms` of `get_window_state` on every platform.
+pub const TURN_STATE_TIMEOUT_MS_DEFAULT: u64 = crate::tool_schema::TIMEOUT_MS_DEFAULT;
+
+/// Slack past the budget before the recorder stops waiting for a state hook:
+/// one in-flight accessibility request plus serialization.
+pub const TURN_STATE_BACKSTOP_GRACE: Duration = Duration::from_millis(1000);
+
+/// Budget handed to the platform application-state hook for one turn phase.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct StateCaptureBudget {
+ pub timeout_ms: u64,
+}
-type AxSnapshotFnBox = Box<dyn Fn(Option<u64>, Option<i64>) -> Option<Vec<u8>> + Send + Sync>;
-static AX_SNAPSHOT_FN: OnceLock<AxSnapshotFnBox> = OnceLock::new();
+impl StateCaptureBudget {
+ /// The recorder stops waiting for the hook at this point.
+ pub fn backstop(self) -> Duration {
+ Duration::from_millis(self.timeout_ms) + TURN_STATE_BACKSTOP_GRACE
+ }
+}
-/// Register the platform-specific AX/UIA snapshot callback. Call once at startup.
+impl Default for StateCaptureBudget {
+ fn default() -> Self {
+ Self {
+ timeout_ms: TURN_STATE_TIMEOUT_MS_DEFAULT,
+ }
+ }
+}
+
+type AxSnapshotFn =
+ Arc<dyn Fn(Option<u64>, Option<i64>, StateCaptureBudget) -> Option<Vec<u8>> + Send + Sync>;
+static AX_SNAPSHOT_FN: OnceLock<AxSnapshotFn> = OnceLock::new();
+
+/// Register an AX/UIA snapshot callback that ignores the per-turn budget.
+/// The recorder's backstop still bounds how long a turn waits for it. Call
+/// once at startup; platform runtimes use [`set_budgeted_ax_snapshot_fn`].
pub fn set_ax_snapshot_fn(
f: impl Fn(Option<u64>, Option<i64>) -> Option<Vec<u8>> + Send + Sync + 'static,
) {
- let _ = AX_SNAPSHOT_FN.set(Box::new(f));
+ set_budgeted_ax_snapshot_fn(move |window_id, pid, _budget| f(window_id, pid));
+}
+
+/// Register the platform-specific AX/UIA/AT-SPI snapshot callback. The hook
+/// must bound its walk by `budget.timeout_ms` and should report the shared
+/// walk fields in its JSON payload. Call once at startup.
+pub fn set_budgeted_ax_snapshot_fn(
+ f: impl Fn(Option<u64>, Option<i64>, StateCaptureBudget) -> Option<Vec<u8>> + Send + Sync + 'static,
+) {
+ let _ = AX_SNAPSHOT_FN.set(Arc::new(f));
+}
+
+/// Hooks abandoned at their backstop and still running, per target pid. While
+/// one is outstanding the recorder does not start another walk of the same
+/// application, so a hung accessibility provider cannot accumulate threads.
+static ABANDONED_STATE_CAPTURES: OnceLock<Mutex<HashMap<i64, usize>>> = OnceLock::new();
+
+fn abandoned_state_captures() -> std::sync::MutexGuard<'static, HashMap<i64, usize>> {
+ ABANDONED_STATE_CAPTURES
+ .get_or_init(Default::default)
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+}
+
+fn release_abandoned_state_capture(key: i64) {
+ let mut abandoned = abandoned_state_captures();
+ if let Some(count) = abandoned.get_mut(&key) {
+ *count = count.saturating_sub(1);
+ if *count == 0 {
+ abandoned.remove(&key);
+ }
+ }
+}
+
+/// Outcome of one phase's application-state capture.
+#[derive(Debug, Default)]
+struct StateCapture {
+ bytes: Option<Vec<u8>>,
+ /// Stable classification when `bytes` is absent for a known reason.
+ classification: Option<&'static str>,
+ /// `not_applicable` instead of `unavailable` (deliberate skip).
+ not_applicable: bool,
+ /// Shared walk fields copied from the payload for the evidence manifest.
+ walk: Option<serde_json::Map<String, Value>>,
+}
+
+impl StateCapture {
+ fn skipped(classification: &'static str) -> Self {
+ Self {
+ classification: Some(classification),
+ not_applicable: true,
+ ..Self::default()
+ }
+ }
+
+ fn unavailable(classification: &'static str) -> Self {
+ Self {
+ classification: Some(classification),
+ ..Self::default()
+ }
+ }
+
+ fn captured(bytes: Vec<u8>) -> Self {
+ let walk = walk_fields(&bytes);
+ Self {
+ bytes: Some(bytes),
+ walk,
+ ..Self::default()
+ }
+ }
+}
+
+/// Copy the shared walk fields (see [`crate::walk_budget::WalkOutcome::apply`])
+/// out of a state payload so the manifest can say when the tree is partial.
+fn walk_fields(bytes: &[u8]) -> Option<serde_json::Map<String, Value>> {
+ const KEYS: [&str; 6] = [
+ "truncated",
+ "truncation_reason",
+ "nodes_visited",
+ "nodes_pending",
+ "walk_elapsed_ms",
+ "timeout_ms",
+ ];
+ let payload: Value = serde_json::from_slice(bytes).ok()?;
+ let fields: serde_json::Map<String, Value> = KEYS
+ .iter()
+ .filter_map(|key| {
+ payload
+ .get(*key)
+ .map(|value| ((*key).to_owned(), value.clone()))
+ })
+ .collect();
+ (!fields.is_empty()).then_some(fields)
+}
+
+/// Run a state hook with a hard deadline. The hook runs on its own thread; if
+/// it has not answered by `budget.backstop()` the turn proceeds without state
+/// (classified `state_capture_timeout`) and the thread is left to finish on
+/// its own. Its late result is discarded.
+fn run_state_hook(
+ hook: AxSnapshotFn,
+ window_id: Option<u64>,
+ pid: Option<i64>,
+ budget: StateCaptureBudget,
+) -> StateCapture {
+ use std::sync::atomic::{AtomicBool, Ordering};
+ let key = pid.unwrap_or(-1);
+ if abandoned_state_captures()
+ .get(&key)
+ .is_some_and(|count| *count > 0)
+ {
+ return StateCapture::unavailable("state_capture_busy");
+ }
+ let abandoned = Arc::new(AtomicBool::new(false));
+ let (sender, receiver) = std::sync::mpsc::sync_channel(1);
+ let worker_abandoned = abandoned.clone();
+ let spawned = std::thread::Builder::new()
+ .name("cua-recording-state".into())
+ .spawn(move || {
+ let result = hook(window_id, pid, budget);
+ if worker_abandoned.swap(true, Ordering::SeqCst) {
+ release_abandoned_state_capture(key);
+ } else {
+ let _ = sender.send(result);
+ }
+ });
+ if spawned.is_err() {
+ return StateCapture::unavailable("capture_failed");
+ }
+ let received = match receiver.recv_timeout(budget.backstop()) {
+ Ok(result) => Some(result),
+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => None,
+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
+ *abandoned_state_captures().entry(key).or_insert(0) += 1;
+ if abandoned.swap(true, Ordering::SeqCst) {
+ // The worker finished between the timeout and the mark.
+ release_abandoned_state_capture(key);
+ receiver.try_recv().ok()
+ } else {
+ tracing::warn!(
+ target: "recording",
+ "application-state capture for pid {key} exceeded its {} ms budget; \
+ recording the turn without state",
+ budget.timeout_ms
+ );
+ return StateCapture::unavailable("state_capture_timeout");
+ }
+ }
+ };
+ match received.flatten() {
+ Some(bytes) => StateCapture::captured(bytes),
+ None => StateCapture::unavailable("capture_failed"),
+ }
+}
+
+fn capture_state(
+ window_id: Option<u64>,
+ pid: Option<i64>,
+ budget: StateCaptureBudget,
+) -> StateCapture {
+ match AX_SNAPSHOT_FN.get() {
+ Some(hook) => run_state_hook(hook.clone(), window_id, pid, budget),
+ None => StateCapture::default(),
+ }
+}
+
+/// How a turn phase should treat the application-state walk.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum StatePlan {
+ /// Walk under this budget.
+ Capture(StateCaptureBudget),
+ /// Deliberately skipped: recorded as `not_applicable` with this reason.
+ Skip(&'static str),
}
+/// Classification for application state that was not walked because the
+/// action was refused before any input was dispatched.
+pub const STATE_SKIPPED_REFUSED: &str = "action_refused_before_dispatch";
+/// Classification for application state disabled by `start_recording`.
+pub const STATE_SKIPPED_DISABLED: &str = "state_capture_disabled";
+/// Classification for before-state skipped on a predicted refusal that the
+/// tool did not in fact refuse.
+pub const STATE_SKIPPED_PREDICTION_MISSED: &str = "skipped_predicted_refusal";
+
// ── Platform element-bounds callback ─────────────────────────────────────────
//
// Resolves an element_index to its center point in window-local screenshot
@@ -132,7 +355,7 @@ pub fn set_element_bounds_fn(
#[derive(Default)]
struct TurnCapture {
- state: Option<Vec<u8>>,
+ state: StateCapture,
screenshot: Option<Vec<u8>>,
screenshot_classification: Option<&'static str>,
}
@@ -220,6 +443,12 @@ pub struct PendingTurn {
pid: Option<i64>,
click_point: Option<(f64, f64)>,
capture_visual_state: bool,
+ /// Per-turn application-state budget; `None` when state capture is
+ /// disabled for this recording.
+ state_budget: Option<StateCaptureBudget>,
+ /// Refusal code predicted before dispatch (for example an unknown or
+ /// expired `capture_id`). The before-state walk was skipped for it.
+ predicted_refusal: Option<&'static str>,
before: TurnCapture,
dispatch_click: Option<Arc<DispatchClickScope>>,
}
@@ -246,6 +475,9 @@ struct RecordingInner {
/// than a monotonic counter, and it doubles as the config-override key.
owner: Option<String>,
output_dir: Option<PathBuf>,
+ /// Per-turn application-state budget, or `None` when the caller disabled
+ /// state capture with `include_accessibility_tree: false`.
+ state_budget: Option<StateCaptureBudget>,
next_turn: u32,
session_start_ms: u64,
/// Monotonic clock anchor for the cursor sampler so its `t_ms`
@@ -300,6 +532,7 @@ impl RecordingSession {
generation: 0,
owner: None,
output_dir: None,
+ state_budget: Some(StateCaptureBudget::default()),
next_turn: 1,
session_start_ms: 0,
session_monotonic_start: None,
@@ -346,6 +579,24 @@ impl RecordingSession {
output_dir: &str,
record_video: bool,
owner: Option<&str>,
+ ) -> anyhow::Result<()> {
+ self.start_with_state_budget(
+ output_dir,
+ record_video,
+ owner,
+ Some(StateCaptureBudget::default()),
+ )
+ }
+
+ /// [`Self::start`] with an explicit per-turn application-state budget.
+ /// `None` disables the before/after accessibility walks entirely; the
+ /// turn evidence then classifies state as `state_capture_disabled`.
+ pub fn start_with_state_budget(
+ &self,
+ output_dir: &str,
+ record_video: bool,
+ owner: Option<&str>,
+ state_budget: Option<StateCaptureBudget>,
) -> anyhow::Result<()> {
let mut inner = self.inner.lock().unwrap();
// Write-boundary resurrection guard — checked INSIDE the lock so the
@@ -437,6 +688,7 @@ impl RecordingSession {
inner.generation = inner.generation.wrapping_add(1);
inner.enabled = true;
inner.output_dir = Some(dir);
+ inner.state_budget = state_budget;
inner.next_turn = 1;
inner.session_start_ms = now_ms();
inner.session_monotonic_start = Some(monotonic_start);
@@ -562,10 +814,32 @@ impl RecordingSession {
}
}
+ /// Per-turn application-state budget of the current (or next) recording;
+ /// `None` when state capture is disabled.
+ pub fn state_budget(&self) -> Option<StateCaptureBudget> {
+ self.inner.lock().unwrap().state_budget
+ }
+
/// Reserve a turn and capture its target immediately before tool dispatch.
/// No-op when recording is disabled.
pub fn begin_turn(&self, tool_name: &str, args: &Value, start_ms: u64) -> Option<PendingTurn> {
- self.begin_turn_with_capture(tool_name, args, start_ms, true)
+ self.begin_turn_with_capture(tool_name, args, start_ms, true, None)
+ }
+
+ /// [`Self::begin_turn`] for a call the dispatcher already knows the tool
+ /// will refuse before dispatching input (`predicted_refusal` carries the
+ /// refusal code, e.g. `capture_expired`). The before screenshot is still
+ /// captured, but the application-state walk is skipped because the action
+ /// cannot change the application. If the tool does not in fact refuse, the
+ /// after phase is captured normally and the manifest records the miss.
+ pub fn begin_turn_with_refusal_hint(
+ &self,
+ tool_name: &str,
+ args: &Value,
+ start_ms: u64,
+ predicted_refusal: Option<&'static str>,
+ ) -> Option<PendingTurn> {
+ self.begin_turn_with_capture(tool_name, args, start_ms, true, predicted_refusal)
}
/// Reserve a turn while deliberately suppressing visual and accessibility
@@ -578,7 +852,7 @@ impl RecordingSession {
args: &Value,
start_ms: u64,
) -> Option<PendingTurn> {
- self.begin_turn_with_capture(tool_name, args, start_ms, false)
+ self.begin_turn_with_capture(tool_name, args, start_ms, false, None)
}
fn begin_turn_with_capture(
@@ -587,8 +861,9 @@ impl RecordingSession {
args: &Value,
start_ms: u64,
capture_visual_state: bool,
+ predicted_refusal: Option<&'static str>,
) -> Option<PendingTurn> {
- let (turn_dir, session_start_ms, generation) = {
+ let (turn_dir, session_start_ms, generation, state_budget) = {
let mut inner = self.inner.lock().unwrap();
if !inner.enabled {
return None;
@@ -600,6 +875,7 @@ impl RecordingSession {
out.join(format!("turn-{idx:05}")),
inner.session_start_ms,
inner.generation,
+ inner.state_budget,
)
};
@@ -630,13 +906,17 @@ impl RecordingSession {
self.pixel_point_fn.get(),
);
let before = if capture_visual_state {
- capture_turn(window_id, pid)
+ capture_turn(
+ window_id,
+ pid,
+ match (state_budget, predicted_refusal) {
+ (None, _) => StatePlan::Skip(STATE_SKIPPED_DISABLED),
+ (Some(_), Some(_)) => StatePlan::Skip(STATE_SKIPPED_REFUSED),
+ (Some(budget), None) => StatePlan::Capture(budget),
+ },
+ )
} else {
- TurnCapture {
- state: None,
- screenshot: None,
- screenshot_classification: Some("privacy_suppressed"),
- }
+ TurnCapture::privacy_suppressed()
};
let mut inner = self.inner.lock().unwrap();
@@ -659,6 +939,8 @@ impl RecordingSession {
pid,
click_point,
capture_visual_state,
+ state_budget,
+ predicted_refusal,
before,
dispatch_click: if capture_visual_state
&& matches!(tool_name, "click" | "double_click" | "right_click")
@@ -705,15 +987,29 @@ impl RecordingSession {
action_record: Option<&crate::action_record::ActionExecutionRecord>,
result_is_error: bool,
) {
+ let is_live = |inner: &RecordingInner| {
+ let live = inner.enabled && inner.generation == pending.generation;
+ if !live {
+ tracing::warn!(
+ target: "recording",
+ "discarding a turn from an inactive recording generation"
+ );
+ }
+ live
+ };
+ if !is_live(&self.inner.lock().unwrap()) {
+ return;
+ }
+ // Capture outside the session lock: the after phase is bounded, but
+ // recording-state queries and unrelated turns must not wait on it.
+ let after = capture_after(&pending, action_record, result_is_error);
let mut inner = self.inner.lock().unwrap();
- if !inner.enabled || inner.generation != pending.generation {
- tracing::warn!(
- target: "recording",
- "discarding a turn from an inactive recording generation"
- );
+ if !is_live(&inner) {
return;
}
- if let Err(error) = write_turn(pending, result_text, action_record, result_is_error) {
+ if let Err(error) =
+ write_turn_with_after(pending, after, result_text, action_record, result_is_error)
+ {
inner.last_error = Some(error.to_string());
}
}
@@ -737,20 +1033,69 @@ impl Default for RecordingSession {
// ── helpers ───────────────────────────────────────────────────────────────────
-fn capture_turn(window_id: Option<u64>, pid: Option<i64>) -> TurnCapture {
+impl TurnCapture {
+ fn privacy_suppressed() -> Self {
+ Self {
+ state: StateCapture::default(),
+ screenshot: None,
+ screenshot_classification: Some("privacy_suppressed"),
+ }
+ }
+}
+
+fn capture_turn(window_id: Option<u64>, pid: Option<i64>, state: StatePlan) -> TurnCapture {
let screenshot = SCREENSHOT_FN
.get()
.map(|capture| capture(window_id, pid))
.unwrap_or_else(|| ScreenshotCapture::unavailable("capture_hook_unavailable"));
TurnCapture {
- state: AX_SNAPSHOT_FN
- .get()
- .and_then(|capture| capture(window_id, pid)),
+ state: match state {
+ // No target application: there is no state to walk.
+ _ if pid.is_none() => StateCapture::default(),
+ StatePlan::Capture(budget) => capture_state(window_id, pid, budget),
+ StatePlan::Skip(classification) => StateCapture::skipped(classification),
+ },
screenshot: screenshot.png,
screenshot_classification: screenshot.classification,
}
}
+/// Whether the tool refused the call before any input reached the target, so
+/// the application cannot have changed. A click-family call rejected with no
+/// action record is only treated as refused when the dispatcher predicted it.
+fn refused_before_any_dispatch(
+ pending: &PendingTurn,
+ action_record: Option<&crate::action_record::ActionExecutionRecord>,
+ result_is_error: bool,
+) -> bool {
+ result_is_error
+ && match action_record {
+ Some(record) => record.effect == crate::action_record::ActionEffect::Refused,
+ None => pending.predicted_refusal.is_some(),
+ }
+}
+
+/// Plan and run the post-action capture. The after screenshot is always taken
+/// (it is cheap and keeps the visual trajectory continuous); the state walk is
+/// skipped when the call was refused before dispatch.
+fn capture_after(
+ pending: &PendingTurn,
+ action_record: Option<&crate::action_record::ActionExecutionRecord>,
+ result_is_error: bool,
+) -> TurnCapture {
+ if !pending.capture_visual_state {
+ return TurnCapture::privacy_suppressed();
+ }
+ let plan = match pending.state_budget {
+ None => StatePlan::Skip(STATE_SKIPPED_DISABLED),
+ Some(_) if refused_before_any_dispatch(pending, action_record, result_is_error) => {
+ StatePlan::Skip(STATE_SKIPPED_REFUSED)
+ }
+ Some(budget) => StatePlan::Capture(budget),
+ };
+ capture_turn(pending.window_id, pending.pid, plan)
+}
+
fn resolve_click_point(
tool_name: &str,
args: &Value,
@@ -778,7 +1123,7 @@ fn write_phase_artifacts(
capture: &TurnCapture,
) -> anyhow::Result<()> {
std::fs::create_dir_all(turn_dir)?;
- if let Some(state) = &capture.state {
+ if let Some(state) = &capture.state.bytes {
std::fs::write(turn_dir.join(format!("{phase}_state.json")), state)?;
}
if let Some(screenshot) = &capture.screenshot {
@@ -803,6 +1148,21 @@ fn capture_status(captured: bool, expected: bool, classification: Option<&'stati
}
}
+fn state_status(state: &StateCapture, expected: bool) -> Value {
+ if state.not_applicable && expected {
+ return serde_json::json!({
+ "status": "not_applicable",
+ "classification": state.classification.unwrap_or("not_applicable"),
+ });
+ }
+ let mut status = capture_status(state.bytes.is_some(), expected, state.classification);
+ if let (Some(walk), Some(object)) = (&state.walk, status.as_object_mut()) {
+ // A partial tree is still captured evidence; say where it stopped.
+ object.extend(walk.clone());
+ }
+ status
+}
+
fn write_evidence_manifest(
turn_dir: &Path,
before: &TurnCapture,
@@ -816,7 +1176,7 @@ fn write_evidence_manifest(
let mut manifest = serde_json::json!({
"schema": "cua-turn-evidence/v1",
"before": {
- "state": capture_status(before.state.is_some(), state_expected, None),
+ "state": state_status(&before.state, state_expected),
"screenshot": capture_status(
before.screenshot.is_some(),
true,
@@ -824,7 +1184,7 @@ fn write_evidence_manifest(
),
},
"after": {
- "state": capture_status(after.state.is_some(), state_expected, None),
+ "state": state_status(&after.state, state_expected),
"screenshot": capture_status(
after.screenshot.is_some(),
true,
@@ -936,11 +1296,23 @@ fn semantic_action_without_point(action: &Value) -> bool {
})
}
+#[cfg(test)]
fn write_turn(
pending: PendingTurn,
result_text: &str,
action_record: Option<&crate::action_record::ActionExecutionRecord>,
result_is_error: bool,
+) -> anyhow::Result<()> {
+ let after = capture_after(&pending, action_record, result_is_error);
+ write_turn_with_after(pending, after, result_text, action_record, result_is_error)
+}
+
+fn write_turn_with_after(
+ pending: PendingTurn,
+ after: TurnCapture,
+ result_text: &str,
+ action_record: Option<&crate::action_record::ActionExecutionRecord>,
+ result_is_error: bool,
) -> anyhow::Result<()> {
let PendingTurn {
generation: _,
@@ -949,11 +1321,13 @@ fn write_turn(
args,
start_ms,
session_start_ms,
- window_id,
+ window_id: _,
pid,
click_point,
capture_visual_state,
- before,
+ state_budget: _,
+ predicted_refusal,
+ mut before,
dispatch_click,
} = pending;
let supplemental = dispatch_click
@@ -991,15 +1365,6 @@ fn write_turn(
};
std::fs::create_dir_all(&turn_dir)?;
let now = now_ms();
- let after = if capture_visual_state {
- capture_turn(window_id, pid)
- } else {
- TurnCapture {
- state: None,
- screenshot: None,
- screenshot_classification: Some("privacy_suppressed"),
- }
- };
let click_family = matches!(tool_name.as_str(), "click" | "double_click" | "right_click");
let action_refused = action_record
.is_some_and(|record| record.effect == crate::action_record::ActionEffect::Refused);
@@ -1012,6 +1377,14 @@ fn write_turn(
// resolved point in action.json is useful diagnostic context, but a
// crosshair would falsely imply that a click was delivered.
let refused_before_dispatch = click_family && result_is_error && action_refused;
+ // The before-state walk was skipped on a predicted refusal. If the tool
+ // did not in fact refuse, say so instead of claiming a refusal.
+ if predicted_refusal.is_some()
+ && before.state.classification == Some(STATE_SKIPPED_REFUSED)
+ && !(result_is_error && (action_refused || action_record.is_none()))
+ {
+ before.state = StateCapture::unavailable(STATE_SKIPPED_PREDICTION_MISSED);
+ }
let mut payload = serde_json::json!({
"tool": tool_name,
"arguments": args,
@@ -1043,7 +1416,7 @@ fn write_turn(
}
// Preserve the original post-action names for existing trajectory readers.
- if let Some(state) = &after.state {
+ if let Some(state) = &after.state.bytes {
std::fs::write(turn_dir.join("app_state.json"), state)?;
}
if let Some(screenshot) = &after.screenshot {
@@ -1257,9 +1630,11 @@ mod tests {
pid: Some(901),
click_point: Some((500.0, 500.0)),
capture_visual_state: true,
+ state_budget: Some(StateCaptureBudget::default()),
+ predicted_refusal: None,
before: TurnCapture {
screenshot: Some(crate::image_utils::encode_rgba_to_png(&[0; 24], 3, 2).unwrap()),
- state: Some(b"original state".to_vec()),
+ state: StateCapture::captured(b"original state".to_vec()),
screenshot_classification: None,
},
dispatch_click: Some(Arc::new(DispatchClickScope {
@@ -1643,6 +2018,8 @@ mod tests {
pid: Some(1),
click_point: point,
capture_visual_state: false,
+ state_budget: Some(StateCaptureBudget::default()),
+ predicted_refusal: None,
before: TurnCapture::default(),
dispatch_click: None,
};
@@ -1708,6 +2085,8 @@ mod tests {
pid: Some(1),
click_point: Some(point),
capture_visual_state: false,
+ state_budget: Some(StateCaptureBudget::default()),
+ predicted_refusal: None,
before: TurnCapture {
screenshot: Some(png.clone()),
..Default::default()
@@ -1945,6 +2324,23 @@ mod tests {
resolved_refusal_manifest["click"]["classification"],
"action_refused_before_dispatch"
);
+ // A refusal before dispatch cannot change the application, so the
+ // expensive after-state walk is skipped and classified, while the
+ // cheap after screenshot keeps the visual trajectory continuous.
+ assert_eq!(
+ resolved_refusal_manifest["after"]["state"],
+ serde_json::json!({
+ "status": "not_applicable",
+ "classification": STATE_SKIPPED_REFUSED,
+ })
+ );
+ assert_eq!(
+ resolved_refusal_manifest["before"]["state"]["status"],
+ "captured"
+ );
+ assert!(!resolved_refusal_turn.join("after_state.json").exists());
+ assert!(!resolved_refusal_turn.join("app_state.json").exists());
+ assert!(resolved_refusal_turn.join("after.png").exists());
let files = [
"action.json",
@@ -1963,7 +2359,16 @@ mod tests {
}
std::fs::remove_dir(directory).expect("remove turn fixture directory");
}
- for directory in [&refused_turn, &resolved_refusal_turn] {
+ for file in files
+ .iter()
+ .copied()
+ .filter(|file| !matches!(*file, "click.png" | "after_state.json" | "app_state.json"))
+ {
+ std::fs::remove_file(resolved_refusal_turn.join(file))
+ .expect("remove resolved refusal fixture file");
+ }
+ std::fs::remove_dir(&resolved_refusal_turn).expect("remove resolved refusal directory");
+ for directory in [&refused_turn] {
for file in files.iter().copied().filter(|file| *file != "click.png") {
std::fs::remove_file(directory.join(file))
.expect("remove refused turn fixture file");
@@ -2003,6 +2408,176 @@ mod tests {
std::fs::remove_dir(&output_dir).expect("remove recording directory");
}
+ #[test]
+ fn state_status_reports_partial_walks_and_classified_skips() {
+ let partial = StateCapture::captured(
+ serde_json::to_vec(&serde_json::json!({
+ "tree_markdown": "- AXWindow",
+ "truncated": true,
+ "truncation_reason": "timeout",
+ "nodes_visited": 3,
+ "nodes_pending": 9,
+ "walk_elapsed_ms": 1004,
+ "timeout_ms": 1000,
+ }))
+ .unwrap(),
+ );
+ assert_eq!(
+ state_status(&partial, true),
+ serde_json::json!({
+ "status": "captured",
+ "truncated": true,
+ "truncation_reason": "timeout",
+ "nodes_visited": 3,
+ "nodes_pending": 9,
+ "walk_elapsed_ms": 1004,
+ "timeout_ms": 1000,
+ })
+ );
+ // Payloads without walk fields (older hooks) keep the plain status.
+ assert_eq!(
+ state_status(&StateCapture::captured(b"{}".to_vec()), true),
+ serde_json::json!({"status": "captured"})
+ );
+ assert_eq!(
+ state_status(&StateCapture::skipped(STATE_SKIPPED_REFUSED), true),
+ serde_json::json!({
+ "status": "not_applicable",
+ "classification": "action_refused_before_dispatch",
+ })
+ );
+ assert_eq!(
+ state_status(&StateCapture::unavailable("state_capture_timeout"), true),
+ serde_json::json!({
+ "status": "unavailable",
+ "classification": "state_capture_timeout",
+ })
+ );
+ // Without a target pid nothing was expected, whatever the plan said.
+ assert_eq!(
+ state_status(&StateCapture::skipped(STATE_SKIPPED_REFUSED), false),
+ serde_json::json!({"status": "not_applicable", "classification": "no_target_pid"})
+ );
+ }
+
+ #[test]
+ fn state_hook_backstop_releases_a_hung_provider_and_recovers() {
+ // A distinct pid keeps this test's abandoned-walk accounting apart
+ // from other tests sharing the process-global table.
+ const PID: i64 = 990_001;
+ let (release, gate) = std::sync::mpsc::channel::<()>();
+ let gate = Mutex::new(gate);
+ let hung: AxSnapshotFn = Arc::new(move |_, _, _| {
+ let _ = gate.lock().unwrap().recv();
+ Some(b"{}".to_vec())
+ });
+ let budget = StateCaptureBudget { timeout_ms: 50 };
+ let started = Instant::now();
+ let timed_out = run_state_hook(hung.clone(), None, Some(PID), budget);
+ assert!(started.elapsed() < budget.backstop() + Duration::from_millis(500));
+ assert_eq!(timed_out.classification, Some("state_capture_timeout"));
+ assert!(timed_out.bytes.is_none());
+
+ let calls = Arc::new(AtomicUsize::new(0));
+ let counting_calls = calls.clone();
+ let fast: AxSnapshotFn = Arc::new(move |_, _, budget| {
+ counting_calls.fetch_add(1, Ordering::SeqCst);
+ Some(format!(r#"{{"timeout_ms":{}}}"#, budget.timeout_ms).into_bytes())
+ });
+ let busy = run_state_hook(fast.clone(), None, Some(PID), budget);
+ assert_eq!(busy.classification, Some("state_capture_busy"));
+ assert_eq!(
+ calls.load(Ordering::SeqCst),
+ 0,
+ "no second walk of a hung app"
+ );
+ // Another application is not blocked by the hung one.
+ let other = run_state_hook(fast.clone(), None, Some(PID + 1), budget);
+ assert!(other.bytes.is_some());
+
+ release.send(()).unwrap();
+ let deadline = Instant::now() + Duration::from_secs(5);
+ while abandoned_state_captures().contains_key(&PID) {
+ assert!(Instant::now() < deadline, "abandoned walk never released");
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ let recovered = run_state_hook(fast, None, Some(PID), budget);
+ assert_eq!(
+ recovered.walk.unwrap()["timeout_ms"],
+ serde_json::json!(50),
+ "the hook receives the configured budget"
+ );
+ }
+
+ #[test]
+ fn predicted_refusal_skips_state_and_records_a_missed_prediction() {
+ let refusal = crate::action_record::ActionExecutionRecord::builder(
+ crate::action_record::ActionEffect::Refused,
+ crate::action_record::ActionTransport::MacosCgEventPid,
+ crate::action_record::RequestedDelivery::Background,
+ )
+ .build()
+ .unwrap();
+ let delivered = crate::action_record::ActionExecutionRecord::builder(
+ crate::action_record::ActionEffect::Unverifiable,
+ crate::action_record::ActionTransport::MacosCgEventPid,
+ crate::action_record::RequestedDelivery::Background,
+ )
+ .actual_delivery(crate::action_record::ActualDelivery::Background)
+ .build()
+ .unwrap();
+ let pending = |root: &Path| PendingTurn {
+ generation: 0,
+ turn_dir: root.to_path_buf(),
+ tool_name: "click".into(),
+ args: serde_json::json!({"pid": 1, "window_id": 2, "x": 0, "y": 0}),
+ start_ms: 0,
+ session_start_ms: 0,
+ window_id: Some(2),
+ pid: Some(1),
+ click_point: None,
+ capture_visual_state: true,
+ state_budget: Some(StateCaptureBudget::default()),
+ predicted_refusal: Some("capture_expired"),
+ before: TurnCapture {
+ state: StateCapture::skipped(STATE_SKIPPED_REFUSED),
+ ..Default::default()
+ },
+ dispatch_click: None,
+ };
+ let manifest = |root: &Path| -> Value {
+ serde_json::from_slice(&std::fs::read(root.join("evidence.json")).unwrap()).unwrap()
+ };
+
+ let refused = tempfile::tempdir().unwrap();
+ let turn = pending(refused.path());
+ let after = capture_after(&turn, Some(&refusal), true);
+ assert_eq!(after.state.classification, Some(STATE_SKIPPED_REFUSED));
+ write_turn_with_after(turn, after, "refused", Some(&refusal), true).unwrap();
+ let evidence = manifest(refused.path());
+ assert_eq!(evidence["before"]["state"]["status"], "not_applicable");
+ assert_eq!(evidence["after"]["state"]["status"], "not_applicable");
+
+ // The adapter is the authority: if it dispatched anyway, the skipped
+ // before-state must not be presented as a refusal.
+ let missed = tempfile::tempdir().unwrap();
+ write_turn_with_after(
+ pending(missed.path()),
+ TurnCapture::default(),
+ "clicked",
+ Some(&delivered),
+ false,
+ )
+ .unwrap();
+ assert_eq!(
+ manifest(missed.path())["before"]["state"],
+ serde_json::json!({
+ "status": "unavailable",
+ "classification": STATE_SKIPPED_PREDICTION_MISSED,
+ })
+ );
+ }
+
#[test]
fn private_turn_records_metadata_without_visual_or_ax_artifacts() {
let output_dir = std::env::temp_dir().join(format!(
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs
index b6a6bcf277..dc6751a545 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs
@@ -62,6 +62,17 @@ impl Tool for StartRecordingTool {
- `click.png` — for dispatched click-family actions only, `before.png` \
with a red marker at the click point. A call refused before target \
resolution is explicitly not applicable instead.\n\n\
+ The per-turn accessibility walk is bounded like `get_window_state`: \
+ `state_timeout_ms` (default 1000) caps each before/after walk, a walk \
+ that runs out of budget records the PARTIAL tree, and `evidence.json` \
+ carries `truncated`, `truncation_reason`, `nodes_visited`, \
+ `nodes_pending` and `timeout_ms` for that phase. A provider that stops \
+ answering is abandoned after the budget plus a short grace and the \
+ phase is classified `state_capture_timeout`. Actions refused before \
+ dispatch (for example an unknown or expired `capture_id`) skip the \
+ state walk; their state is classified `not_applicable` / \
+ `action_refused_before_dispatch`. Pass `include_accessibility_tree: \
+ false` to record screenshots and actions without state.\n\n\
Turn folders are named `turn-00001/`, `turn-00002/`, etc. Turn \
numbering restarts at 1 each time recording is (re-)started.\n\n\
**Video is off by default.** Pass `record_video: true` to also \
@@ -106,6 +117,24 @@ impl Tool for StartRecordingTool {
direct screen-capture consent on first use (see \
`cua-driver permissions grant`). On Windows + Linux it \
requires ffmpeg on PATH."
+ },
+ "state_timeout_ms": {
+ "type": "integer",
+ "minimum": crate::tool_schema::TIMEOUT_MS_MIN,
+ "maximum": crate::tool_schema::TIMEOUT_MS_MAX,
+ "default": crate::recording::TURN_STATE_TIMEOUT_MS_DEFAULT,
+ "description": "Wall-clock budget in milliseconds for EACH per-turn \
+ before/after accessibility walk (same default and bounds as \
+ get_window_state's timeout_ms). A walk that runs out records \
+ the partial tree and marks it truncated in evidence.json."
+ },
+ "include_accessibility_tree": {
+ "type": "boolean",
+ "default": true,
+ "description": "Default true. Set false to skip the per-turn \
+ before/after accessibility walks entirely; screenshots, \
+ click markers and action.json are still recorded and state \
+ is classified `state_capture_disabled`."
}
},
"additionalProperties": false
@@ -129,10 +158,17 @@ impl Tool for StartRecordingTool {
// (session_end) only stops the recording its own session started.
let owner = args.opt_str("_session_id");
- match self.session.start(
+ let state_budget = args.bool_or("include_accessibility_tree", true).then(|| {
+ crate::recording::StateCaptureBudget {
+ timeout_ms: crate::tool_schema::resolve_timeout_ms(args.get("state_timeout_ms")),
+ }
+ });
+
+ match self.session.start_with_state_budget(
output_dir.as_deref().unwrap(),
record_video,
owner.as_deref(),
+ state_budget,
) {
Ok(()) => {
let state = self.session.current_state();
@@ -625,3 +661,42 @@ impl Tool for InstallFfmpegTool {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[tokio::test]
+ async fn start_recording_resolves_the_per_turn_state_budget() {
+ let session = Arc::new(RecordingSession::new());
+ let tool = StartRecordingTool::new(session.clone());
+ let properties = &tool.def().input_schema["properties"];
+ assert_eq!(properties["state_timeout_ms"]["default"], 1000);
+ assert_eq!(properties["include_accessibility_tree"]["default"], true);
+
+ let directory = tempfile::tempdir().unwrap();
+ let dir = directory.path().to_str().unwrap();
+ for (args, expected) in [
+ (json!({"output_dir": dir}), Some(1000)),
+ (
+ json!({"output_dir": dir, "state_timeout_ms": 250}),
+ Some(250),
+ ),
+ // Clamped to the shared timeout_ms bounds.
+ (json!({"output_dir": dir, "state_timeout_ms": 1}), Some(100)),
+ (
+ json!({"output_dir": dir, "include_accessibility_tree": false}),
+ None,
+ ),
+ ] {
+ let result = tool.invoke(args.clone()).await;
+ assert_ne!(result.is_error, Some(true), "{args}");
+ assert_eq!(
+ session.state_budget().map(|budget| budget.timeout_ms),
+ expected,
+ "{args}"
+ );
+ session.stop_owner(None).unwrap();
+ }
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index b96bdc0d80..3c7ff8ac7b 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -1564,8 +1564,20 @@ impl ToolRegistry {
self.recording
.begin_private_turn(resolved_name, &recording_args, start_ms)
} else {
- self.recording
- .begin_turn(resolved_name, &recording_args, start_ms)
+ // A capture-bound click whose capture is already unknown,
+ // expired, or superseded is refused by every platform
+ // adapter before dispatch. Do not walk the application for
+ // evidence of an action that cannot happen.
+ let predicted_refusal =
+ matches!(resolved_name, "click" | "double_click" | "right_click")
+ .then(|| self.capture_service.predict_action_refusal(&args))
+ .flatten();
+ self.recording.begin_turn_with_refusal_hint(
+ resolved_name,
+ &recording_args,
+ start_ms,
+ predicted_refusal,
+ )
}
})
.flatten();
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_turn_latency.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_turn_latency.rs
new file mode 100644
index 0000000000..730a2d2521
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_turn_latency.rs
@@ -0,0 +1,210 @@
+//! D-MAC-7 regression: recording must not multiply action latency.
+//!
+//! The platform application-state hook is process-global, so this file runs
+//! in its own test binary with a deliberately slow fake accessibility
+//! provider. A walk that ignored its budget used to hold every recorded action
+//! for the full provider latency (33 s per click on a Tk window), including
+//! clicks refused before any input was dispatched.
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use cua_driver_core::protocol::ToolResult;
+use cua_driver_core::recording::{
+ set_budgeted_ax_snapshot_fn, set_screenshot_fn, StateCaptureBudget,
+};
+use cua_driver_core::tool::{Tool, ToolDef, ToolRegistry};
+use serde_json::{json, Value};
+
+/// Simulated provider latency when the hook ignores its budget.
+const SLOW_PROVIDER: Duration = Duration::from_secs(6);
+const BUDGET_MS: u64 = 100;
+/// pid whose provider ignores the budget entirely.
+const SLOW_PID: i64 = 4101;
+/// pid whose provider honours the budget and reports a partial tree.
+const BUDGETED_PID: i64 = 4102;
+
+static STATE_CALLS: AtomicUsize = AtomicUsize::new(0);
+
+fn install_hooks() {
+ let png = cua_driver_core::image_utils::encode_rgba_to_png(&[255; 16], 2, 2).unwrap();
+ set_screenshot_fn(move |_, _| Some(png.clone()));
+ set_budgeted_ax_snapshot_fn(|window_id, pid, budget: StateCaptureBudget| {
+ STATE_CALLS.fetch_add(1, Ordering::SeqCst);
+ match pid {
+ Some(SLOW_PID) => {
+ std::thread::sleep(SLOW_PROVIDER);
+ Some(br#"{"tree_markdown":"late"}"#.to_vec())
+ }
+ _ => serde_json::to_vec(&json!({
+ "pid": pid,
+ "window_id": window_id,
+ "element_count": 1,
+ "tree_markdown": "- AXWindow",
+ "truncated": true,
+ "truncation_reason": "timeout",
+ "nodes_visited": 1,
+ "nodes_pending": 7,
+ "walk_elapsed_ms": budget.timeout_ms,
+ "timeout_ms": budget.timeout_ms,
+ }))
+ .ok(),
+ }
+ });
+}
+
+/// Refuses capture-bound calls the way every platform adapter does, and
+/// otherwise reports an immediate unverifiable pixel click.
+struct FakeClick {
+ def: ToolDef,
+}
+
+#[async_trait]
+impl Tool for FakeClick {
+ fn def(&self) -> &ToolDef {
+ &self.def
+ }
+
+ async fn invoke(&self, args: Value) -> ToolResult {
+ if args.get("capture_id").is_some() {
+ return ToolResult::error(
+ "capture binding failed: capture id is unknown. Not dispatching click.",
+ )
+ .with_structured(json!({"code": "capture_not_found", "effect": "refused"}));
+ }
+ ToolResult::text("✅ Posted click (background CGEvent; not driver-verified).")
+ }
+}
+
+fn registry(output_dir: &std::path::Path) -> ToolRegistry {
+ let mut registry = ToolRegistry::new();
+ registry.register(Box::new(FakeClick {
+ def: ToolDef {
+ name: "click".into(),
+ description: "fake click".into(),
+ input_schema: json!({"type": "object"}),
+ read_only: false,
+ destructive: false,
+ idempotent: false,
+ open_world: false,
+ },
+ }));
+ registry
+ .recording
+ .start_with_state_budget(
+ output_dir.to_str().unwrap(),
+ false,
+ None,
+ Some(StateCaptureBudget {
+ timeout_ms: BUDGET_MS,
+ }),
+ )
+ .unwrap();
+ registry
+}
+
+fn evidence(turn: &std::path::Path) -> Value {
+ serde_json::from_slice(&std::fs::read(turn.join("evidence.json")).unwrap()).unwrap()
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn recorded_actions_stay_bounded_with_a_slow_accessibility_provider() {
+ install_hooks();
+ let directory = tempfile::tempdir().unwrap();
+ let registry = registry(directory.path());
+ let backstop = StateCaptureBudget {
+ timeout_ms: BUDGET_MS,
+ }
+ .backstop();
+
+ // 1. A click refused before dispatch walks nothing and returns promptly.
+ let started = Instant::now();
+ let refused = registry
+ .invoke(
+ "click",
+ json!({
+ "pid": SLOW_PID,
+ "window_id": 7,
+ "x": 1,
+ "y": 1,
+ "capture_id": "capture_edd825b82cc0441896ba7e888cd36d94_0000000000000016",
+ }),
+ )
+ .await;
+ let refused_elapsed = started.elapsed();
+ assert_eq!(refused.is_error, Some(true));
+ assert!(
+ refused_elapsed < Duration::from_millis(800),
+ "refused click took {refused_elapsed:?}"
+ );
+ assert_eq!(
+ STATE_CALLS.load(Ordering::SeqCst),
+ 0,
+ "no state walk for a refusal"
+ );
+ let turn = directory.path().join("turn-00001");
+ let manifest = evidence(&turn);
+ for phase in ["before", "after"] {
+ assert_eq!(
+ manifest[phase]["state"],
+ json!({"status": "not_applicable", "classification": "action_refused_before_dispatch"}),
+ "{phase}: {manifest}"
+ );
+ assert_eq!(manifest[phase]["screenshot"]["status"], "captured");
+ assert!(!turn.join(format!("{phase}_state.json")).exists());
+ }
+
+ // 2. A dispatched click against a provider that ignores its budget is
+ // held for at most one backstop, not for the provider's latency.
+ let started = Instant::now();
+ let dispatched = registry
+ .invoke(
+ "click",
+ json!({"pid": SLOW_PID, "window_id": 7, "x": 1, "y": 1}),
+ )
+ .await;
+ let dispatched_elapsed = started.elapsed();
+ assert!(
+ dispatched_elapsed < backstop + Duration::from_millis(900),
+ "recorded click took {dispatched_elapsed:?} (backstop {backstop:?}); result {dispatched:?}"
+ );
+ assert!(dispatched_elapsed < SLOW_PROVIDER);
+ let manifest = evidence(&directory.path().join("turn-00002"));
+ assert_eq!(
+ manifest["before"]["state"],
+ json!({"status": "unavailable", "classification": "state_capture_timeout"})
+ );
+ // The abandoned walk is still running, so the after phase does not start
+ // a second walk of the same hung application.
+ assert_eq!(
+ manifest["after"]["state"],
+ json!({"status": "unavailable", "classification": "state_capture_busy"})
+ );
+ assert_eq!(STATE_CALLS.load(Ordering::SeqCst), 1);
+
+ // 3. A provider that honours its budget receives it and its partial tree
+ // is recorded as truncated evidence.
+ let started = Instant::now();
+ registry
+ .invoke(
+ "click",
+ json!({"pid": BUDGETED_PID, "window_id": 8, "x": 1, "y": 1}),
+ )
+ .await;
+ assert!(started.elapsed() < backstop);
+ let turn = directory.path().join("turn-00003");
+ let manifest = evidence(&turn);
+ for phase in ["before", "after"] {
+ let state = &manifest[phase]["state"];
+ assert_eq!(state["status"], "captured", "{phase}: {manifest}");
+ assert_eq!(state["truncated"], true);
+ assert_eq!(state["truncation_reason"], "timeout");
+ assert_eq!(state["nodes_pending"], 7);
+ assert_eq!(state["timeout_ms"], BUDGET_MS);
+ assert!(turn.join(format!("{phase}_state.json")).is_file());
+ }
+
+ registry.recording.stop_owner(None).unwrap();
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
index fae3e47b15..a7903a7125 100644
--- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
@@ -634,8 +634,8 @@ fn configure_macos_runtime() {
cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
platform_macos::capture::crosshair_png_bytes(png_bytes, x, y).ok()
});
- cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
- platform_macos::recording_hooks::app_state_json_for(window_id, pid)
+ cua_driver_core::recording::set_budgeted_ax_snapshot_fn(|window_id, pid, budget| {
+ platform_macos::recording_hooks::app_state_json_for(window_id, pid, budget)
});
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
platform_macos::recording_hooks::element_window_local_xy(window_id, pid, index)
@@ -653,8 +653,8 @@ fn configure_windows_runtime() {
cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
platform_windows::capture::crosshair_png_bytes(png_bytes, x, y).ok()
});
- cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
- platform_windows::recording_hooks::app_state_json_for(window_id, pid)
+ cua_driver_core::recording::set_budgeted_ax_snapshot_fn(|window_id, pid, budget| {
+ platform_windows::recording_hooks::app_state_json_for(window_id, pid, budget)
});
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
platform_windows::recording_hooks::element_window_local_xy(window_id, pid, index)
@@ -682,8 +682,8 @@ fn configure_linux_runtime(prepare_desktop_environment: bool) {
cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
platform_linux::capture::crosshair_png_bytes(png_bytes, x, y).ok()
});
- cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
- platform_linux::recording_hooks::app_state_json_for(window_id, pid)
+ cua_driver_core::recording::set_budgeted_ax_snapshot_fn(|window_id, pid, budget| {
+ platform_linux::recording_hooks::app_state_json_for(window_id, pid, budget)
});
cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
platform_linux::recording_hooks::element_window_local_xy(window_id, pid, index)
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
index e0e5e81b72..4c4680cfaf 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
@@ -1566,8 +1566,25 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {
.as_ref()
.and_then(|value| value["arguments"]["pid"].as_i64())
.is_some();
+ // An action refused before dispatch cannot change the application, so
+ // the recorder skips its accessibility walk and says so explicitly.
+ let refused_without_dispatch = action.as_ref().is_some_and(|value| {
+ value["result_error"].as_bool() == Some(true)
+ && (value.get("action_truth").is_none()
+ || value["action_truth"]["effect"].as_str() == Some("refused"))
+ });
+ let state_skipped_for_refusal = |phase: &str| {
+ refused_without_dispatch
+ && manifest.as_ref().is_some_and(|value| {
+ value[phase]["state"]["status"] == "not_applicable"
+ && value[phase]["state"]["classification"] == "action_refused_before_dispatch"
+ })
+ };
if state_expected {
for phase in ["before", "after"] {
+ if state_skipped_for_refusal(phase) {
+ continue;
+ }
validate_capture_status(
manifest.as_ref(),
&[phase, "state"],
@@ -1581,6 +1598,9 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {
("after_state.json", "after"),
("app_state.json", "after"),
] {
+ if state_skipped_for_refusal(phase) {
+ continue;
+ }
validate_json_file(
&turn.join(file),
cell_id,
@@ -2536,6 +2556,48 @@ mod tests {
.any(|error| error.contains("turn-00001/click.png")));
}
+ #[test]
+ fn validator_accepts_state_skipped_only_for_a_refusal_before_dispatch() {
+ let refused_evidence = br#"{
+ "schema":"cua-turn-evidence/v1",
+ "before":{"state":{"status":"not_applicable","classification":"action_refused_before_dispatch"},"screenshot":{"status":"captured"}},
+ "after":{"state":{"status":"not_applicable","classification":"action_refused_before_dispatch"},"screenshot":{"status":"captured"}},
+ "click":{"status":"not_applicable","classification":"action_refused_before_dispatch"}
+ }"#;
+ let (root, case, result, turn) = complete_turn_fixture();
+ std::fs::write(
+ turn.join("action.json"),
+ br#"{
+ "tool":"click",
+ "arguments":{"pid":1,"window_id":2,"x":3,"y":4,"capture_id":"expired"},
+ "click_point":{"x":3,"y":4},
+ "result_error":true,
+ "action_truth":{"effect":"refused","refusal":{"code":"capture_expired"}}
+ }"#,
+ )
+ .unwrap();
+ std::fs::write(turn.join("evidence.json"), refused_evidence).unwrap();
+ for file in [
+ "click.png",
+ "before_state.json",
+ "after_state.json",
+ "app_state.json",
+ ] {
+ std::fs::remove_file(turn.join(file)).unwrap();
+ }
+ validate_catalog(&[case], &[result], Some(root.path()), true)
+ .expect("a refusal before dispatch needs no application-state walk");
+
+ // A dispatched click must still carry captured state.
+ let (root, case, result, turn) = complete_turn_fixture();
+ std::fs::write(turn.join("evidence.json"), refused_evidence).unwrap();
+ let errors = validate_catalog(&[case], &[result], Some(root.path()), true)
+ .expect_err("a dispatched action cannot skip state as refused");
+ assert!(errors
+ .iter()
+ .any(|error| error.contains("turn-00001/before state")));
+ }
+
#[test]
fn validator_accepts_click_refused_before_target_resolution_without_marker() {
let (root, case, result, turn) = complete_turn_fixture();
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
index 5e327a79d1..98ee2c4770 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
@@ -1,18 +1,29 @@
//! Application-state snapshots used by trajectory recording on Linux.
+/// Per-turn application state for trajectory recording, walked under the
+/// shared `get_window_state` budget. The walk reports the shared walk fields so
+/// the turn evidence says when the tree is partial.
#[cfg(target_os = "linux")]
-pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> {
+pub fn app_state_json_for(
+ window_id: Option<u64>,
+ pid: Option<i64>,
+ budget: cua_driver_core::recording::StateCaptureBudget,
+) -> Option<Vec<u8>> {
if tokio::runtime::Handle::try_current().is_ok() {
- return std::thread::spawn(move || app_state_json_for_blocking(window_id, pid))
+ return std::thread::spawn(move || app_state_json_for_blocking(window_id, pid, budget))
.join()
.ok()
.flatten();
}
- app_state_json_for_blocking(window_id, pid)
+ app_state_json_for_blocking(window_id, pid, budget)
}
#[cfg(target_os = "linux")]
-fn app_state_json_for_blocking(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> {
+fn app_state_json_for_blocking(
+ window_id: Option<u64>,
+ pid: Option<i64>,
+ budget: cua_driver_core::recording::StateCaptureBudget,
+) -> Option<Vec<u8>> {
let pid = u32::try_from(pid?).ok()?;
let window_id = if crate::wayland::is_inject_mode() {
// Most injected actions already carry the protocol-verified window id.
@@ -26,12 +37,13 @@ fn app_state_json_for_blocking(window_id: Option<u64>, pid: Option<i64>) -> Opti
} else {
resolve_window_for_recording(pid, window_id)?.xid
};
+ let timeout = std::time::Duration::from_millis(budget.timeout_ms);
let result = if crate::wayland::is_inject_mode() {
- // Evidence capture runs inside the daemon call. Keep it below the
- // transport deadline so an unresponsive renderer cannot block input.
- crate::atspi::walk_tree_for_recording(pid, window_id, std::time::Duration::from_secs(2))
+ // Evidence capture runs inside the daemon call. One attempt only, so
+ // an unresponsive renderer cannot block input past the budget.
+ crate::atspi::walk_tree_for_recording(pid, window_id, timeout)
} else {
- crate::atspi::walk_tree(pid, window_id, None)
+ crate::atspi::walk_tree_bounded_within(pid, window_id, None, None, None, timeout)
};
if result.nodes.is_empty() || result.tree_markdown.trim().is_empty() {
return None;
@@ -41,12 +53,21 @@ fn app_state_json_for_blocking(window_id: Option<u64>, pid: Option<i64>) -> Opti
.iter()
.filter(|node| node.element_index.is_some())
.count();
- let payload = serde_json::json!({
+ let mut payload = serde_json::json!({
"pid": pid,
"window_id": window_id,
"element_count": element_count,
"tree_markdown": result.tree_markdown,
+ // The shared walk fields (see cua_driver_core::walk_budget).
+ "truncated": result.truncated,
+ "nodes_visited": result.nodes_visited,
+ "nodes_pending": result.nodes_pending,
+ "walk_elapsed_ms": result.elapsed_ms as u64,
+ "timeout_ms": budget.timeout_ms,
});
+ if let Some(reason) = result.truncation_reason {
+ payload["truncation_reason"] = serde_json::json!(reason);
+ }
serde_json::to_vec_pretty(&payload).ok()
}
@@ -214,7 +235,11 @@ fn resolve_window_for_recording(
}
#[cfg(not(target_os = "linux"))]
-pub fn app_state_json_for(_window_id: Option<u64>, _pid: Option<i64>) -> Option<Vec<u8>> {
+pub fn app_state_json_for(
+ _window_id: Option<u64>,
+ _pid: Option<i64>,
+ _budget: cua_driver_core::recording::StateCaptureBudget,
+) -> Option<Vec<u8>> {
None
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
index c0c02277eb..9158903a82 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
@@ -10,25 +10,43 @@ pub fn set_element_cache(cache: Arc<ElementCache>) {
register_runtime_cache(&cache);
}
-pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> {
+/// Per-turn application state for trajectory recording. The walk shares the
+/// `get_window_state` budget semantics: it stops when `budget.timeout_ms` runs
+/// out and reports the shared walk fields so the turn evidence can say the
+/// tree is partial.
+pub fn app_state_json_for(
+ window_id: Option<u64>,
+ pid: Option<i64>,
+ budget: cua_driver_core::recording::StateCaptureBudget,
+) -> Option<Vec<u8>> {
let pid = i32::try_from(pid?).ok()?;
let resolved_wid = match window_id {
Some(w) => u32::try_from(w).ok()?,
None => crate::windows::resolve_main_window_id(pid).ok()?,
};
- let result = crate::ax::tree::walk_tree(pid, Some(resolved_wid), None);
+ let result = crate::ax::tree::walk_tree_budgeted(
+ pid,
+ Some(resolved_wid),
+ None,
+ crate::ax::tree::DEFAULT_MAX_DEPTH,
+ cua_driver_core::walk_budget::WalkBudget::new(
+ budget.timeout_ms,
+ crate::ax::tree::DEFAULT_MAX_ELEMENTS,
+ ),
+ );
let _payload = CachedSnapshot::from_nodes(&result.nodes);
let element_count = result
.nodes
.iter()
.filter(|node| node.element_index.is_some())
.count();
- let payload = serde_json::json!({
+ let mut payload = serde_json::json!({
"pid": pid,
"window_id": resolved_wid,
"element_count": element_count,
"tree_markdown": result.tree_markdown,
});
+ result.walk.apply(&mut payload);
serde_json::to_vec_pretty(&payload).ok()
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
index 9028d342c0..131e9b044f 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
@@ -96,10 +96,23 @@ pub(crate) fn capture_dispatch_click_target(window_id: u64, pid: u32, x: i32, y:
}
#[cfg(target_os = "windows")]
-pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> {
+/// Per-turn application state for trajectory recording, walked under the
+/// shared `get_window_state` budget. The walk reports the shared walk fields so
+/// the turn evidence says when the tree is partial. UIA's bulk cache fetch is
+/// not interruptible per node; the recorder's backstop bounds it.
+pub fn app_state_json_for(
+ window_id: Option<u64>,
+ pid: Option<i64>,
+ budget: cua_driver_core::recording::StateCaptureBudget,
+) -> Option<Vec<u8>> {
let pid = u32::try_from(pid?).ok()?;
let hwnd = resolve_window_for_recording(window_id, Some(pid.into()))?;
- let result = crate::uia::walk_tree(hwnd, None);
+ let mut walk = cua_driver_core::walk_budget::WalkBudget::new(
+ budget.timeout_ms,
+ crate::uia::DEFAULT_MAX_TOTAL_ELEMENTS,
+ );
+ let result =
+ crate::uia::walk_tree_budgeted(hwnd, None, crate::uia::DEFAULT_MAX_DEPTH, &mut walk);
let kind = if result.nodes.iter().any(|node| node.msaa_role.is_some()) {
SnapshotKind::Msaa
} else {
@@ -111,12 +124,13 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve
.iter()
.filter(|n| n.element_index.is_some())
.count();
- let payload = serde_json::json!({
+ let mut payload = serde_json::json!({
"pid": pid,
"window_id": hwnd,
"element_count": element_count,
"tree_markdown": result.tree_markdown,
});
+ walk.outcome().apply(&mut payload);
serde_json::to_vec_pretty(&payload).ok()
}
@@ -165,7 +179,11 @@ pub fn element_window_local_xy(
}
#[cfg(not(target_os = "windows"))]
-pub fn app_state_json_for(_window_id: Option<u64>, _pid: Option<i64>) -> Option<Vec<u8>> {
+pub fn app_state_json_for(
+ _window_id: Option<u64>,
+ _pid: Option<i64>,
+ _budget: cua_driver_core::recording::StateCaptureBudget,
+) -> Option<Vec<u8>> {
None
}
#[cfg(not(target_os = "windows"))]