Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): initialize recording hooks for embedded platform registries (#5785)

injaneity0a05cd400695
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
index ece86d6452..72c65cff49 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
@@ -26,7 +26,7 @@ use crate::video::{self, VideoBackend, VideoMetadata};
// ── Platform screenshot callback ─────────────────────────────────────────────
//
-// Registered once at startup by each platform crate. Takes (window_id, pid)
+// Installed during tool registration by each platform crate. Takes (window_id, pid)
// and returns raw PNG bytes, or None if capture fails. The callback is called
// synchronously from write_turn (a blocking context).
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/video.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/video.rs
index ab951c4350..72081ec31e 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/video.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/video.rs
@@ -4,8 +4,8 @@
//! Cross-platform video-capture abstraction.
//!
//! The recording session calls into a single `VideoBackend` trait; the
-//! concrete implementation is selected at process startup by the
-//! platform crate. Today:
+//! concrete implementation is installed during platform tool registration.
+//! Today:
//!
//! - **macOS:** native ScreenCaptureKit via `platform_macos::video_sckit`
//! (no extra TCC grant — inherits cua-driver's own Screen Recording
@@ -14,11 +14,11 @@
//! (`gdigrab` / `x11grab` input + libx264 encode).
//! - **Linux Wayland:** compositor-routed capture via
//! `platform_linux::video_wayland` (GNOME Shell helper frames, wlroots
-//! `wf-recorder`, or xdg-desktop-portal ScreenCast over PipeWire).
+//! `wf-recorder`, or xdg-desktop-portal ScreenCast over PipeWire),
+//! selected when capture starts.
//!
-//! The factory is registered with `set_video_backend_factory` from each
-//! platform's `main.rs` startup block, mirroring how `SCREENSHOT_FN` /
-//! `AX_SNAPSHOT_FN` are wired in `recording.rs`.
+//! The factory is registered with `set_video_backend_factory` during platform
+//! tool registration, alongside the recording callbacks in `recording.rs`.
//!
//! `VideoMetadata` is the shape `RecordingSession` stamps into
//! `session.json` after `stop()` — kept identical to the prior concrete
diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
index ee4626294e..6922b94c93 100644
--- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
@@ -563,7 +563,6 @@ pub(crate) fn tool_inventory(options: RuntimeOptions) -> Value {
fn build_registry(options: &RuntimeOptions) -> ToolRegistry {
#[cfg(target_os = "macos")]
let mut registry = {
- configure_macos_runtime();
platform_macos::register_tools_with_cursor_and_provider(
options.authorization_host.clone(),
options.cursor.clone(),
@@ -575,7 +574,6 @@ fn build_registry(options: &RuntimeOptions) -> ToolRegistry {
#[cfg(target_os = "windows")]
let mut registry = {
- configure_windows_runtime();
platform_windows::register_tools_with_cursor_and_provider(
options.authorization_host.clone(),
options.cursor.clone(),
@@ -632,54 +630,6 @@ fn build_registry(options: &RuntimeOptions) -> ToolRegistry {
registry
}
-#[cfg(target_os = "macos")]
-fn configure_macos_runtime() {
- cua_driver_core::recording::set_screenshot_fn(|window_id, pid| {
- if let Some(window_id) = window_id {
- platform_macos::capture::screenshot_window_bytes(window_id as u32).ok()
- } else if let Some(pid) = pid {
- platform_macos::windows::resolve_main_window_id(pid as i32)
- .ok()
- .and_then(|window_id| {
- platform_macos::capture::screenshot_window_bytes(window_id).ok()
- })
- } else {
- platform_macos::capture::screenshot_display_bytes().ok()
- }
- });
- cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
- platform_macos::capture::crosshair_png_bytes(png_bytes, x, y).ok()
- });
- cua_driver_core::recording::set_budgeted_ax_snapshot_fn(|window_id, pid, budget| {
- platform_macos::recording_hooks::app_state_json_for(window_id, pid, budget)
- });
- cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
- platform_macos::recording_hooks::element_window_local_xy(window_id, pid, index)
- });
- cua_driver_core::video::set_video_backend_factory(Box::new(
- platform_macos::video_sckit::SckitVideoBackendFactory,
- ));
-}
-
-#[cfg(target_os = "windows")]
-fn configure_windows_runtime() {
- cua_driver_core::recording::set_classified_screenshot_fn(|window_id, pid| {
- platform_windows::recording_hooks::screenshot_for_recording(window_id, pid)
- });
- cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
- platform_windows::capture::crosshair_png_bytes(png_bytes, x, y).ok()
- });
- cua_driver_core::recording::set_budgeted_ax_snapshot_fn(|window_id, pid, budget| {
- platform_windows::recording_hooks::app_state_json_for(window_id, pid, budget)
- });
- cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
- platform_windows::recording_hooks::element_window_local_xy(window_id, pid, index)
- });
- cua_driver_core::video::set_video_backend_factory(Box::new(
- cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
- ));
-}
-
#[cfg(all(target_os = "linux", not(target_env = "musl")))]
fn configure_linux_runtime(prepare_desktop_environment: bool) {
if prepare_desktop_environment {
@@ -692,27 +642,6 @@ fn configure_linux_runtime(prepare_desktop_environment: bool) {
}
}
}
- cua_driver_core::recording::set_screenshot_fn(|window_id, pid| {
- platform_linux::recording_hooks::screenshot_for_recording(window_id, pid)
- });
- cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
- platform_linux::capture::crosshair_png_bytes(png_bytes, x, y).ok()
- });
- cua_driver_core::recording::set_budgeted_ax_snapshot_fn(|window_id, pid, budget| {
- platform_linux::recording_hooks::app_state_json_for(window_id, pid, budget)
- });
- cua_driver_core::recording::set_element_bounds_fn(|window_id, pid, index| {
- platform_linux::recording_hooks::element_window_local_xy(window_id, pid, index)
- });
- if platform_linux::wayland::is_wayland() {
- cua_driver_core::video::set_video_backend_factory(Box::new(
- platform_linux::video_wayland::WaylandVideoBackendFactory,
- ));
- } else {
- cua_driver_core::video::set_video_backend_factory(Box::new(
- cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
- ));
- }
}
#[cfg(test)]
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
index 11ccd69656..057c5ff383 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
@@ -3,6 +3,34 @@
//! Application-state snapshots used by trajectory recording on Linux.
+#[cfg(target_os = "linux")]
+use cua_driver_core::video::{VideoBackend, VideoBackendFactory};
+
+#[cfg(target_os = "linux")]
+pub(crate) fn install() {
+ cua_driver_core::recording::set_screenshot_fn(screenshot_for_recording);
+ cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
+ crate::capture::crosshair_png_bytes(png_bytes, x, y).ok()
+ });
+ cua_driver_core::recording::set_budgeted_ax_snapshot_fn(app_state_json_for);
+ cua_driver_core::recording::set_element_bounds_fn(element_window_local_xy);
+ cua_driver_core::video::set_video_backend_factory(Box::new(LinuxVideoBackendFactory));
+}
+
+#[cfg(target_os = "linux")]
+struct LinuxVideoBackendFactory;
+
+#[cfg(target_os = "linux")]
+impl VideoBackendFactory for LinuxVideoBackendFactory {
+ fn start(&self, output_path: &std::path::Path) -> anyhow::Result<Box<dyn VideoBackend>> {
+ if crate::wayland::is_wayland() {
+ crate::video_wayland::WaylandVideoBackendFactory.start(output_path)
+ } else {
+ cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory.start(output_path)
+ }
+ }
+}
+
/// Per-turn application state for trajectory recording, walked under the
/// shared `get_window_state` budget. The walk reports the shared walk fields so
/// the turn evidence says when the tree is partial.
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 5aa551d2ec..8c5325c670 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -4433,6 +4433,7 @@ pub fn build_registry_with_provider(
) -> ToolRegistry {
let mut r = ToolRegistry::new_with_protected_consent_provider(provider);
let state = ToolState::new_with_capture_service(r.capture_service());
+ crate::recording_hooks::install();
if crate::wayland::is_wayland() && crate::wayland::hyprland::is_session() {
let recording_state = Arc::downgrade(&state);
r.recording
diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring/mod.rs b/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring/mod.rs
new file mode 100644
index 0000000000..5abfff8a75
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring/mod.rs
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: MIT
+
+use cua_driver_core::tool::ToolRegistry;
+use serde_json::{json, Value};
+
+pub async fn assert_recording_wired(registry: ToolRegistry) {
+ let directory = tempfile::tempdir().unwrap();
+ let video_error = match cua_driver_core::video::start_video(&directory.path().join("probe.mp4"))
+ {
+ Ok(video) => {
+ let _ = video.stop();
+ String::new()
+ }
+ Err(error) => error.to_string(),
+ };
+ let started = registry
+ .invoke("start_recording", json!({"output_dir": directory.path()}))
+ .await;
+ assert_ne!(started.is_error, Some(true), "{started:?}");
+ let result = registry
+ .invoke("set_config", json!({"max_image_dimension": 0}))
+ .await;
+ assert_ne!(result.is_error, Some(true), "{result:?}");
+ let stopped = registry.invoke("stop_recording", json!({})).await;
+ assert_ne!(stopped.is_error, Some(true), "{stopped:?}");
+ let evidence: Value = serde_json::from_slice(
+ &std::fs::read(directory.path().join("turn-00001/evidence.json")).unwrap(),
+ )
+ .unwrap();
+ for phase in ["before", "after"] {
+ assert_ne!(
+ evidence[phase]["screenshot"]["classification"], "capture_hook_unavailable",
+ "{evidence}"
+ );
+ }
+ assert!(
+ !video_error.contains("no video backend registered"),
+ "{video_error}"
+ );
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_nested_wayland.rs b/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_nested_wayland.rs
new file mode 100644
index 0000000000..3c7548f1d7
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_nested_wayland.rs
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: MIT
+
+#![cfg(target_os = "linux")]
+
+use std::os::unix::fs::PermissionsExt;
+
+#[test]
+fn video_uses_the_nested_session_created_after_registration() {
+ std::env::remove_var("DISPLAY");
+ std::env::remove_var("WAYLAND_DISPLAY");
+ std::env::remove_var("CUA_WAYLAND_NEST");
+ let _registry = platform_linux::register_tools();
+ let directory = tempfile::tempdir().unwrap();
+ let compositor = directory.path().join("compositor");
+ std::fs::write(
+ &compositor,
+ "#!/bin/sh\n: > \"$XDG_RUNTIME_DIR/wayland-1\"\n",
+ )
+ .unwrap();
+ std::fs::set_permissions(&compositor, std::fs::Permissions::from_mode(0o700)).unwrap();
+ std::env::set_var("XDG_RUNTIME_DIR", directory.path());
+ std::env::set_var(
+ "DBUS_SESSION_BUS_ADDRESS",
+ format!("unix:path={}/missing-bus", directory.path().display()),
+ );
+ std::env::set_var("CUA_WAYLAND_NEST", "1");
+ std::env::set_var("CUA_WAYLAND_NEST_COMPOSITOR", compositor);
+ std::env::set_var(platform_linux::wayland::ENABLE_WAYLAND_ENV, "1");
+ platform_linux::wayland::ensure_nested_session();
+ assert_eq!(std::env::var("WAYLAND_DISPLAY").unwrap(), "wayland-1");
+ std::env::set_var("PATH", directory.path());
+ let error = match cua_driver_core::video::start_video(&directory.path().join("recording.mp4")) {
+ Ok(video) => {
+ let _ = video.stop();
+ panic!("video cannot start without a recorder executable");
+ }
+ Err(error) => error.to_string(),
+ };
+ assert!(error.contains("wf-recorder not found"), "{error}");
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_register_tools.rs b/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_register_tools.rs
new file mode 100644
index 0000000000..129f09fc8a
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_register_tools.rs
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: MIT
+
+#![cfg(target_os = "linux")]
+
+mod recording_wiring;
+
+#[tokio::test]
+async fn registration_installs_recording_support() {
+ std::env::remove_var("DISPLAY");
+ std::env::remove_var("WAYLAND_DISPLAY");
+ std::env::remove_var("CUA_WAYLAND_NEST");
+ recording_wiring::assert_recording_wired(platform_linux::register_tools()).await;
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_with_cursor.rs b/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_with_cursor.rs
new file mode 100644
index 0000000000..c3fd6b9d14
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/tests/recording_wiring_with_cursor.rs
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: MIT
+
+#![cfg(target_os = "linux")]
+
+mod recording_wiring;
+
+#[tokio::test]
+async fn cursor_registration_installs_recording_support() {
+ std::env::remove_var("DISPLAY");
+ std::env::remove_var("WAYLAND_DISPLAY");
+ std::env::remove_var("CUA_WAYLAND_NEST");
+ let config = cursor_overlay::CursorConfig {
+ enabled: false,
+ ..Default::default()
+ };
+ recording_wiring::assert_recording_wired(platform_linux::register_tools_with_cursor(
+ config, false,
+ ))
+ .await;
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
index a7b2a2b742..fbcd85a9bf 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/recording_hooks.rs
@@ -7,6 +7,28 @@ use cua_driver_core::element_token::ResolvedElement;
use cua_driver_core::snapshot_store::current_runtime_store;
use serde_json::Value;
+pub(crate) fn install() {
+ cua_driver_core::recording::set_screenshot_fn(|window_id, pid| {
+ if let Some(window_id) = window_id {
+ crate::capture::screenshot_window_bytes(window_id as u32).ok()
+ } else if let Some(pid) = pid {
+ crate::windows::resolve_main_window_id(pid as i32)
+ .ok()
+ .and_then(|window_id| crate::capture::screenshot_window_bytes(window_id).ok())
+ } else {
+ crate::capture::screenshot_display_bytes().ok()
+ }
+ });
+ cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
+ crate::capture::crosshair_png_bytes(png_bytes, x, y).ok()
+ });
+ cua_driver_core::recording::set_budgeted_ax_snapshot_fn(app_state_json_for);
+ cua_driver_core::recording::set_element_bounds_fn(element_window_local_xy);
+ cua_driver_core::video::set_video_backend_factory(Box::new(
+ crate::video_sckit::SckitVideoBackendFactory,
+ ));
+}
+
/// Per-turn application state for trajectory recording. The walk shares the
/// `get_window_state` budget semantics: it stops when `budget.timeout_ms` runs
/// out and reports the shared walk fields so the turn evidence can say the
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index 52528fad48..9bc79d38c9 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -1063,6 +1063,7 @@ pub fn register_all(
// Share the element cache with the recording-hook layer so it can
// resolve element_index → window-local screenshot coords for click.png.
cua_driver_core::snapshot_store::register_runtime_store(&state.snapshots);
+ crate::recording_hooks::install();
// Drop a disconnecting session's config overrides + owned cursor on
// `session_end`. The daemon fans the session id out to this hook;
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
index 6f5b90c61c..1ad4656572 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
@@ -34,6 +34,19 @@ use windows::Win32::UI::WindowsAndMessaging::{
GetLastActivePopup, GetWindowThreadProcessId, IsWindow, IsWindowVisible,
};
+#[cfg(target_os = "windows")]
+pub(crate) fn install() {
+ cua_driver_core::recording::set_classified_screenshot_fn(screenshot_for_recording);
+ cua_driver_core::recording::set_click_marker_fn(|png_bytes, x, y| {
+ crate::capture::crosshair_png_bytes(png_bytes, x, y).ok()
+ });
+ cua_driver_core::recording::set_budgeted_ax_snapshot_fn(app_state_json_for);
+ cua_driver_core::recording::set_element_bounds_fn(element_window_local_xy);
+ cua_driver_core::video::set_video_backend_factory(Box::new(
+ cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
+ ));
+}
+
#[cfg(target_os = "windows")]
pub fn set_snapshots(cache: Arc<Snapshots>) {
register_runtime_store(&cache);
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 46c231da24..b51ef4c591 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -1290,6 +1290,7 @@ pub fn build_registry_with_provider(
// Share the element cache with the recording-hook layer so it can
// resolve element_index → window-local screenshot coords for click.png.
crate::recording_hooks::set_snapshots(state.snapshots.clone());
+ crate::recording_hooks::install();
// Drop a session's owned cursor on `session_end` (explicit end_session, the
// CLI `session end` verb, or the daemon idle-TTL sweep). The session id IS
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/tests/linux_desktop.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/tests/linux_desktop.rs
index 53192e5672..d23f3ecec9 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/tests/linux_desktop.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/tests/linux_desktop.rs
@@ -1006,6 +1006,189 @@ async fn presence_two_clients_and_an_agent_cursor() {
assert!(saw_agent, "alice sees the agent's cursor");
}
+#[tokio::test]
+async fn driver_recording_captures_screenshots_state_and_video() {
+ use cua_proto::env::v1::{driver_service_client::DriverServiceClient, CallToolRequest};
+ use serde_json::{json, Value};
+
+ async fn call(driver: &mut DriverServiceClient<Ch>, name: &str, args: Value) -> Value {
+ let response = driver
+ .call_tool(CallToolRequest {
+ name: name.into(),
+ arguments_json: args.to_string(),
+ ..Default::default()
+ })
+ .await
+ .unwrap()
+ .into_inner();
+ assert!(!response.is_error, "{name}: {response:?}");
+ if response.structured_json.is_empty() {
+ Value::Null
+ } else {
+ serde_json::from_str(&response.structured_json).unwrap()
+ }
+ }
+
+ if !enabled() {
+ return;
+ }
+ let title = unique("gtk-recording");
+ let log = format!("/tmp/{title}.jsonl");
+ let child = spawn_fixture(
+ "python3",
+ &[
+ std::env::var("CUA_ENV_GTK_FIXTURE").expect("CUA_ENV_GTK_FIXTURE"),
+ "--title".into(),
+ title.clone(),
+ "--log".into(),
+ log.clone(),
+ ],
+ )
+ .await;
+ let fixture = Fixture { child, log, title };
+ fixture.wait_for("ready", Duration::from_secs(15)).await;
+ let pid = fixture
+ .events()
+ .await
+ .iter()
+ .find_map(|event| event["pid"].as_u64())
+ .unwrap();
+ let mut driver = DriverServiceClient::new(channel().await);
+ call(
+ &mut driver,
+ "start_session",
+ json!({"session": fixture.title}),
+ )
+ .await;
+ let mut token = None;
+ for _ in 0..30 {
+ let state = call(
+ &mut driver,
+ "get_window_state",
+ json!({"pid": pid, "tree_format": "elements", "session": fixture.title}),
+ )
+ .await;
+ token = state["elements"].as_array().and_then(|elements| {
+ elements
+ .iter()
+ .find(|element| matches!(element["role"].as_str(), Some("text" | "entry")))
+ .and_then(|element| element["element_token"].as_str())
+ .map(str::to_owned)
+ });
+ if token.is_some() {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(300)).await;
+ }
+ let token = token.expect("GTK text entry token");
+ let output = format!("/tmp/{}-trajectory", fixture.title);
+ let started = call(
+ &mut driver,
+ "start_recording",
+ json!({"output_dir": output, "record_video": true, "session": fixture.title}),
+ )
+ .await;
+ call(
+ &mut driver,
+ "type_text",
+ json!({"pid": pid, "element_token": token, "text": "recorded text", "session": fixture.title}),
+ )
+ .await;
+ call(
+ &mut driver,
+ "stop_recording",
+ json!({"session": fixture.title}),
+ )
+ .await;
+ call(
+ &mut driver,
+ "end_session",
+ json!({"session": fixture.title}),
+ )
+ .await;
+ // The video backend must be registered. Whether it can encode depends on
+ // the target image shipping ffmpeg, so a finalized MP4 is required only there.
+ let has_ffmpeg = !run("sh", &["-c", "command -v ffmpeg || true"])
+ .await
+ .is_empty();
+ if has_ffmpeg {
+ assert_eq!(started["video_active"], true, "{started}");
+ } else {
+ assert!(
+ !started["last_error"]
+ .as_str()
+ .unwrap_or_default()
+ .contains("no video backend registered"),
+ "{started}"
+ );
+ }
+ assert!(fixture
+ .events()
+ .await
+ .iter()
+ .any(|event| event["value"] == "recorded text"));
+ let evidence: Value = serde_json::from_str(
+ &run(
+ "python3",
+ &[
+ "-c",
+ "import pathlib,sys; print(pathlib.Path(sys.argv[1]).read_text())",
+ &format!("{output}/turn-00001/evidence.json"),
+ ],
+ )
+ .await,
+ )
+ .unwrap();
+ for phase in ["before", "after"] {
+ assert_eq!(
+ evidence[phase]["screenshot"]["status"], "captured",
+ "{evidence}"
+ );
+ assert_eq!(evidence[phase]["state"]["status"], "captured", "{evidence}");
+ let state: Value = serde_json::from_str(
+ &run(
+ "python3",
+ &[
+ "-c",
+ "import pathlib,sys; print(pathlib.Path(sys.argv[1]).read_text())",
+ &format!("{output}/turn-00001/{phase}_state.json"),
+ ],
+ )
+ .await,
+ )
+ .unwrap();
+ assert!(state["element_count"].as_u64().unwrap() > 0, "{state}");
+ assert!(
+ !state["tree_markdown"].as_str().unwrap().is_empty(),
+ "{state}"
+ );
+ let decoded = run(
+ "python3",
+ &["-c", "import sys; from gi.repository import GdkPixbuf; p=GdkPixbuf.Pixbuf.new_from_file(sys.argv[1]); print(p.get_width() > 0 and p.get_height() > 0)", &format!("{output}/turn-00001/{phase}.png")],
+ ).await;
+ assert_eq!(decoded, "True");
+ }
+ if !has_ffmpeg {
+ return;
+ }
+ let duration = run(
+ "ffprobe",
+ &[
+ "-v",
+ "error",
+ "-select_streams",
+ "v:0",
+ "-show_entries",
+ "stream=duration",
+ "-of",
+ "default=nw=1:nk=1",
+ &format!("{output}/recording.mp4"),
+ ],
+ )
+ .await;
+ assert!(duration.parse::<f64>().unwrap() > 0.0);
+}
+
#[tokio::test]
async fn accessibility_finds_and_presses_a_gtk_button() {
if !enabled() {
diff --git a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs b/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
index 0758355e06..934a77aab9 100644
--- a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
@@ -1245,6 +1245,64 @@ mod tests {
use super::*;
+ #[cfg(all(target_os = "linux", not(cua_headless)))]
+ #[tokio::test]
+ async fn fallback_platform_tools_install_recording_support() {
+ if std::env::var_os("CUA_TEST_RECORDING_WIRING").is_none() {
+ let home = tempfile::tempdir().unwrap();
+ let status = std::process::Command::new(std::env::current_exe().unwrap())
+ .args([
+ "--exact",
+ "tests::fallback_platform_tools_install_recording_support",
+ "--nocapture",
+ ])
+ .env("CUA_TEST_RECORDING_WIRING", "1")
+ .env("HOME", home.path())
+ .env_remove("DISPLAY")
+ .env_remove("WAYLAND_DISPLAY")
+ .env_remove("CUA_WAYLAND_NEST")
+ .status()
+ .unwrap();
+ assert!(status.success());
+ return;
+ }
+ let tools = platform_tools().unwrap();
+ let directory = tempfile::tempdir().unwrap();
+ let video_error =
+ match cua_driver_core::video::start_video(&directory.path().join("probe.mp4")) {
+ Ok(video) => {
+ let _ = video.stop();
+ String::new()
+ }
+ Err(error) => error.to_string(),
+ };
+ for (name, args) in [
+ (
+ "start_recording",
+ serde_json::json!({"output_dir": directory.path()}),
+ ),
+ ("set_config", serde_json::json!({"max_image_dimension": 0})),
+ ("stop_recording", serde_json::json!({})),
+ ] {
+ let result = tools.invoke_tool(name, args).await.unwrap();
+ assert_ne!(result["isError"], true, "{result}");
+ }
+ let evidence: serde_json::Value = serde_json::from_slice(
+ &std::fs::read(directory.path().join("turn-00001/evidence.json")).unwrap(),
+ )
+ .unwrap();
+ for phase in ["before", "after"] {
+ assert_ne!(
+ evidence[phase]["screenshot"]["classification"], "capture_hook_unavailable",
+ "{evidence}"
+ );
+ }
+ assert!(
+ !video_error.contains("no video backend registered"),
+ "{video_error}"
+ );
+ }
+
/// The WireGuard library's retry/expiry messages stay out of the log
/// (also when a service was installed with `CUA_ENV_LOG=info`), unless
/// the filter names them.