Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
perf(cua-driver): reuse one session-bus connection for the GNOME helper (#4879)
Francesco Bonacci48cc49146349
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rsindex 111b4ed3a5..0f055c7d98 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs@@ -18,12 +18,11 @@//!//! Everything here is **best-effort**: if the extension isn't installed/enabled//! the calls return `None` / no-op and callers keep the prior behaviour (no-//! screen coords, no Wayland cursor). Uses a short-lived `gdbus` subprocess so-//! there's no zbus blocking-feature or async-context coupling — the calls are-//! infrequent (once per `get_window_state`, a few per click).+//! screen coords, no Wayland cursor). Reuses one session-bus connection on a+//! dedicated runtime; preview frames do not spawn subprocesses or poll for exit.use std::os::unix::fs::{MetadataExt, PermissionsExt};-use std::process::Command;+mod transport;use std::time::Duration;use crate::x11::WindowInfo;@@ -85,30 +84,7 @@ fn gdbus_call_to(args: &[String],timeout: Duration,) -> Option<String> {- let mut cmd = Command::new("gdbus");- cmd.arg("call")- .arg("--session")- .arg("--dest")- .arg(destination)- .arg("--object-path")- .arg(object_path)- .arg("--method")- .arg(method);- for a in args {- cmd.arg(a);- }- // gdbus is local IPC; cap it so a wedged shell can't stall the caller.- let child = cmd- .stdin(std::process::Stdio::null())- .stdout(std::process::Stdio::piped())- .stderr(std::process::Stdio::null())- .spawn()- .ok()?;- let out = wait_timeout(child, timeout)?;- if !out.status.success() {- return None;- }- Some(String::from_utf8_lossy(&out.stdout).into_owned())+ transport::call(destination, object_path, method, args, timeout)}/// Resolve the helper's immutable unique bus name and prove that it is hosted@@ -247,53 +223,6 @@ fn decode_capture(raw: &str) -> Option<Vec<u8>> {B64.decode(&raw[start..end]).ok()}-/// `Child::wait` with a deadline (no extra crates). Kills + reaps on timeout.-fn wait_timeout(mut child: std::process::Child, dur: Duration) -> Option<std::process::Output> {- use std::io::Read;-- // Drain stdout while the child is running. Capture() returns a base64 PNG- // that readily exceeds a pipe's ~64 KiB capacity; waiting for exit before- // reading deadlocks the child on a full pipe and turns a healthy Shell- // response into a false timeout.- let stdout = child.stdout.take()?;- let reader = std::thread::spawn(move || {- let mut stdout = stdout;- let mut bytes = Vec::new();- stdout.read_to_end(&mut bytes).ok()?;- Some(bytes)- });- let deadline = std::time::Instant::now() + dur;- let status = loop {- match child.try_wait() {- Ok(Some(status)) => break status,- Ok(None) => {- if std::time::Instant::now() >= deadline {- let _ = child.kill();- let status = child.wait().ok()?;- let _ = reader.join();- if !status.success() {- return None;- }- return None;- }- std::thread::sleep(Duration::from_millis(15));- }- Err(_) => {- let _ = child.kill();- let _ = child.wait();- let _ = reader.join();- return None;- }- }- };- let stdout = reader.join().ok().flatten()?;- Some(std::process::Output {- status,- stdout,- stderr: Vec::new(),- })-}-/// Screen origin of the compositor frame backing `pid`.////// `screenshot_window_dispatch` crops the Shell stage to this same frame@@ -308,13 +237,7 @@ pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> {}fn parse_window_origin(raw: &str, pid: u32) -> Option<(i32, i32)> {- // gdbus prints a GVariant tuple like `('[{"pid":..,"x":..}]',)`. Pull the- // JSON array out robustly (first '[' .. last ']') rather than parsing the- // GVariant wrapper, so an apostrophe in a window title can't break it.- let start = raw.find('[')?;- let end = raw.rfind(']')?;- let json = &raw[start..=end];- let arr: Vec<serde_json::Value> = serde_json::from_str(json).ok()?;+ let arr: Vec<serde_json::Value> = serde_json::from_str(raw).ok()?;for w in &arr {if w.get("pid").and_then(|p| p.as_u64()) == Some(pid as u64) {let x = w.get("x").and_then(serde_json::Value::as_i64)? as i32;@@ -458,18 +381,11 @@ fn window_is_focused(window_id: u32) -> bool {let Some(raw) = gdbus_call("GetRects", &[]) else {return false;};- let (Some(start), Some(end)) = (raw.find('['), raw.rfind(']')) else {- return false;- };- serde_json::from_str::<Vec<serde_json::Value>>(&raw[start..=end])- .ok()- .and_then(|windows| {- windows.into_iter().find(|window| {- window.get("id").and_then(serde_json::Value::as_u64) == Some(window_id as u64)- })- })- .and_then(|window| window.get("focused").and_then(serde_json::Value::as_bool))- .unwrap_or(false)+ parse_shell_windows(&raw, None).is_some_and(|windows| {+ windows+ .into_iter()+ .any(|window| window.info.xid == u64::from(window_id) && window.focused)+ })}fn parse_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> {@@ -482,9 +398,7 @@ fn parse_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<WindowInfo>>}fn parse_shell_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<ShellWindow>> {- let start = raw.find('[')?;- let end = raw.rfind(']')?;- let windows: Vec<serde_json::Value> = serde_json::from_str(&raw[start..=end]).ok()?;+ let windows: Vec<serde_json::Value> = serde_json::from_str(raw).ok()?;Some(windows@@ -590,9 +504,17 @@ mod tests {const EXTENSION_METADATA: &str =include_str!("../../../../../wayland-helper/winrects@cua/metadata.json");+ #[test]+ fn rejects_malformed_or_display_formatted_snapshots() {+ for raw in ["not JSON", "{}", "('[{\"pid\":6079,\"x\":14,\"y\":12}]',)"] {+ assert!(parse_shell_windows(raw, None).is_none());+ assert!(parse_window_origin(raw, 6079).is_none());+ }+ }+#[test]fn parses_and_filters_shell_windows() {- let raw = r#"('[{"id":46,"pid":6079,"title":"Sentinel's window","x":66,"y":32,"w":958,"h":736,"focused":true,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#;+ let raw = r#"[{"id":46,"pid":6079,"title":"Sentinel's window","x":66,"y":32,"w":958,"h":736,"focused":true,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]"#;let windows = parse_windows(raw, Some(6079)).expect("valid helper response");assert_eq!(windows.len(), 1);assert_eq!(windows[0].xid, 46);@@ -606,14 +528,14 @@ mod tests {#[test]fn accessibility_origin_matches_the_frame_cropped_screenshot() {- let raw = r#"('[{"id":46,"pid":6079,"title":"Floating GTK","x":14,"y":12,"w":560,"h":736,"buffer_x":0,"buffer_y":0}]',)"#;+ let raw = r#"[{"id":46,"pid":6079,"title":"Floating GTK","x":14,"y":12,"w":560,"h":736,"buffer_x":0,"buffer_y":0}]"#;assert_eq!(parse_window_origin(raw, 6079), Some((14, 12)));}#[test]fn marks_minimized_shell_windows_off_screen() {- let raw = r#"('[{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#;+ let raw = r#"[{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]"#;let windows = parse_windows(raw, None).expect("valid helper response");assert_eq!(windows.len(), 1);assert!(!windows[0].is_on_screen);@@ -634,7 +556,7 @@ mod tests {#[test]fn preserves_exact_focus_from_shell_snapshot() {- let raw = r#"('[{"id":46,"pid":6079,"title":"Target","x":66,"y":32,"w":958,"h":736,"focused":false,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Sentinel","x":0,"y":0,"w":100,"h":100,"focused":true,"minimized":false,"visible":true,"stacking":3}]',)"#;+ let raw = r#"[{"id":46,"pid":6079,"title":"Target","x":66,"y":32,"w":958,"h":736,"focused":false,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Sentinel","x":0,"y":0,"w":100,"h":100,"focused":true,"minimized":false,"visible":true,"stacking":3}]"#;let windows = parse_shell_windows(raw, None).expect("valid helper response");assert_eq!(windows.len(), 2);assert!(!windows[0].focused);diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rsnew file mode 100644index 0000000000..4c1988bff0--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs@@ -0,0 +1,310 @@+//! Persistent session-bus transport for synchronous compositor helpers.+//! A separate runtime also serves callers already inside Tokio without nested block_on.+use std::sync::{mpsc, OnceLock};+use std::time::Duration;+use tokio::sync::Mutex;+use zbus::{Connection, Message};++static CONNECTION: Mutex<Option<Connection>> = Mutex::const_new(None);++fn runtime() -> &'static tokio::runtime::Runtime {+ static RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();+ RUNTIME.get_or_init(|| {+ tokio::runtime::Builder::new_multi_thread()+ .worker_threads(1)+ .enable_all()+ .build()+ .expect("compositor D-Bus runtime")+ })+}++pub(super) fn call(+ destination: &str,+ path: &str,+ method: &str,+ args: &[String],+ timeout: Duration,+) -> Option<String> {+ let destination = destination.to_owned();+ let path = path.to_owned();+ let method = method.to_owned();+ let args = args.to_vec();+ let (tx, rx) = mpsc::sync_channel(1);+ runtime().spawn(async move {+ let result = tokio::time::timeout(timeout, async {+ let connection = {+ let mut shared = CONNECTION.lock().await;+ if shared.is_none() {+ *shared = Some(+ Connection::session()+ .await+ .map_err(|_| Failure::Transport)?,+ );+ }+ shared.as_ref().ok_or(Failure::Transport)?.clone()+ };+ invoke(&connection, &destination, &path, &method, &args).await+ })+ .await+ .unwrap_or(Err(Failure::Transport));+ // A timeout or transport error may mean the session bus went away.+ // Reconnect on the next call; never replay a possibly applied mutation.+ // An error reply (no helper on the bus, a refused capture) keeps the+ // connection: reconnecting would not change the answer.+ if matches!(result, Err(Failure::Transport)) {+ CONNECTION.lock().await.take();+ }+ let _ = tx.send(result.ok());+ });+ rx.recv_timeout(timeout + Duration::from_millis(100))+ .ok()+ .flatten()+}++#[derive(Debug, PartialEq, Eq)]+enum Failure {+ /// The peer answered with an error, or the call or reply had an+ /// unexpected shape. The connection is healthy.+ Reply,+ /// The connection failed or the call timed out.+ Transport,+}++fn classify(error: zbus::Error) -> Failure {+ match error {+ zbus::Error::MethodError(..) | zbus::Error::FDO(_) => Failure::Reply,+ _ => Failure::Transport,+ }+}++// Preserve the small textual reply contract consumed by shell_helper's existing+// parsers. Bodies on the wire are typed; strings are never shell or GVariant code.+async fn invoke(+ connection: &Connection,+ destination: &str,+ path: &str,+ method: &str,+ args: &[String],+) -> Result<String, Failure> {+ let (interface, member) = method.rsplit_once('.').ok_or(Failure::Reply)?;+ let parse_u32 = |value: &String| value.parse::<u32>().map_err(|_| Failure::Reply);+ let parse_i32 = |value: &String| value.parse::<i32>().map_err(|_| Failure::Reply);+ macro_rules! send {+ ($body:expr) => {+ connection+ .call_method(Some(destination), path, Some(interface), member, &$body)+ .await+ .map_err(classify)?+ };+ }+ let reply: Message = match (member, args) {+ (+ "GetNameOwner"+ | "GetConnectionUnixProcessID"+ | "GetConnectionUnixUser"+ | "SetCursorColor"+ | "SetSessionLabel",+ [value],+ ) => send!((value.as_str(),)),+ ("GetRects" | "GetVersion" | "Capture" | "HideCursor", []) => send!(()),+ ("Activate", [id]) => send!((parse_u32(id)?,)),+ ("MoveCursor" | "ClickPulse", [x, y]) => {+ send!((parse_i32(x)?, parse_i32(y)?))+ }+ ("SetCursorState", [action, delivery, target, active]) => send!((+ action.as_str(),+ delivery.as_str(),+ target.as_str(),+ active.parse::<bool>().map_err(|_| Failure::Reply)?+ )),+ _ => return Err(Failure::Reply),+ };+ let body = reply.body();+ let decoded = match member {+ "GetRects" => body.deserialize::<String>().ok(),+ "GetNameOwner" | "Capture" => body+ .deserialize::<String>()+ .ok()+ .map(|value| format!("('{value}',)")),+ "GetVersion" | "GetConnectionUnixProcessID" | "GetConnectionUnixUser" => body+ .deserialize::<u32>()+ .ok()+ .map(|value| format!("(uint32 {value},)")),+ "Activate" => body+ .deserialize::<bool>()+ .ok()+ .map(|value| format!("({value},)")),+ _ => body.deserialize::<()>().ok().map(|()| "()".into()),+ };+ decoded.ok_or(Failure::Reply)+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::io::BufRead;+ use std::process::{Child, Command, Stdio};++ struct PrivateBus(Child);+ impl Drop for PrivateBus {+ fn drop(&mut self) {+ let _ = self.0.kill();+ let _ = self.0.wait();+ }+ }++ // Titles gdbus's GVariant text would escape: an apostrophe, quotes, a+ // backslash, a newline and non-ASCII. A typed reply carries them intact.+ const RECTS: &str = r#"[{"id":3,"pid":41,"title":"Tomorrow's \"plan\" C:\\ 1\n2 飞书","x":0,"y":0,"w":10,"h":10},{"id":7,"pid":42,"title":"Target","x":14,"y":12,"w":800,"h":600}]"#;++ struct Helper;+ #[zbus::interface(name = "org.cua.WinRects")]+ impl Helper {+ fn get_version(&self) -> u32 {+ 8+ }+ fn get_rects(&self) -> String {+ RECTS.into()+ }+ fn capture(&self) -> String {+ "cG5n".into()+ }+ fn activate(&self, id: u32) -> bool {+ id == 7+ }+ fn set_cursor_state(+ &self,+ action: &str,+ delivery: &str,+ target: &str,+ active: bool,+ ) -> zbus::fdo::Result<()> {+ if (action, delivery, target, active) != ("observe", "", "window", false) {+ return Err(zbus::fdo::Error::InvalidArgs("wrong state".into()));+ }+ Ok(())+ }+ }++ #[tokio::test]+ async fn typed_roundtrips_on_one_private_connection() {+ // Use an explicit config: a sandbox (Nix) has no session.conf, and+ // then the daemon exits without printing an address.+ let dir = tempfile::tempdir().unwrap();+ let config = dir.path().join("bus.conf");+ std::fs::write(+ &config,+ format!(+ "<busconfig><type>session</type>\+ <listen>unix:dir={}</listen><auth>EXTERNAL</auth>\+ <policy context=\"default\"><allow send_destination=\"*\" eavesdrop=\"true\"/>\+ <allow eavesdrop=\"true\"/><allow own=\"*\"/></policy></busconfig>",+ dir.path().display()+ ),+ )+ .unwrap();+ let spawned = Command::new("dbus-daemon")+ .arg(format!("--config-file={}", config.display()))+ .args(["--nofork", "--print-address=1"])+ .stdout(Stdio::piped())+ .spawn();+ let mut bus = match spawned {+ Ok(child) => PrivateBus(child),+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {+ eprintln!("skipping: dbus-daemon is not installed");+ return;+ }+ Err(error) => panic!("private test bus: {error}"),+ };+ let mut address = String::new();+ std::io::BufReader::new(bus.0.stdout.take().unwrap())+ .read_line(&mut address)+ .unwrap();+ assert!(!address.trim().is_empty(), "dbus-daemon printed no address");+ let service = zbus::connection::Builder::address(address.trim())+ .unwrap()+ .name("org.cua.WinRects")+ .unwrap()+ .serve_at("/org/cua/WinRects", Helper)+ .unwrap()+ .build()+ .await+ .unwrap();+ let client = zbus::connection::Builder::address(address.trim())+ .unwrap()+ .build()+ .await+ .unwrap();+ let call = |method: &'static str, args: Vec<String>| {+ let client = client.clone();+ async move {+ invoke(+ &client,+ "org.cua.WinRects",+ "/org/cua/WinRects",+ &format!("org.cua.WinRects.{method}"),+ &args,+ )+ .await+ }+ };+ // One connection serves repeated calls.+ for _ in 0..3 {+ assert_eq!(+ call("GetVersion", vec![]).await.as_deref(),+ Ok("(uint32 8,)")+ );+ // GetRects is the raw JSON, so punctuation in one title no longer+ // hides every window from the parsers.+ let rects = call("GetRects", vec![]).await.unwrap();+ assert_eq!(rects, RECTS);+ let windows = super::super::parse_windows(&rects, None).unwrap();+ assert_eq!(windows.len(), 2);+ assert_eq!(windows[0].title, "Tomorrow's \"plan\" C:\\ 1\n2 飞书");+ assert_eq!(+ super::super::parse_window_origin(&rects, 42),+ Some((14, 12))+ );+ let raw = call("Capture", vec![]).await.unwrap();+ assert_eq!(raw, "('cG5n',)");+ assert_eq!(super::super::decode_capture(&raw), Some(b"png".to_vec()));+ }+ assert_eq!(+ call("Activate", vec!["7".into()]).await.as_deref(),+ Ok("(true,)")+ );+ assert_eq!(+ call("Activate", vec!["8".into()]).await.as_deref(),+ Ok("(false,)")+ );+ assert_eq!(+ call(+ "SetCursorState",+ vec!["observe".into(), "".into(), "window".into(), "false".into()]+ )+ .await+ .as_deref(),+ Ok("()")+ );+ // Error replies and malformed arguments leave the connection healthy:+ // they are not a reason to reconnect.+ assert_eq!(+ call(+ "SetCursorState",+ vec!["click".into(), "".into(), "window".into(), "true".into()]+ )+ .await,+ Err(Failure::Reply)+ );+ assert_eq!(+ call("Activate", vec!["-1".into()]).await,+ Err(Failure::Reply)+ );+ assert_eq!(call("NoSuchMethod", vec![]).await, Err(Failure::Reply));+ assert_eq!(+ call("GetVersion", vec![]).await.as_deref(),+ Ok("(uint32 8,)")+ );+ drop(service);+ }+}diff --git a/nix/cua-driver/tests/rust-unit.nix b/nix/cua-driver/tests/rust-unit.nixindex 78885da8ef..01b911fa71 100644--- a/nix/cua-driver/tests/rust-unit.nix+++ b/nix/cua-driver/tests/rust-unit.nix@@ -45,6 +45,8 @@ pkgs.rustPlatform.buildRustPackage {clangpython3];+ # Transport tests launch an isolated D-Bus session instead of using the host bus.+ nativeCheckInputs = [ pkgs.dbus ];CUA_TEST_PYTHON = "${pkgs.python3}/bin/python3";CUA_TEST_PYTHON_DYNAMIC_LINKER = pkgs.stdenv.cc.bintools.dynamicLinker;CUA_TEST_GLIBC_STATIC_LIB = "${pkgs.glibc.static}/lib";