Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

perf(cua-driver): reuse one session-bus connection for the GNOME helper (#4879)

Francesco Bonacci48cc49146349
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
index 111b4ed3a5..0f055c7d98 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
@@ -18,12 +18,11 @@
//!
//! Everything here is **best-effort**: if the extension isn't installed/enabled
//! the calls return `None` / no-op and callers keep the prior behaviour (no
-//! screen coords, no Wayland cursor). Uses a short-lived `gdbus` subprocess so
-//! there's no zbus blocking-feature or async-context coupling — the calls are
-//! infrequent (once per `get_window_state`, a few per click).
+//! screen coords, no Wayland cursor). Reuses one session-bus connection on a
+//! dedicated runtime; preview frames do not spawn subprocesses or poll for exit.
use std::os::unix::fs::{MetadataExt, PermissionsExt};
-use std::process::Command;
+mod transport;
use std::time::Duration;
use crate::x11::WindowInfo;
@@ -85,30 +84,7 @@ fn gdbus_call_to(
args: &[String],
timeout: Duration,
) -> Option<String> {
- let mut cmd = Command::new("gdbus");
- cmd.arg("call")
- .arg("--session")
- .arg("--dest")
- .arg(destination)
- .arg("--object-path")
- .arg(object_path)
- .arg("--method")
- .arg(method);
- for a in args {
- cmd.arg(a);
- }
- // gdbus is local IPC; cap it so a wedged shell can't stall the caller.
- let child = cmd
- .stdin(std::process::Stdio::null())
- .stdout(std::process::Stdio::piped())
- .stderr(std::process::Stdio::null())
- .spawn()
- .ok()?;
- let out = wait_timeout(child, timeout)?;
- if !out.status.success() {
- return None;
- }
- Some(String::from_utf8_lossy(&out.stdout).into_owned())
+ transport::call(destination, object_path, method, args, timeout)
}
/// Resolve the helper's immutable unique bus name and prove that it is hosted
@@ -247,53 +223,6 @@ fn decode_capture(raw: &str) -> Option<Vec<u8>> {
B64.decode(&raw[start..end]).ok()
}
-/// `Child::wait` with a deadline (no extra crates). Kills + reaps on timeout.
-fn wait_timeout(mut child: std::process::Child, dur: Duration) -> Option<std::process::Output> {
- use std::io::Read;
-
- // Drain stdout while the child is running. Capture() returns a base64 PNG
- // that readily exceeds a pipe's ~64 KiB capacity; waiting for exit before
- // reading deadlocks the child on a full pipe and turns a healthy Shell
- // response into a false timeout.
- let stdout = child.stdout.take()?;
- let reader = std::thread::spawn(move || {
- let mut stdout = stdout;
- let mut bytes = Vec::new();
- stdout.read_to_end(&mut bytes).ok()?;
- Some(bytes)
- });
- let deadline = std::time::Instant::now() + dur;
- let status = loop {
- match child.try_wait() {
- Ok(Some(status)) => break status,
- Ok(None) => {
- if std::time::Instant::now() >= deadline {
- let _ = child.kill();
- let status = child.wait().ok()?;
- let _ = reader.join();
- if !status.success() {
- return None;
- }
- return None;
- }
- std::thread::sleep(Duration::from_millis(15));
- }
- Err(_) => {
- let _ = child.kill();
- let _ = child.wait();
- let _ = reader.join();
- return None;
- }
- }
- };
- let stdout = reader.join().ok().flatten()?;
- Some(std::process::Output {
- status,
- stdout,
- stderr: Vec::new(),
- })
-}
-
/// Screen origin of the compositor frame backing `pid`.
///
/// `screenshot_window_dispatch` crops the Shell stage to this same frame
@@ -308,13 +237,7 @@ pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> {
}
fn parse_window_origin(raw: &str, pid: u32) -> Option<(i32, i32)> {
- // gdbus prints a GVariant tuple like `('[{"pid":..,"x":..}]',)`. Pull the
- // JSON array out robustly (first '[' .. last ']') rather than parsing the
- // GVariant wrapper, so an apostrophe in a window title can't break it.
- let start = raw.find('[')?;
- let end = raw.rfind(']')?;
- let json = &raw[start..=end];
- let arr: Vec<serde_json::Value> = serde_json::from_str(json).ok()?;
+ let arr: Vec<serde_json::Value> = serde_json::from_str(raw).ok()?;
for w in &arr {
if w.get("pid").and_then(|p| p.as_u64()) == Some(pid as u64) {
let x = w.get("x").and_then(serde_json::Value::as_i64)? as i32;
@@ -458,18 +381,11 @@ fn window_is_focused(window_id: u32) -> bool {
let Some(raw) = gdbus_call("GetRects", &[]) else {
return false;
};
- let (Some(start), Some(end)) = (raw.find('['), raw.rfind(']')) else {
- return false;
- };
- serde_json::from_str::<Vec<serde_json::Value>>(&raw[start..=end])
- .ok()
- .and_then(|windows| {
- windows.into_iter().find(|window| {
- window.get("id").and_then(serde_json::Value::as_u64) == Some(window_id as u64)
- })
- })
- .and_then(|window| window.get("focused").and_then(serde_json::Value::as_bool))
- .unwrap_or(false)
+ parse_shell_windows(&raw, None).is_some_and(|windows| {
+ windows
+ .into_iter()
+ .any(|window| window.info.xid == u64::from(window_id) && window.focused)
+ })
}
fn parse_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> {
@@ -482,9 +398,7 @@ fn parse_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<WindowInfo>>
}
fn parse_shell_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<ShellWindow>> {
- let start = raw.find('[')?;
- let end = raw.rfind(']')?;
- let windows: Vec<serde_json::Value> = serde_json::from_str(&raw[start..=end]).ok()?;
+ let windows: Vec<serde_json::Value> = serde_json::from_str(raw).ok()?;
Some(
windows
@@ -590,9 +504,17 @@ mod tests {
const EXTENSION_METADATA: &str =
include_str!("../../../../../wayland-helper/winrects@cua/metadata.json");
+ #[test]
+ fn rejects_malformed_or_display_formatted_snapshots() {
+ for raw in ["not JSON", "{}", "('[{\"pid\":6079,\"x\":14,\"y\":12}]',)"] {
+ assert!(parse_shell_windows(raw, None).is_none());
+ assert!(parse_window_origin(raw, 6079).is_none());
+ }
+ }
+
#[test]
fn parses_and_filters_shell_windows() {
- let raw = r#"('[{"id":46,"pid":6079,"title":"Sentinel's window","x":66,"y":32,"w":958,"h":736,"focused":true,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#;
+ let raw = r#"[{"id":46,"pid":6079,"title":"Sentinel's window","x":66,"y":32,"w":958,"h":736,"focused":true,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]"#;
let windows = parse_windows(raw, Some(6079)).expect("valid helper response");
assert_eq!(windows.len(), 1);
assert_eq!(windows[0].xid, 46);
@@ -606,14 +528,14 @@ mod tests {
#[test]
fn accessibility_origin_matches_the_frame_cropped_screenshot() {
- let raw = r#"('[{"id":46,"pid":6079,"title":"Floating GTK","x":14,"y":12,"w":560,"h":736,"buffer_x":0,"buffer_y":0}]',)"#;
+ let raw = r#"[{"id":46,"pid":6079,"title":"Floating GTK","x":14,"y":12,"w":560,"h":736,"buffer_x":0,"buffer_y":0}]"#;
assert_eq!(parse_window_origin(raw, 6079), Some((14, 12)));
}
#[test]
fn marks_minimized_shell_windows_off_screen() {
- let raw = r#"('[{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#;
+ let raw = r#"[{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]"#;
let windows = parse_windows(raw, None).expect("valid helper response");
assert_eq!(windows.len(), 1);
assert!(!windows[0].is_on_screen);
@@ -634,7 +556,7 @@ mod tests {
#[test]
fn preserves_exact_focus_from_shell_snapshot() {
- let raw = r#"('[{"id":46,"pid":6079,"title":"Target","x":66,"y":32,"w":958,"h":736,"focused":false,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Sentinel","x":0,"y":0,"w":100,"h":100,"focused":true,"minimized":false,"visible":true,"stacking":3}]',)"#;
+ let raw = r#"[{"id":46,"pid":6079,"title":"Target","x":66,"y":32,"w":958,"h":736,"focused":false,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Sentinel","x":0,"y":0,"w":100,"h":100,"focused":true,"minimized":false,"visible":true,"stacking":3}]"#;
let windows = parse_shell_windows(raw, None).expect("valid helper response");
assert_eq!(windows.len(), 2);
assert!(!windows[0].focused);
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs
new file mode 100644
index 0000000000..4c1988bff0
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs
@@ -0,0 +1,310 @@
+//! Persistent session-bus transport for synchronous compositor helpers.
+//! A separate runtime also serves callers already inside Tokio without nested block_on.
+use std::sync::{mpsc, OnceLock};
+use std::time::Duration;
+use tokio::sync::Mutex;
+use zbus::{Connection, Message};
+
+static CONNECTION: Mutex<Option<Connection>> = Mutex::const_new(None);
+
+fn runtime() -> &'static tokio::runtime::Runtime {
+ static RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
+ RUNTIME.get_or_init(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(1)
+ .enable_all()
+ .build()
+ .expect("compositor D-Bus runtime")
+ })
+}
+
+pub(super) fn call(
+ destination: &str,
+ path: &str,
+ method: &str,
+ args: &[String],
+ timeout: Duration,
+) -> Option<String> {
+ let destination = destination.to_owned();
+ let path = path.to_owned();
+ let method = method.to_owned();
+ let args = args.to_vec();
+ let (tx, rx) = mpsc::sync_channel(1);
+ runtime().spawn(async move {
+ let result = tokio::time::timeout(timeout, async {
+ let connection = {
+ let mut shared = CONNECTION.lock().await;
+ if shared.is_none() {
+ *shared = Some(
+ Connection::session()
+ .await
+ .map_err(|_| Failure::Transport)?,
+ );
+ }
+ shared.as_ref().ok_or(Failure::Transport)?.clone()
+ };
+ invoke(&connection, &destination, &path, &method, &args).await
+ })
+ .await
+ .unwrap_or(Err(Failure::Transport));
+ // A timeout or transport error may mean the session bus went away.
+ // Reconnect on the next call; never replay a possibly applied mutation.
+ // An error reply (no helper on the bus, a refused capture) keeps the
+ // connection: reconnecting would not change the answer.
+ if matches!(result, Err(Failure::Transport)) {
+ CONNECTION.lock().await.take();
+ }
+ let _ = tx.send(result.ok());
+ });
+ rx.recv_timeout(timeout + Duration::from_millis(100))
+ .ok()
+ .flatten()
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum Failure {
+ /// The peer answered with an error, or the call or reply had an
+ /// unexpected shape. The connection is healthy.
+ Reply,
+ /// The connection failed or the call timed out.
+ Transport,
+}
+
+fn classify(error: zbus::Error) -> Failure {
+ match error {
+ zbus::Error::MethodError(..) | zbus::Error::FDO(_) => Failure::Reply,
+ _ => Failure::Transport,
+ }
+}
+
+// Preserve the small textual reply contract consumed by shell_helper's existing
+// parsers. Bodies on the wire are typed; strings are never shell or GVariant code.
+async fn invoke(
+ connection: &Connection,
+ destination: &str,
+ path: &str,
+ method: &str,
+ args: &[String],
+) -> Result<String, Failure> {
+ let (interface, member) = method.rsplit_once('.').ok_or(Failure::Reply)?;
+ let parse_u32 = |value: &String| value.parse::<u32>().map_err(|_| Failure::Reply);
+ let parse_i32 = |value: &String| value.parse::<i32>().map_err(|_| Failure::Reply);
+ macro_rules! send {
+ ($body:expr) => {
+ connection
+ .call_method(Some(destination), path, Some(interface), member, &$body)
+ .await
+ .map_err(classify)?
+ };
+ }
+ let reply: Message = match (member, args) {
+ (
+ "GetNameOwner"
+ | "GetConnectionUnixProcessID"
+ | "GetConnectionUnixUser"
+ | "SetCursorColor"
+ | "SetSessionLabel",
+ [value],
+ ) => send!((value.as_str(),)),
+ ("GetRects" | "GetVersion" | "Capture" | "HideCursor", []) => send!(()),
+ ("Activate", [id]) => send!((parse_u32(id)?,)),
+ ("MoveCursor" | "ClickPulse", [x, y]) => {
+ send!((parse_i32(x)?, parse_i32(y)?))
+ }
+ ("SetCursorState", [action, delivery, target, active]) => send!((
+ action.as_str(),
+ delivery.as_str(),
+ target.as_str(),
+ active.parse::<bool>().map_err(|_| Failure::Reply)?
+ )),
+ _ => return Err(Failure::Reply),
+ };
+ let body = reply.body();
+ let decoded = match member {
+ "GetRects" => body.deserialize::<String>().ok(),
+ "GetNameOwner" | "Capture" => body
+ .deserialize::<String>()
+ .ok()
+ .map(|value| format!("('{value}',)")),
+ "GetVersion" | "GetConnectionUnixProcessID" | "GetConnectionUnixUser" => body
+ .deserialize::<u32>()
+ .ok()
+ .map(|value| format!("(uint32 {value},)")),
+ "Activate" => body
+ .deserialize::<bool>()
+ .ok()
+ .map(|value| format!("({value},)")),
+ _ => body.deserialize::<()>().ok().map(|()| "()".into()),
+ };
+ decoded.ok_or(Failure::Reply)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::io::BufRead;
+ use std::process::{Child, Command, Stdio};
+
+ struct PrivateBus(Child);
+ impl Drop for PrivateBus {
+ fn drop(&mut self) {
+ let _ = self.0.kill();
+ let _ = self.0.wait();
+ }
+ }
+
+ // Titles gdbus's GVariant text would escape: an apostrophe, quotes, a
+ // backslash, a newline and non-ASCII. A typed reply carries them intact.
+ const RECTS: &str = r#"[{"id":3,"pid":41,"title":"Tomorrow's \"plan\" C:\\ 1\n2 飞书","x":0,"y":0,"w":10,"h":10},{"id":7,"pid":42,"title":"Target","x":14,"y":12,"w":800,"h":600}]"#;
+
+ struct Helper;
+ #[zbus::interface(name = "org.cua.WinRects")]
+ impl Helper {
+ fn get_version(&self) -> u32 {
+ 8
+ }
+ fn get_rects(&self) -> String {
+ RECTS.into()
+ }
+ fn capture(&self) -> String {
+ "cG5n".into()
+ }
+ fn activate(&self, id: u32) -> bool {
+ id == 7
+ }
+ fn set_cursor_state(
+ &self,
+ action: &str,
+ delivery: &str,
+ target: &str,
+ active: bool,
+ ) -> zbus::fdo::Result<()> {
+ if (action, delivery, target, active) != ("observe", "", "window", false) {
+ return Err(zbus::fdo::Error::InvalidArgs("wrong state".into()));
+ }
+ Ok(())
+ }
+ }
+
+ #[tokio::test]
+ async fn typed_roundtrips_on_one_private_connection() {
+ // Use an explicit config: a sandbox (Nix) has no session.conf, and
+ // then the daemon exits without printing an address.
+ let dir = tempfile::tempdir().unwrap();
+ let config = dir.path().join("bus.conf");
+ std::fs::write(
+ &config,
+ format!(
+ "<busconfig><type>session</type>\
+ <listen>unix:dir={}</listen><auth>EXTERNAL</auth>\
+ <policy context=\"default\"><allow send_destination=\"*\" eavesdrop=\"true\"/>\
+ <allow eavesdrop=\"true\"/><allow own=\"*\"/></policy></busconfig>",
+ dir.path().display()
+ ),
+ )
+ .unwrap();
+ let spawned = Command::new("dbus-daemon")
+ .arg(format!("--config-file={}", config.display()))
+ .args(["--nofork", "--print-address=1"])
+ .stdout(Stdio::piped())
+ .spawn();
+ let mut bus = match spawned {
+ Ok(child) => PrivateBus(child),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ eprintln!("skipping: dbus-daemon is not installed");
+ return;
+ }
+ Err(error) => panic!("private test bus: {error}"),
+ };
+ let mut address = String::new();
+ std::io::BufReader::new(bus.0.stdout.take().unwrap())
+ .read_line(&mut address)
+ .unwrap();
+ assert!(!address.trim().is_empty(), "dbus-daemon printed no address");
+ let service = zbus::connection::Builder::address(address.trim())
+ .unwrap()
+ .name("org.cua.WinRects")
+ .unwrap()
+ .serve_at("/org/cua/WinRects", Helper)
+ .unwrap()
+ .build()
+ .await
+ .unwrap();
+ let client = zbus::connection::Builder::address(address.trim())
+ .unwrap()
+ .build()
+ .await
+ .unwrap();
+ let call = |method: &'static str, args: Vec<String>| {
+ let client = client.clone();
+ async move {
+ invoke(
+ &client,
+ "org.cua.WinRects",
+ "/org/cua/WinRects",
+ &format!("org.cua.WinRects.{method}"),
+ &args,
+ )
+ .await
+ }
+ };
+ // One connection serves repeated calls.
+ for _ in 0..3 {
+ assert_eq!(
+ call("GetVersion", vec![]).await.as_deref(),
+ Ok("(uint32 8,)")
+ );
+ // GetRects is the raw JSON, so punctuation in one title no longer
+ // hides every window from the parsers.
+ let rects = call("GetRects", vec![]).await.unwrap();
+ assert_eq!(rects, RECTS);
+ let windows = super::super::parse_windows(&rects, None).unwrap();
+ assert_eq!(windows.len(), 2);
+ assert_eq!(windows[0].title, "Tomorrow's \"plan\" C:\\ 1\n2 飞书");
+ assert_eq!(
+ super::super::parse_window_origin(&rects, 42),
+ Some((14, 12))
+ );
+ let raw = call("Capture", vec![]).await.unwrap();
+ assert_eq!(raw, "('cG5n',)");
+ assert_eq!(super::super::decode_capture(&raw), Some(b"png".to_vec()));
+ }
+ assert_eq!(
+ call("Activate", vec!["7".into()]).await.as_deref(),
+ Ok("(true,)")
+ );
+ assert_eq!(
+ call("Activate", vec!["8".into()]).await.as_deref(),
+ Ok("(false,)")
+ );
+ assert_eq!(
+ call(
+ "SetCursorState",
+ vec!["observe".into(), "".into(), "window".into(), "false".into()]
+ )
+ .await
+ .as_deref(),
+ Ok("()")
+ );
+ // Error replies and malformed arguments leave the connection healthy:
+ // they are not a reason to reconnect.
+ assert_eq!(
+ call(
+ "SetCursorState",
+ vec!["click".into(), "".into(), "window".into(), "true".into()]
+ )
+ .await,
+ Err(Failure::Reply)
+ );
+ assert_eq!(
+ call("Activate", vec!["-1".into()]).await,
+ Err(Failure::Reply)
+ );
+ assert_eq!(call("NoSuchMethod", vec![]).await, Err(Failure::Reply));
+ assert_eq!(
+ call("GetVersion", vec![]).await.as_deref(),
+ Ok("(uint32 8,)")
+ );
+ drop(service);
+ }
+}
diff --git a/nix/cua-driver/tests/rust-unit.nix b/nix/cua-driver/tests/rust-unit.nix
index 78885da8ef..01b911fa71 100644
--- a/nix/cua-driver/tests/rust-unit.nix
+++ b/nix/cua-driver/tests/rust-unit.nix
@@ -45,6 +45,8 @@ pkgs.rustPlatform.buildRustPackage {
clang
python3
];
+ # Transport tests launch an isolated D-Bus session instead of using the host bus.
+ nativeCheckInputs = [ pkgs.dbus ];
CUA_TEST_PYTHON = "${pkgs.python3}/bin/python3";
CUA_TEST_PYTHON_DYNAMIC_LINKER = pkgs.stdenv.cc.bintools.dynamicLinker;
CUA_TEST_GLIBC_STATIC_LIB = "${pkgs.glibc.static}/lib";