Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): harden Linux AT-SPI and input delivery

Francesco Bonacciea599fdfd76f
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
index 8fe7efc889..f0cd44fb20 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
@@ -87,11 +87,11 @@ fn host_specs() -> Vec<HostSpec> {
],
title: "CuaTestHarness Electron",
});
- // WebKitGTK's AT-SPI tree is not exposed reliably by the headless
- // Xvfb lane even though the Tauri window itself launches. Keep that
- // lane available for a real Linux desktop with
- // CUA_INCLUDE_TAURI_LINUX=1, while the default browser-equivalent
- // matrix remains deterministic.
+ // WebKitGTK's AT-SPI tree is exposed through a separate WebProcess;
+ // the Rust walker handles that reference shape, but headless Xvfb
+ // still does not provide a reliable input-delivery contract for the
+ // Tauri renderer. Keep this strict lane opt-in until that renderer
+ // path is fixed, while the Electron matrix remains deterministic.
if std::env::var_os("CUA_INCLUDE_TAURI_LINUX").is_some() {
hosts.push(HostSpec {
name: "tauri",
@@ -413,6 +413,18 @@ fn shared_web_keyboard_routes_are_state_verified() {
continue;
}
}
+ #[cfg(target_os = "linux")]
+ {
+ if hotkey.is_error()
+ && hotkey.structured()["code"].as_str() == Some("background_unavailable")
+ {
+ println!(
+ "✅ {} keyboard AX route: background hotkey refused honestly",
+ fixture.name
+ );
+ continue;
+ }
+ }
assert!(
!hotkey.is_error(),
"{}: Ctrl+Shift+7 failed: {}",
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs
index 87f2f51776..9be13a143c 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs
@@ -44,7 +44,11 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
}
driver
.reaper()
- .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))
+ .spawn(
+ Command::new(&exe)
+ .stdout(Stdio::null())
+ .stderr(Stdio::null()),
+ )
.ok()?;
// GTK app cold-start + window map + AT-SPI registration.
std::thread::sleep(Duration::from_millis(1500));
@@ -58,7 +62,11 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
let r = driver.call("list_windows", serde_json::json!({}));
if let Some(wins) = r.structured()["windows"].as_array() {
for w in wins {
- if w["title"].as_str().unwrap_or("").contains("CuaTestHarness GTK3") {
+ if w["title"]
+ .as_str()
+ .unwrap_or("")
+ .contains("CuaTestHarness GTK3")
+ {
let pid = w["pid"].as_u64().unwrap_or(0) as u32;
let wid = w["window_id"].as_u64().unwrap_or(0);
if pid != 0 && wid != 0 {
@@ -91,8 +99,12 @@ fn ax_empty(text: &str) -> bool {
#[test]
#[ignore]
fn harness_gtk3_smoke() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some((pid, wid)) = launch(&mut driver) else { return };
+ let Some(mut driver) = McpDriver::spawn() else {
+ return;
+ };
+ let Some((pid, wid)) = launch(&mut driver) else {
+ return;
+ };
println!("gtk3 harness pid={pid} wid={wid}");
let text = snapshot(&mut driver, pid, wid);
@@ -103,12 +115,31 @@ fn harness_gtk3_smoke() {
println!("snapshot:\n{text}");
// Actionable controls expose their aid as the AT-SPI accessible name.
- for name in ["btn-increment", "btn-reset", "txt-input", "btn-open-popover", "btn-exit"] {
- assert!(text.contains(name), "missing control named {name:?} in GTK3 AT-SPI tree");
+ for name in [
+ "btn-increment",
+ "btn-reset",
+ "txt-input",
+ "btn-open-popover",
+ "btn-exit",
+ ] {
+ assert!(
+ text.contains(name),
+ "missing control named {name:?} in GTK3 AT-SPI tree"
+ );
}
// Marker label + counter label carry their text as the accessible name.
- assert!(text.contains("HARNESS_TEXT_MARKER_v1"), "text_body marker not in tree");
- assert!(text.contains("counter=0"), "initial counter label not in tree");
+ assert!(
+ text.contains("HARNESS_TEXT_MARKER_v1"),
+ "text_body marker not in tree"
+ );
+ assert!(
+ text.contains("counter=0"),
+ "initial counter label not in tree"
+ );
+ assert!(
+ text.contains("popover_open=False"),
+ "initial popover state not in tree"
+ );
println!("✅ harness_gtk3_smoke: all scenarios present in AT-SPI tree");
}
@@ -116,8 +147,12 @@ fn harness_gtk3_smoke() {
#[test]
#[ignore]
fn harness_gtk3_counter_click() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some((pid, wid)) = launch(&mut driver) else { return };
+ let Some(mut driver) = McpDriver::spawn() else {
+ return;
+ };
+ let Some((pid, wid)) = launch(&mut driver) else {
+ return;
+ };
let pre = snapshot(&mut driver, pid, wid);
if ax_empty(&pre) {
@@ -142,7 +177,10 @@ fn harness_gtk3_counter_click() {
assert!(
post.contains("counter=1"),
"counter did not advance after clicking btn-increment. counter lines: {}",
- post.lines().filter(|l| l.contains("counter=")).collect::<Vec<_>>().join(" / ")
+ post.lines()
+ .filter(|l| l.contains("counter="))
+ .collect::<Vec<_>>()
+ .join(" / ")
);
println!("✅ harness_gtk3_counter_click: counter advanced via AT-SPI click");
}
@@ -150,15 +188,22 @@ fn harness_gtk3_counter_click() {
#[test]
#[ignore]
fn harness_gtk3_popover() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some((pid, wid)) = launch(&mut driver) else { return };
+ let Some(mut driver) = McpDriver::spawn() else {
+ return;
+ };
+ let Some((pid, wid)) = launch(&mut driver) else {
+ return;
+ };
let pre = snapshot(&mut driver, pid, wid);
if ax_empty(&pre) {
eprintln!("AT-SPI empty — skipping");
return;
}
- assert!(!pre.contains("POPOVER_MARKER_v1"), "popover body present BEFORE open");
+ assert!(
+ pre.contains("popover_open=False"),
+ "popover state not initially closed"
+ );
let idx = match ax::element_index_containing(&pre, "btn-open-popover") {
Some(i) => i,
@@ -175,14 +220,25 @@ fn harness_gtk3_popover() {
// GtkPopover may surface as a separate top-level; check the main window
// first, then any new window of this pid.
- let mut found = snapshot(&mut driver, pid, wid).contains("POPOVER_MARKER_v1");
+ let post = snapshot(&mut driver, pid, wid);
+ assert!(
+ post.contains("popover_open=True"),
+ "popover state did not flip open after click. Popover lines: {}",
+ post.lines()
+ .filter(|l| l.contains("popover_open="))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
+ let mut found = post.contains("POPOVER_MARKER_v1");
if !found {
let r = driver.call("list_windows", serde_json::json!({}));
if let Some(wins) = r.structured()["windows"].as_array() {
for w in wins {
if w["pid"].as_u64() == Some(pid as u64) {
if let Some(other) = w["window_id"].as_u64() {
- if other != wid && snapshot(&mut driver, pid, other).contains("POPOVER_MARKER_v1") {
+ if other != wid
+ && snapshot(&mut driver, pid, other).contains("POPOVER_MARKER_v1")
+ {
found = true;
break;
}
@@ -191,6 +247,9 @@ fn harness_gtk3_popover() {
}
}
}
- assert!(found, "popover body marker POPOVER_MARKER_v1 not found after open");
+ assert!(
+ found,
+ "popover body marker POPOVER_MARKER_v1 not found after open"
+ );
println!("✅ harness_gtk3_popover: popover body enumerated after open");
}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_capture_mode_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_capture_mode_test.rs
index 751d1d0ecc..feec766cbc 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_capture_mode_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_capture_mode_test.rs
@@ -33,6 +33,16 @@ use cua_driver_testkit::{harness_app, Driver, McpDriver, ToolResponse};
/// (WPF AutomationId / AppKit AX identifier / GTK3 AT-SPI accessible name).
const TREE_MARKER: &str = "btn-increment";
+#[cfg(target_os = "macos")]
+fn spawn_driver() -> Option<McpDriver> {
+ McpDriver::spawn_macos_daemon_proxy()
+}
+
+#[cfg(not(target_os = "macos"))]
+fn spawn_driver() -> Option<McpDriver> {
+ McpDriver::spawn()
+}
+
/// Does the response carry a screenshot? Checks both the MCP `image` content
/// entry and the canonical `screenshot_png_b64` structured field (the CLI path
/// surfaces only the latter), so the oracle is transport-agnostic.
@@ -92,7 +102,9 @@ fn snapshot_settled_default(driver: &mut McpDriver, pid: u32, wid: u64) -> ToolR
let args = serde_json::json!({ "pid": pid as i64, "window_id": wid });
let mut resp = driver.call("get_window_state", args.clone());
for _ in 0..16 {
- if tree_has_marker(&resp) { break; }
+ if tree_has_marker(&resp) {
+ break;
+ }
std::thread::sleep(Duration::from_millis(500));
resp = driver.call("get_window_state", args.clone());
}
@@ -102,10 +114,13 @@ fn snapshot_settled_default(driver: &mut McpDriver, pid: u32, wid: u64) -> ToolR
/// Snapshot with `include_screenshot:false` (the perf opt-out), retrying until
/// the tree settles.
fn snapshot_settled_tree_only(driver: &mut McpDriver, pid: u32, wid: u64) -> ToolResponse {
- let args = serde_json::json!({ "pid": pid as i64, "window_id": wid, "include_screenshot": false });
+ let args =
+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "include_screenshot": false });
let mut resp = driver.call("get_window_state", args.clone());
for _ in 0..16 {
- if tree_has_marker(&resp) { break; }
+ if tree_has_marker(&resp) {
+ break;
+ }
std::thread::sleep(Duration::from_millis(500));
resp = driver.call("get_window_state", args.clone());
}
@@ -150,7 +165,11 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
/// Spawn `exe` under the driver's reaper and resolve the window of the *new*
/// instance (its pid was not among the harness windows before the spawn).
-fn launch_new_instance(driver: &mut McpDriver, exe: &std::path::Path, title: &str) -> Option<(u32, u64)> {
+fn launch_new_instance(
+ driver: &mut McpDriver,
+ exe: &std::path::Path,
+ title: &str,
+) -> Option<(u32, u64)> {
if !exe.exists() {
eprintln!("[capture_mode] harness not built ({exe:?}) — skipping");
return None;
@@ -158,7 +177,11 @@ fn launch_new_instance(driver: &mut McpDriver, exe: &std::path::Path, title: &st
let before = harness_pids(driver, title);
driver
.reaper()
- .spawn(Command::new(exe).stdout(Stdio::null()).stderr(Stdio::null()))
+ .spawn(
+ Command::new(exe)
+ .stdout(Stdio::null())
+ .stderr(Stdio::null()),
+ )
.ok()?;
resolve_new_window(driver, title, &before)
}
@@ -217,11 +240,19 @@ fn resolve_new_window(
#[test]
#[ignore]
fn default_returns_tree_and_screenshot() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some((pid, wid)) = launch(&mut driver) else { return };
+ let Some(mut driver) = spawn_driver() else {
+ return;
+ };
+ let Some((pid, wid)) = launch(&mut driver) else {
+ return;
+ };
let resp = snapshot_settled_default(&mut driver, pid, wid);
- assert!(!resp.is_error(), "get_window_state(default) errored: {}", resp.text());
+ assert!(
+ !resp.is_error(),
+ "get_window_state(default) errored: {}",
+ resp.text()
+ );
let tree = tree_has_marker(&resp);
let img = has_image(&resp);
@@ -244,11 +275,19 @@ fn default_returns_tree_and_screenshot() {
#[test]
#[ignore]
fn include_screenshot_false_returns_tree_only() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some((pid, wid)) = launch(&mut driver) else { return };
+ let Some(mut driver) = spawn_driver() else {
+ return;
+ };
+ let Some((pid, wid)) = launch(&mut driver) else {
+ return;
+ };
let resp = snapshot_settled_tree_only(&mut driver, pid, wid);
- assert!(!resp.is_error(), "get_window_state(include_screenshot:false) errored: {}", resp.text());
+ assert!(
+ !resp.is_error(),
+ "get_window_state(include_screenshot:false) errored: {}",
+ resp.text()
+ );
if !tree_has_marker(&resp) {
eprintln!("[capture_mode] tree-only: no {TREE_MARKER:?} — accessibility grant likely missing; skipping");
@@ -267,11 +306,19 @@ fn include_screenshot_false_returns_tree_only() {
#[test]
#[ignore]
fn deprecated_capture_mode_is_ignored() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some((pid, wid)) = launch(&mut driver) else { return };
+ let Some(mut driver) = spawn_driver() else {
+ return;
+ };
+ let Some((pid, wid)) = launch(&mut driver) else {
+ return;
+ };
let resp = snapshot_settled(&mut driver, pid, wid, "vision");
- assert!(!resp.is_error(), "get_window_state(capture_mode=vision) errored: {}", resp.text());
+ assert!(
+ !resp.is_error(),
+ "get_window_state(capture_mode=vision) errored: {}",
+ resp.text()
+ );
// The legacy "vision" value must NOT suppress the tree anymore.
if has_image(&resp) && tree_has_marker(&resp) {
@@ -284,6 +331,8 @@ fn deprecated_capture_mode_is_ignored() {
if tree_has_marker(&resp) {
println!("✅ capture_mode=vision ignored: tree still present (image grant may be missing)");
} else {
- eprintln!("[capture_mode] vision: no tree marker — accessibility grant likely missing; skipping");
+ eprintln!(
+ "[capture_mode] vision: no tree marker — accessibility grant likely missing; skipping"
+ );
}
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
index 0e46e1a782..6678910d0b 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
@@ -71,8 +71,15 @@ pub fn walk_tree_bounded(
// keep waiting on the degenerate case, and accept it anyway on the
// final attempt rather than discarding a (minimal) valid result.
if !raw_md.is_empty() && (nodes.len() > 1 || attempt == MAX_ATTEMPTS - 1) {
- let md = if let Some(q) = query { filter_tree(&raw_md, q) } else { raw_md };
- return AtspiTreeResult { tree_markdown: md, nodes };
+ let md = if let Some(q) = query {
+ filter_tree(&raw_md, q)
+ } else {
+ raw_md
+ };
+ return AtspiTreeResult {
+ tree_markdown: md,
+ nodes,
+ };
}
}
if attempt < MAX_ATTEMPTS - 1 {
@@ -93,6 +100,15 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
native::perform_action(pid, idx)
}
+/// Give an indexed AT-SPI element keyboard focus without activating its window.
+pub fn focus_element(pid: u32, idx: usize) -> Result<bool> {
+ native::focus_element(pid, idx)
+}
+
+pub fn scroll_element(pid: u32, idx: usize, direction: &str, amount: usize) -> Result<()> {
+ native::scroll_element(pid, idx, direction, amount)
+}
+
/// Enumerate top-level windows from the AT-SPI registry. The window-listing
/// fallback for Wayland compositors without `zwlr_foreign_toplevel_management`
/// (GNOME Mutter / KDE KWin), where native apps have no X11 XID. Returns one
@@ -131,7 +147,8 @@ pub fn perform_action_at_screen_point(
/// Returns Ok if an editable was found and text was set, Err otherwise.
pub fn type_into_editable(pid: u32, text: &str) -> Result<()> {
let safe_text = text.replace('\\', "\\\\").replace('\'', "\\'");
- let script = format!(r#"
+ let script = format!(
+ r#"
import pyatspi, sys
def find_editable(acc, depth=0):
@@ -178,9 +195,15 @@ try:
except Exception as e:
print(f"ERROR: {{e}}", file=sys.stderr)
sys.exit(1)
-"#, pid = pid, safe_text = safe_text);
-
- let out = std::process::Command::new("python3").arg("-c").arg(&script).output()?;
+"#,
+ pid = pid,
+ safe_text = safe_text
+ );
+
+ let out = std::process::Command::new("python3")
+ .arg("-c")
+ .arg(&script)
+ .output()?;
if !out.status.success() {
anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());
}
@@ -229,13 +252,16 @@ pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)>
/// Minimal X11 property-based tree (fallback when AT-SPI is unavailable).
fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
- use x11rb::connection::Connection;
- use x11rb::protocol::xproto::*;
use x11rb::rust_connection::RustConnection;
let (conn, _) = match RustConnection::connect(None) {
Ok(r) => r,
- Err(_) => return AtspiTreeResult { tree_markdown: String::new(), nodes: vec![] },
+ Err(_) => {
+ return AtspiTreeResult {
+ tree_markdown: String::new(),
+ nodes: vec![],
+ }
+ }
};
let window = xid as u32;
@@ -252,15 +278,26 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
let root_node = AtspiNode {
element_index: Some(0),
role: "window".into(),
- name: if title.is_empty() { None } else { Some(title.clone()) },
+ name: if title.is_empty() {
+ None
+ } else {
+ Some(title.clone())
+ },
value: None,
- description: if wm_class.is_empty() { None } else { Some(wm_class.clone()) },
+ description: if wm_class.is_empty() {
+ None
+ } else {
+ Some(wm_class.clone())
+ },
actions: vec!["activate".into()],
element_key: xid,
depth: 0,
parent_element_index: None,
};
- md.push_str(&format!("- [0] window \"{}\" [actions=[activate]]\n", title));
+ md.push_str(&format!(
+ "- [0] window \"{}\" [actions=[activate]]\n",
+ title
+ ));
nodes.push(root_node);
let raw_md = md;
@@ -270,26 +307,51 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
raw_md
};
- AtspiTreeResult { tree_markdown, nodes }
+ AtspiTreeResult {
+ tree_markdown,
+ nodes,
+ }
}
fn get_x11_title(conn: &x11rb::rust_connection::RustConnection, window: u32) -> Option<String> {
use x11rb::protocol::xproto::*;
// Try _NET_WM_NAME first.
- let net_wm_name = conn.intern_atom(false, b"_NET_WM_NAME").ok()?.reply().ok()?.atom;
- let utf8_string = conn.intern_atom(false, b"UTF8_STRING").ok()?.reply().ok()?.atom;
- if let Ok(reply) = conn.get_property(false, window, net_wm_name, utf8_string, 0, 1024).ok()?.reply() {
+ let net_wm_name = conn
+ .intern_atom(false, b"_NET_WM_NAME")
+ .ok()?
+ .reply()
+ .ok()?
+ .atom;
+ let utf8_string = conn
+ .intern_atom(false, b"UTF8_STRING")
+ .ok()?
+ .reply()
+ .ok()?
+ .atom;
+ if let Ok(reply) = conn
+ .get_property(false, window, net_wm_name, utf8_string, 0, 1024)
+ .ok()?
+ .reply()
+ {
if !reply.value.is_empty() {
return Some(String::from_utf8_lossy(&reply.value).into_owned());
}
}
- let reply = conn.get_property(false, window, AtomEnum::WM_NAME, AtomEnum::STRING, 0, 1024).ok()?.reply().ok()?;
+ let reply = conn
+ .get_property(false, window, AtomEnum::WM_NAME, AtomEnum::STRING, 0, 1024)
+ .ok()?
+ .reply()
+ .ok()?;
Some(String::from_utf8_lossy(&reply.value).into_owned())
}
fn get_x11_wm_class(conn: &x11rb::rust_connection::RustConnection, window: u32) -> Option<String> {
use x11rb::protocol::xproto::*;
- let reply = conn.get_property(false, window, AtomEnum::WM_CLASS, AtomEnum::STRING, 0, 512).ok()?.reply().ok()?;
+ let reply = conn
+ .get_property(false, window, AtomEnum::WM_CLASS, AtomEnum::STRING, 0, 512)
+ .ok()?
+ .reply()
+ .ok()?;
let s = String::from_utf8_lossy(&reply.value);
// WM_CLASS is two NUL-separated strings: instance_name\0class_name\0
Some(s.trim_end_matches('\0').replace('\0', "."))
@@ -304,13 +366,22 @@ fn filter_tree(markdown: &str, query: &str) -> String {
for line in &lines {
let depth = line.chars().take_while(|c| *c == ' ').count() / 2;
- while ancestors.len() <= depth { ancestors.push(""); last_emitted.push(None); }
- for d in (depth+1)..ancestors.len() { last_emitted[d] = None; }
+ while ancestors.len() <= depth {
+ ancestors.push("");
+ last_emitted.push(None);
+ }
+ for d in (depth + 1)..ancestors.len() {
+ last_emitted[d] = None;
+ }
ancestors[depth] = line;
if line.to_lowercase().contains(&needle) {
for d in 0..depth {
- if ancestors[d].is_empty() { continue; }
- if last_emitted[d] == Some(ancestors[d]) { continue; }
+ if ancestors[d].is_empty() {
+ continue;
+ }
+ if last_emitted[d] == Some(ancestors[d]) {
+ continue;
+ }
last_emitted[d] = Some(ancestors[d]);
output.push(ancestors[d]);
}
@@ -318,6 +389,10 @@ fn filter_tree(markdown: &str, query: &str) -> String {
output.push(line);
}
}
- if output.is_empty() { return String::new(); }
- let mut r = output.join("\n"); r.push('\n'); r
+ if output.is_empty() {
+ return String::new();
+ }
+ let mut r = output.join("\n");
+ r.push('\n');
+ r
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 3fed71d85f..5f2c6feab5 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -15,7 +15,7 @@ use std::sync::OnceLock;
use std::time::Duration;
use anyhow::{anyhow, Result};
-use atspi::connection::AccessibilityConnection;
+use atspi::connection::{AccessibilityConnection, P2P};
use atspi::proxy::accessible::AccessibleProxy;
use atspi::proxy::proxy_ext::ProxyExt;
use atspi::{CoordType, Interface, State};
@@ -130,27 +130,90 @@ fn is_document_role(role: &str) -> bool {
/// unaffected (they already tolerate `GetAll`), and the sub-interface proxies
/// from `proxies()` already use `CacheProperties::No`.
async fn accessible_for<'a>(
- conn: &'a atspi::zbus::Connection,
- oref: &atspi::ObjectRefOwned,
+ conn: &'a AccessibilityConnection,
+ oref: &RawObjectRef,
) -> Result<AccessibleProxy<'a>> {
- let dest = oref
- .name_as_str()
- .ok_or_else(|| anyhow!("object has no bus name"))?
- .to_owned();
- let path = oref.path_as_str().to_owned();
- AccessibleProxy::builder(conn)
+ // Keep the atspi crate's peer-to-peer path for normal AT-SPI unique names.
+ // Electron/Chromium uses this path for focused-child input delivery. The
+ // raw string path below is only needed for WebKitGTK's well-known
+ // WebProcess references, which cannot be represented as ObjectRef names.
+ if oref.name.starts_with(':') {
+ let name = atspi::zbus::names::UniqueName::try_from(oref.name.clone())
+ .map_err(|e| anyhow!("bad a11y unique name: {e}"))?;
+ let path = atspi::zbus::zvariant::ObjectPath::try_from(oref.path.clone())
+ .map_err(|e| anyhow!("bad a11y path: {e}"))?;
+ let object = atspi::ObjectRef::new_owned(name, path);
+ // `object_as_accessible` chooses a P2P peer when the toolkit exposes
+ // one and falls back to the shared accessibility bus otherwise.
+ return conn
+ .object_as_accessible(&object)
+ .await
+ .map_err(|e| anyhow!("AccessibleProxy build failed: {e}"));
+ }
+ AccessibleProxy::builder(conn.connection())
.cache_properties(atspi::zbus::proxy::CacheProperties::No)
- .destination(dest)
+ .destination(oref.name.clone())
.map_err(|e| anyhow!("bad a11y destination: {e}"))?
- .path(path)
+ .path(oref.path.clone())
.map_err(|e| anyhow!("bad a11y path: {e}"))?
.build()
.await
.map_err(|e| anyhow!("AccessibleProxy build failed: {e}"))
}
+/// AT-SPI's `(so)` object references are documented as unique bus names, but
+/// WebKitGTK uses its well-known WebProcess name for the embedded web tree.
+/// Keep the wire values as strings while walking so zbus does not reject that
+/// validly addressable well-known name before we can call it.
+#[derive(Clone, Debug)]
+struct RawObjectRef {
+ name: String,
+ path: String,
+}
+
+impl RawObjectRef {
+ fn from_atspi(oref: &atspi::ObjectRefOwned) -> Option<Self> {
+ Some(Self {
+ name: oref.name_as_str()?.to_owned(),
+ path: oref.path_as_str().to_owned(),
+ })
+ }
+}
+
+/// Read Accessible.GetChildren without deserializing the bus-name field as a
+/// `UniqueName`. WebKitGTK's embedded WebProcess exposes a well-known name
+/// containing a UUID; D-Bus can address it, but the stricter AT-SPI wrapper
+/// rejects it as an invalid unique name.
+async fn raw_children(
+ conn: &atspi::zbus::Connection,
+ oref: &RawObjectRef,
+) -> Result<Vec<RawObjectRef>> {
+ let proxy = atspi::zbus::Proxy::new(
+ conn,
+ oref.name.as_str(),
+ oref.path.as_str(),
+ "org.a11y.atspi.Accessible",
+ )
+ .await
+ .map_err(|e| anyhow!("Accessible proxy unavailable: {e}"))?;
+ let refs: Vec<(String, atspi::zbus::zvariant::OwnedObjectPath)> = proxy
+ .call("GetChildren", &())
+ .await
+ .map_err(|e| anyhow!("Accessible.GetChildren failed: {e}"))?;
+ Ok(refs
+ .into_iter()
+ .map(|(name, path)| RawObjectRef {
+ name,
+ path: path.to_string(),
+ })
+ .collect())
+}
+
/// Resolve the process id behind an application accessible's D-Bus name.
-async fn pid_of(dbus: &atspi::zbus::fdo::DBusProxy<'_>, oref: &atspi::ObjectRefOwned) -> Option<u32> {
+async fn pid_of(
+ dbus: &atspi::zbus::fdo::DBusProxy<'_>,
+ oref: &atspi::ObjectRefOwned,
+) -> Option<u32> {
let bus = atspi::zbus::names::BusName::try_from(oref.name_as_str()?.to_owned()).ok()?;
dbus.get_connection_unix_process_id(bus).await.ok()
}
@@ -187,20 +250,30 @@ async fn app_for_pid<'a>(
return Ok(None);
}
};
- dlog!("registry root has {} application(s); seeking pid {pid}", apps.len());
+ dlog!(
+ "registry root has {} application(s); seeking pid {pid}",
+ apps.len()
+ );
for child in apps {
// A modal-grabbed app can't answer the pid query; skip it after
// CALL_TIMEOUT rather than blocking the whole walk on it.
let cpid = match call(pid_of(&dbus, &child)).await {
Some(p) => p,
None => {
- dlog!(" pid_of timed out for bus={:?}, skipping", child.name_as_str());
+ dlog!(
+ " pid_of timed out for bus={:?}, skipping",
+ child.name_as_str()
+ );
continue;
}
};
dlog!(" app bus={:?} pid={:?}", child.name_as_str(), cpid);
if cpid == Some(pid) {
- return match call(accessible_for(zconn, &child)).await {
+ let child = match RawObjectRef::from_atspi(&child) {
+ Some(child) => child,
+ None => continue,
+ };
+ return match call(accessible_for(conn, &child)).await {
Some(r) => r.map(Some),
None => {
dlog!(" accessible_for timed out for pid {pid}");
@@ -245,8 +318,13 @@ async fn collect_visited_bounded<'a>(
// push children reversed so siblings pop left-to-right and each subtree
// completes before the next sibling (pre-order). `in_web_doc` is inherited
// from ancestors so editables in page content can be told from chrome.
- let mut stack: Vec<(atspi::ObjectRefOwned, usize, bool)> = match call(app.get_children()).await {
- Some(Ok(children)) => children.into_iter().rev().map(|r| (r, 0usize, false)).collect(),
+ let mut stack: Vec<(RawObjectRef, usize, bool)> = match call(app.get_children()).await {
+ Some(Ok(children)) => children
+ .into_iter()
+ .filter_map(|child| RawObjectRef::from_atspi(&child))
+ .rev()
+ .map(|r| (r, 0usize, false))
+ .collect(),
_ => Vec::new(),
};
@@ -287,9 +365,12 @@ async fn collect_visited_bounded<'a>(
// otherwise the loop never returns to the deadline check at the top and
// the walk stalls past OP_TIMEOUT for callers without an outer guard
// (get_all_element_bounds, insert_text). That was the residual #1936 hang.
- let acc = match call(accessible_for(zconn, &oref)).await {
+ let acc = match call(accessible_for(conn, &oref)).await {
Some(Ok(a)) => a,
- Some(Err(_)) => continue,
+ Some(Err(error)) => {
+ dlog!(" accessible_for failed: {error:#}");
+ continue;
+ }
None => {
consecutive_timeouts += 1;
if consecutive_timeouts >= 3 {
@@ -311,7 +392,10 @@ async fn collect_visited_bounded<'a>(
i
}
// A completed-but-errored call is node-specific; keep walking.
- Some(Err(_)) => continue,
+ Some(Err(error)) => {
+ dlog!(" get_interfaces failed: {error:#}");
+ continue;
+ }
// A timeout means the app didn't answer in CALL_TIMEOUT. A run of
// these means the whole app is wedged — bail so callers fall back.
None => {
@@ -339,7 +423,7 @@ async fn collect_visited_bounded<'a>(
call(acc.get_role_name()),
call(acc.name()),
call(acc.get_state()),
- call(acc.get_children()),
+ call(raw_children(zconn, &oref)),
);
let role = match role_r {
Some(Ok(r)) => r,
@@ -371,14 +455,20 @@ async fn collect_visited_bounded<'a>(
}
if has_value {
if let Some(Ok(vp)) = call(proxies.value()).await {
- value = call(vp.current_value()).await.and_then(|r| r.ok()).map(format_value);
+ value = call(vp.current_value())
+ .await
+ .and_then(|r| r.ok())
+ .map(format_value);
}
}
// Text content is where editable/entry text (the typed string)
// lives; `name` is usually empty for such widgets.
if has_text {
if let Some(Ok(tp)) = call(proxies.text()).await {
- let count = call(tp.character_count()).await.and_then(|r| r.ok()).unwrap_or(0);
+ let count = call(tp.character_count())
+ .await
+ .and_then(|r| r.ok())
+ .unwrap_or(0);
if count > 0 {
let end = count.min(4096);
if let Some(Ok(t)) = call(tp.get_text(0, end)).await {
@@ -403,10 +493,14 @@ async fn collect_visited_bounded<'a>(
// would exceed the cap.
let descend = max_depth.map(|d| depth + 1 <= d).unwrap_or(true);
if descend {
- if let Some(Ok(children)) = children_r {
- for c in children.into_iter().rev() {
- stack.push((c, depth + 1, child_in_web_doc));
+ match children_r {
+ Some(Ok(children)) => {
+ for c in children.into_iter().rev() {
+ stack.push((c, depth + 1, child_in_web_doc));
+ }
}
+ Some(Err(error)) => dlog!(" get_children failed: {error:#}"),
+ None => dlog!(" get_children timed out"),
}
}
@@ -453,7 +547,9 @@ fn render(visited: &[Visited<'_>]) -> (String, Vec<AtspiNode>) {
let parent_element_index = if v.depth == 0 {
None
} else {
- (0..v.depth).rev().find_map(|d| parent_at_depth.get(d).copied().flatten())
+ (0..v.depth)
+ .rev()
+ .find_map(|d| parent_at_depth.get(d).copied().flatten())
};
if is_indexable(v) {
@@ -470,7 +566,11 @@ fn render(visited: &[Visited<'_>]) -> (String, Vec<AtspiNode>) {
nodes.push(AtspiNode {
element_index: Some(idx),
role: v.role.clone(),
- name: if v.name.is_empty() { None } else { Some(v.name.clone()) },
+ name: if v.name.is_empty() {
+ None
+ } else {
+ Some(v.name.clone())
+ },
value: v.value.clone().filter(|s| !s.is_empty()),
description: None,
actions: v.actions.clone(),
@@ -603,18 +703,29 @@ pub fn list_windows(filter_pid: Option<u32>) -> Vec<crate::x11::WindowInfo> {
continue;
}
}
- let app = match call(accessible_for(zconn, &app_ref)).await {
+ let app_ref = match RawObjectRef::from_atspi(&app_ref) {
+ Some(app_ref) => app_ref,
+ None => continue,
+ };
+ let app = match call(accessible_for(&conn, &app_ref)).await {
Some(Ok(a)) => a,
_ => continue,
};
- let app_name = call(app.name()).await.and_then(|r| r.ok()).unwrap_or_default();
+ let app_name = call(app.name())
+ .await
+ .and_then(|r| r.ok())
+ .unwrap_or_default();
let frames = match call(app.get_children()).await {
Some(Ok(c)) => c,
_ => Vec::new(),
};
let mut emitted = 0usize;
for (i, frame_ref) in frames.iter().enumerate() {
- let frame = match call(accessible_for(zconn, frame_ref)).await {
+ let frame_ref = match RawObjectRef::from_atspi(frame_ref) {
+ Some(frame_ref) => frame_ref,
+ None => continue,
+ };
+ let frame = match call(accessible_for(&conn, &frame_ref)).await {
Some(Ok(f)) => f,
_ => continue,
};
@@ -702,7 +813,10 @@ async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result<bool
};
dlog!(
"insert target: role={:?} in_web_doc={} focused={} has_component={}",
- target.role, target.in_web_doc, target.focused, target.has_component
+ target.role,
+ target.in_web_doc,
+ target.focused,
+ target.has_component
);
let proxies = target
@@ -751,100 +865,106 @@ async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result<bool
}
pub fn insert_text(pid: u32, text: &str) -> Result<bool> {
- bounded(async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = match collect_visited(&conn, pid).await? {
- Some(v) => v,
- None => return Ok(false),
- };
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = match collect_visited(&conn, pid).await? {
+ Some(v) => v,
+ None => return Ok(false),
+ };
- dlog!(
- "insert_text: {} node(s), {} editable, {} entry/text-role",
- visited.len(),
- visited.iter().filter(|v| v.has_editable).count(),
- visited.iter().filter(|v| v.role.contains("entry") || v.role.contains("text")).count(),
- );
+ dlog!(
+ "insert_text: {} node(s), {} editable, {} entry/text-role",
+ visited.len(),
+ visited.iter().filter(|v| v.has_editable).count(),
+ visited
+ .iter()
+ .filter(|v| v.role.contains("entry") || v.role.contains("text"))
+ .count(),
+ );
- // Primary attempt: write into an editable exposed by the current tree.
- // GrabFocus (inside write_into_editable) gives the widget internal
- // keyboard focus without activating the window, so toolkits that expose
- // EditableText on an unfocused window (Qt6, and GTK4 with GTK_A11Y=atspi)
- // accept the write here.
- if write_into_editable(&visited, text).await? {
- return Ok(true);
- }
+ // Primary attempt: write into an editable exposed by the current tree.
+ // GrabFocus (inside write_into_editable) gives the widget internal
+ // keyboard focus without activating the window, so toolkits that expose
+ // EditableText on an unfocused window (Qt6, and GTK4 with GTK_A11Y=atspi)
+ // accept the write here.
+ if write_into_editable(&visited, text).await? {
+ return Ok(true);
+ }
- // GTK3 fallback: the toolkit exposes entry/text nodes in the tree (so
- // get_text reads work) but gates EditableText on focus/activation. Try
- // finding an entry/text role with Component bounds and use X11 click+type.
- dlog!("AT-SPI EditableText unavailable; checking for entry/text with Component for X11 fallback");
+ // GTK3 fallback: the toolkit exposes entry/text nodes in the tree (so
+ // get_text reads work) but gates EditableText on focus/activation. Try
+ // finding an entry/text role with Component bounds and use X11 click+type.
+ dlog!("AT-SPI EditableText unavailable; checking for entry/text with Component for X11 fallback");
- let entry_candidate = visited
- .iter()
- .find(|v| {
+ let entry_candidate = visited.iter().find(|v| {
let r = v.role.to_ascii_lowercase();
(r.contains("entry") || r.contains("text")) && v.has_component
});
- if let Some(entry) = entry_candidate {
- dlog!(
+ if let Some(entry) = entry_candidate {
+ dlog!(
"GTK3 fallback: found entry role={:?} with Component; attempting X11 click+type",
entry.role
);
- // Get the entry widget's screen bounds via Component.GetExtents.
- if let Ok(proxies) = entry.acc.proxies().await {
- if let Ok(comp) = proxies.component().await {
- if let Some(Ok((x, y, w, h))) = call(comp.get_extents(CoordType::Screen)).await {
- // Click the center of the entry to establish widget focus (not window focus).
- let cx = x + (w.max(0) / 2);
- let cy = y + (h.max(0) / 2);
- dlog!("GTK3 fallback: entry bounds ({x},{y} {w}x{h}), clicking center ({cx},{cy})");
-
- // Get the window XID for this app so we can send X11 events to it.
- let Some(xid) = entry_find_window_xid(pid).await else {
- dlog!("GTK3 fallback: could not find window XID");
- return Ok(false);
- };
-
- // Translate screen coords to window-local coords for XSendEvent.
- let Some((wx, wy)) = screen_to_window_coords(xid, cx, cy) else {
- dlog!("GTK3 fallback: screen-to-window coord translation failed");
- return Ok(false);
- };
-
- dlog!("GTK3 fallback: window XID {xid}, local coords ({wx},{wy})");
-
- // Click the entry to focus the widget (widget focus, not window focus).
- if let Err(e) = crate::input::send_click(xid as u64, wx, wy, 1, 1) {
- dlog!("GTK3 fallback: click failed: {e}");
- return Ok(false);
- };
-
- // Small delay for the click to register and the widget to update focus.
- tokio::time::sleep(tokio::time::Duration::from_millis(150)).await;
-
- // Now type via X11 XSendEvent — the entry widget has internal focus
- // so it should accept the keystrokes even though the window is unfocused.
- if let Err(e) = crate::input::send_type_text(xid as u64, text) {
- dlog!("GTK3 fallback: send_type_text failed: {e}");
- return Ok(false);
- }
+ // Get the entry widget's screen bounds via Component.GetExtents.
+ if let Ok(proxies) = entry.acc.proxies().await {
+ if let Ok(comp) = proxies.component().await {
+ if let Some(Ok((x, y, w, h))) =
+ call(comp.get_extents(CoordType::Screen)).await
+ {
+ // Click the center of the entry to establish widget focus (not window focus).
+ let cx = x + (w.max(0) / 2);
+ let cy = y + (h.max(0) / 2);
+ dlog!("GTK3 fallback: entry bounds ({x},{y} {w}x{h}), clicking center ({cx},{cy})");
+
+ // Get the window XID for this app so we can send X11 events to it.
+ let Some(xid) = entry_find_window_xid(pid).await else {
+ dlog!("GTK3 fallback: could not find window XID");
+ return Ok(false);
+ };
+
+ // Translate screen coords to window-local coords for XSendEvent.
+ let Some((wx, wy)) = screen_to_window_coords(xid, cx, cy) else {
+ dlog!("GTK3 fallback: screen-to-window coord translation failed");
+ return Ok(false);
+ };
+
+ dlog!("GTK3 fallback: window XID {xid}, local coords ({wx},{wy})");
+
+ // Click the entry to focus the widget (widget focus, not window focus).
+ if let Err(e) = crate::input::send_click(xid as u64, wx, wy, 1, 1) {
+ dlog!("GTK3 fallback: click failed: {e}");
+ return Ok(false);
+ };
+
+ // Small delay for the click to register and the widget to update focus.
+ tokio::time::sleep(tokio::time::Duration::from_millis(150)).await;
+
+ // Now type via X11 XSendEvent — the entry widget has internal focus
+ // so it should accept the keystrokes even though the window is unfocused.
+ if let Err(e) = crate::input::send_type_text(xid as u64, text) {
+ dlog!("GTK3 fallback: send_type_text failed: {e}");
+ return Ok(false);
+ }
- dlog!("GTK3 fallback: X11 click+type succeeded");
- return Ok(true);
+ dlog!("GTK3 fallback: X11 click+type succeeded");
+ return Ok(true);
+ }
}
}
}
- }
- Ok(false)
- }, || {
- dlog!("insert_text timed out for pid {pid}; falling back to synthetic typing");
- Ok(false)
- })
+ Ok(false)
+ },
+ || {
+ dlog!("insert_text timed out for pid {pid}; falling back to synthetic typing");
+ Ok(false)
+ },
+ )
}
/// Classify what holds keyboard focus in `pid`'s tree, so `type_text` can target
@@ -859,16 +979,19 @@ pub fn insert_text(pid: u32, text: &str) -> Result<bool> {
/// `None` — nothing is focused (or the app is unreachable): fall back to
/// the focus-free "first editable" path for background typing.
pub fn focused_is_editable(pid: u32) -> Result<Option<bool>> {
- bounded(async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = match collect_visited(&conn, pid).await? {
- Some(v) => v,
- None => return Ok(None),
- };
- Ok(visited.iter().find(|v| v.focused).map(|v| v.has_editable))
- }, || Ok(None))
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = match collect_visited(&conn, pid).await? {
+ Some(v) => v,
+ None => return Ok(None),
+ };
+ Ok(visited.iter().find(|v| v.focused).map(|v| v.has_editable))
+ },
+ || Ok(None),
+ )
}
/// Find the window XID for a PID by listing its X11 windows.
@@ -883,7 +1006,6 @@ async fn entry_find_window_xid(pid: u32) -> Option<u64> {
/// Translate screen coordinates to window-local coordinates.
fn screen_to_window_coords(xid: u64, screen_x: i32, screen_y: i32) -> Option<(i32, i32)> {
- use x11rb::connection::Connection;
use x11rb::protocol::xproto::*;
use x11rb::rust_connection::RustConnection;
@@ -894,47 +1016,168 @@ fn screen_to_window_coords(xid: u64, screen_x: i32, screen_y: i32) -> Option<(i3
let geom = conn.get_geometry(window).ok()?.reply().ok()?;
// Translate to root coordinates (screen coords of window's origin).
- let trans = conn.translate_coordinates(window, geom.root, 0, 0).ok()?.reply().ok()?;
+ let trans = conn
+ .translate_coordinates(window, geom.root, 0, 0)
+ .ok()?
+ .reply()
+ .ok()?;
// Window-local = screen - window_origin.
Some((screen_x - trans.dst_x as i32, screen_y - trans.dst_y as i32))
}
pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
- bounded(async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
- .await?
- .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
- let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
- let target = action_nodes
- .get(idx)
- .ok_or_else(|| anyhow!("element {idx} not found (total: {})", action_nodes.len()))?;
-
- // Suspected no-op: actuating `do_action(0)` on a passive display role
- // (a `label`/`static`/`image` indexed only for its Value interface) or a
- // node that advertises no action at all is the AT-SPI analogue of macOS'
- // "element does not advertise this action" — the call returns success but
- // likely changes nothing. Reuses the same passive-role detector
- // `select_click_target` leans on for the coordinate paths. The caller
- // turns this into `effect: "suspected_noop"` + an escalation hint.
- let suspected_noop = target.actions.is_empty() || is_passive_role(&target.role);
-
- let ap = target
- .acc
- .proxies()
- .await
- .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?
- .action()
- .await
- .map_err(|e| anyhow!("Action unavailable: {e}"))?;
- ap.do_action(0)
- .await
- .map_err(|e| anyhow!("doAction failed: {e}"))?;
- Ok((target.actions.first().cloned().unwrap_or_default(), suspected_noop))
- }, || Err(anyhow!("perform_action timed out for pid {pid} (app unresponsive to AT-SPI)")))
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = collect_visited(&conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
+ let target = action_nodes.get(idx).ok_or_else(|| {
+ anyhow!("element {idx} not found (total: {})", action_nodes.len())
+ })?;
+
+ // Suspected no-op: actuating `do_action(0)` on a passive display role
+ // (a `label`/`static`/`image` indexed only for its Value interface) or a
+ // node that advertises no action at all is the AT-SPI analogue of macOS'
+ // "element does not advertise this action" — the call returns success but
+ // likely changes nothing. Reuses the same passive-role detector
+ // `select_click_target` leans on for the coordinate paths. The caller
+ // turns this into `effect: "suspected_noop"` + an escalation hint.
+ let suspected_noop = target.actions.is_empty() || is_passive_role(&target.role);
+
+ let ap = target
+ .acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?
+ .action()
+ .await
+ .map_err(|e| anyhow!("Action unavailable: {e}"))?;
+ let action = target.actions.first().cloned().unwrap_or_default();
+ ap.do_action(0)
+ .await
+ .map_err(|e| anyhow!("doAction failed: {e}"))?;
+ // AT-SPI's doAction acknowledgement can precede the renderer's
+ // queued DOM mutation. Give WebKit/Chromium one short event-loop
+ // turn before returning success so a caller's immediate external
+ // state read observes the action it was told was delivered.
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ Ok((action, suspected_noop))
+ },
+ || {
+ Err(anyhow!(
+ "perform_action timed out for pid {pid} (app unresponsive to AT-SPI)"
+ ))
+ },
+ )
+}
+
+/// Invoke an indexed scroll target's directional AT-SPI action.
+///
+/// Chromium exposes scrollable web regions as named actions such as
+/// `scrollDown`/`scrollForward`; using that accessibility route avoids the
+/// X11 `Button5` event path that Chromium silently drops in background mode.
+pub fn scroll_element(pid: u32, idx: usize, direction: &str, amount: usize) -> Result<()> {
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = collect_visited(&conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let target = visited
+ .iter()
+ .filter(|v| is_indexable(v))
+ .nth(idx)
+ .ok_or_else(|| anyhow!("element {idx} not found (total: {})", visited.len()))?;
+ let proxies = target
+ .acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
+ let action = proxies
+ .action()
+ .await
+ .map_err(|e| anyhow!("Action interface unavailable: {e}"))?;
+ let wanted = match direction {
+ "up" => ["scrollup", "scrollbackward"],
+ "left" => ["scrollleft", "scrollbackward"],
+ "right" => ["scrollright", "scrollforward"],
+ _ => ["scrolldown", "scrollforward"],
+ };
+ let count = call(action.n_actions())
+ .await
+ .and_then(|result| result.ok())
+ .unwrap_or(0);
+ let mut selected = None;
+ for action_index in 0..count {
+ if let Some(Ok(name)) = call(action.get_name(action_index)).await {
+ let normalized: String = name
+ .chars()
+ .filter(|ch| ch.is_ascii_alphanumeric())
+ .flat_map(|ch| ch.to_lowercase())
+ .collect();
+ if wanted.iter().any(|candidate| *candidate == normalized) {
+ selected = Some(action_index);
+ break;
+ }
+ }
+ }
+ let action_index = selected.ok_or_else(|| {
+ anyhow!("element {idx} exposes no directional scroll action for {direction}")
+ })?;
+ for _ in 0..amount.max(1) {
+ match call(action.do_action(action_index)).await {
+ Some(Ok(true)) => {}
+ Some(Ok(false)) => return Err(anyhow!("scroll action returned false")),
+ Some(Err(e)) => return Err(anyhow!("scroll action failed: {e}")),
+ None => return Err(anyhow!("scroll action timed out")),
+ }
+ }
+ Ok(())
+ },
+ || Err(anyhow!("scroll_element timed out for pid {pid}")),
+ )
+}
+
+/// Give an indexed element keyboard focus through AT-SPI Component.GrabFocus
+/// without activating or raising its toplevel window.
+pub fn focus_element(pid: u32, idx: usize) -> Result<bool> {
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = collect_visited(&conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let target = visited
+ .iter()
+ .filter(|v| is_indexable(v))
+ .nth(idx)
+ .ok_or_else(|| anyhow!("element {idx} not found (total: {})", visited.len()))?;
+ let proxies = target
+ .acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
+ let component = proxies
+ .component()
+ .await
+ .map_err(|e| anyhow!("Component interface unavailable: {e}"))?;
+ match call(component.grab_focus()).await {
+ Some(Ok(focused)) => Ok(focused),
+ Some(Err(e)) => Err(anyhow!("Component.GrabFocus failed for element {idx}: {e}")),
+ None => Err(anyhow!("Component.GrabFocus timed out for element {idx}")),
+ }
+ },
+ || Err(anyhow!("focus_element timed out for pid {pid}")),
+ )
}
/// Resolve a window-local pixel `(win_x, win_y)` to the deepest actionable
@@ -955,52 +1198,61 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
/// when an element was actuated, `Ok(None)` when no actionable element covers the
/// point (the caller then falls back to the synthetic X11 path).
pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result<Option<String>> {
- bounded(async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = match collect_visited(&conn, pid).await? {
- Some(v) => v,
- None => return Ok(None),
- };
-
- // Collect actionable nodes whose window-local bounds contain the point,
- // then let `select_click_target` pick the innermost *real actuator* —
- // preferring a button over its slightly-smaller inner label (GTK4 nests
- // one inside every button; an area-only pick lands on the inert label
- // and `do_action` silently no-ops). Pre-order keeps containers ahead of
- // children, but the area/role split is what actually disambiguates.
- let mut frames: Vec<(usize, i32, i32, u32, u32, bool)> = Vec::new();
- for (i, v) in visited.iter().enumerate() {
- if v.actions.is_empty() || !v.has_component {
- continue;
- }
- let Some(Ok(proxies)) = call(v.acc.proxies()).await else { continue };
- let Some(Ok(comp)) = call(proxies.component()).await else { continue };
- let Some(Ok((x, y, w, h))) = call(comp.get_extents(CoordType::Window)).await else {
- continue;
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = match collect_visited(&conn, pid).await? {
+ Some(v) => v,
+ None => return Ok(None),
};
- if w <= 0 || h <= 0 {
- continue;
+
+ // Collect actionable nodes whose window-local bounds contain the point,
+ // then let `select_click_target` pick the innermost *real actuator* —
+ // preferring a button over its slightly-smaller inner label (GTK4 nests
+ // one inside every button; an area-only pick lands on the inert label
+ // and `do_action` silently no-ops). Pre-order keeps containers ahead of
+ // children, but the area/role split is what actually disambiguates.
+ let mut frames: Vec<(usize, i32, i32, u32, u32, bool)> = Vec::new();
+ for (i, v) in visited.iter().enumerate() {
+ if v.actions.is_empty() || !v.has_component {
+ continue;
+ }
+ let Some(Ok(proxies)) = call(v.acc.proxies()).await else {
+ continue;
+ };
+ let Some(Ok(comp)) = call(proxies.component()).await else {
+ continue;
+ };
+ let Some(Ok((x, y, w, h))) = call(comp.get_extents(CoordType::Window)).await else {
+ continue;
+ };
+ if w <= 0 || h <= 0 {
+ continue;
+ }
+ frames.push((i, x, y, w as u32, h as u32, is_passive_role(&v.role)));
}
- frames.push((i, x, y, w as u32, h as u32, is_passive_role(&v.role)));
- }
- let Some(idx) = select_click_target(&frames, win_x, win_y) else { return Ok(None) };
- let target = &visited[idx];
- let ap = target
- .acc
- .proxies()
- .await
- .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?
- .action()
- .await
- .map_err(|e| anyhow!("Action unavailable: {e}"))?;
- ap.do_action(0)
- .await
- .map_err(|e| anyhow!("doAction failed: {e}"))?;
- Ok(Some(target.actions.first().cloned().unwrap_or_default()))
- }, || Ok(None))
+ let Some(idx) = select_click_target(&frames, win_x, win_y) else {
+ return Ok(None);
+ };
+ let target = &visited[idx];
+ let ap = target
+ .acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?
+ .action()
+ .await
+ .map_err(|e| anyhow!("Action unavailable: {e}"))?;
+ ap.do_action(0)
+ .await
+ .map_err(|e| anyhow!("doAction failed: {e}"))?;
+ Ok(Some(target.actions.first().cloned().unwrap_or_default()))
+ },
+ || Ok(None),
+ )
}
/// Vision/pixel click that actually lands — the Wayland answer (and a robust
@@ -1031,61 +1283,79 @@ pub fn perform_action_at_screen_point(
screen_x: i32,
screen_y: i32,
) -> Result<Option<String>> {
- bounded(async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = match collect_visited(&conn, pid).await? {
- Some(v) => v,
- None => return Ok(None),
- };
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = match collect_visited(&conn, pid).await? {
+ Some(v) => v,
+ None => return Ok(None),
+ };
- // Reconstruct each indexable element's SCREEN frame the same way
- // get_window_state does: WINDOW-relative extents (GTK4 reports these
- // correctly; Screen is (0,0)) plus the window's screen origin (the
- // GNOME Shell helper on Wayland, _GTK_FRAME_EXTENTS on X11). When no
- // offset resolves, fall back to CoordType::Screen (correct on Qt/GTK3).
- let offset = window_to_screen_offset(pid, xid);
- let coord = if offset.is_some() { CoordType::Window } else { CoordType::Screen };
- let (ox, oy) = offset.unwrap_or((0, 0));
-
- // (element_index, x, y, w, h, is_passive_label) over the SAME indexable
- // list `perform_action`/`get_window_state` use, so the chosen index
- // maps straight back to a verified `element_index` actuation.
- let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
- let mut frames: Vec<(usize, i32, i32, u32, u32, bool)> = Vec::new();
- for (idx, node) in action_nodes.iter().enumerate() {
- if !node.has_component {
- continue;
- }
- let Some(Ok(proxies)) = call(node.acc.proxies()).await else { continue };
- let Some(Ok(comp)) = call(proxies.component()).await else { continue };
- let Some(Ok((x, y, w, h))) = call(comp.get_extents(coord)).await else {
- continue;
+ // Reconstruct each indexable element's SCREEN frame the same way
+ // get_window_state does: WINDOW-relative extents (GTK4 reports these
+ // correctly; Screen is (0,0)) plus the window's screen origin (the
+ // GNOME Shell helper on Wayland, _GTK_FRAME_EXTENTS on X11). When no
+ // offset resolves, fall back to CoordType::Screen (correct on Qt/GTK3).
+ let offset = window_to_screen_offset(pid, xid);
+ let coord = if offset.is_some() {
+ CoordType::Window
+ } else {
+ CoordType::Screen
};
- if x == i32::MIN || y == i32::MIN || w <= 1 || h <= 1 {
- continue;
+ let (ox, oy) = offset.unwrap_or((0, 0));
+
+ // (element_index, x, y, w, h, is_passive_label) over the SAME indexable
+ // list `perform_action`/`get_window_state` use, so the chosen index
+ // maps straight back to a verified `element_index` actuation.
+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
+ let mut frames: Vec<(usize, i32, i32, u32, u32, bool)> = Vec::new();
+ for (idx, node) in action_nodes.iter().enumerate() {
+ if !node.has_component {
+ continue;
+ }
+ let Some(Ok(proxies)) = call(node.acc.proxies()).await else {
+ continue;
+ };
+ let Some(Ok(comp)) = call(proxies.component()).await else {
+ continue;
+ };
+ let Some(Ok((x, y, w, h))) = call(comp.get_extents(coord)).await else {
+ continue;
+ };
+ if x == i32::MIN || y == i32::MIN || w <= 1 || h <= 1 {
+ continue;
+ }
+ frames.push((
+ idx,
+ x + ox,
+ y + oy,
+ w as u32,
+ h as u32,
+ is_passive_role(&node.role),
+ ));
}
- frames.push((idx, x + ox, y + oy, w as u32, h as u32, is_passive_role(&node.role)));
- }
- let Some(idx) = select_click_target(&frames, screen_x, screen_y) else {
- return Ok(None);
- };
- let target = action_nodes[idx];
- let ap = target
- .acc
- .proxies()
- .await
- .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?
- .action()
- .await
- .map_err(|e| anyhow!("Action unavailable: {e}"))?;
- ap.do_action(0)
- .await
- .map_err(|e| anyhow!("doAction failed: {e}"))?;
- Ok(Some(target.actions.first().cloned().unwrap_or_default()))
- }, || Ok(None))
+ let Some(idx) = select_click_target(&frames, screen_x, screen_y) else {
+ return Ok(None);
+ };
+ let target = action_nodes[idx];
+ let ap = target
+ .acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?
+ .action()
+ .await
+ .map_err(|e| anyhow!("Action unavailable: {e}"))?;
+ ap.do_action(0)
+ .await
+ .map_err(|e| anyhow!("doAction failed: {e}"))?;
+ Ok(Some(target.actions.first().cloned().unwrap_or_default()))
+ },
+ || Ok(None),
+ )
}
/// Roles that draw text/graphics but don't *do* anything when actuated. GTK4
@@ -1117,7 +1387,11 @@ fn select_click_target(
let (w, h) = (w as i32, h as i32);
if px >= x && px < x + w && py >= y && py < y + h {
let area = (w as i64) * (h as i64);
- let slot = if passive { &mut best_passive } else { &mut best_active };
+ let slot = if passive {
+ &mut best_passive
+ } else {
+ &mut best_active
+ };
if slot.map(|(a, _)| area < a).unwrap_or(true) {
*slot = Some((area, idx));
}
@@ -1127,121 +1401,136 @@ fn select_click_target(
}
pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {
- bounded(async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
- .await?
- .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
- let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
- let target = action_nodes
- .get(idx)
- .ok_or_else(|| anyhow!("element {idx} not found (total: {})", action_nodes.len()))?;
-
- let proxies = target
- .acc
- .proxies()
- .await
- .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
-
- // EditableText write. We don't gate on the cached `has_editable` flag:
- // GTK4 (and similar toolkits) only advertise the EditableText interface
- // on a widget once it holds keyboard focus, so the interface list
- // captured during the unfocused tree walk can be missing it even though
- // the element is a real editable text box. GrabFocus first (internal
- // widget focus, no window activation — same trick as `type_text`'s
- // EditableText path), then resolve the EditableText proxy live over
- // D-Bus and try to write. If the proxy genuinely isn't there the
- // `editable_text()` resolve fails and we fall through to Value below.
- if target.has_component {
- if let Ok(comp) = proxies.component().await {
- let _ = call(comp.grab_focus()).await;
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = collect_visited(&conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
+ let target = action_nodes.get(idx).ok_or_else(|| {
+ anyhow!("element {idx} not found (total: {})", action_nodes.len())
+ })?;
+
+ let proxies = target
+ .acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
+
+ // EditableText write. We don't gate on the cached `has_editable` flag:
+ // GTK4 (and similar toolkits) only advertise the EditableText interface
+ // on a widget once it holds keyboard focus, so the interface list
+ // captured during the unfocused tree walk can be missing it even though
+ // the element is a real editable text box. GrabFocus first (internal
+ // widget focus, no window activation — same trick as `type_text`'s
+ // EditableText path), then resolve the EditableText proxy live over
+ // D-Bus and try to write. If the proxy genuinely isn't there the
+ // `editable_text()` resolve fails and we fall through to Value below.
+ if target.has_component {
+ if let Ok(comp) = proxies.component().await {
+ let _ = call(comp.grab_focus()).await;
+ }
}
- }
- if let Ok(et) = proxies.editable_text().await {
- // Replace whole contents (parity with the Windows/macOS set_value,
- // which overwrite rather than insert at the caret).
- if et.set_text_contents(value).await.unwrap_or(false) {
- return Ok(());
+ if let Ok(et) = proxies.editable_text().await {
+ // Replace whole contents (parity with the Windows/macOS set_value,
+ // which overwrite rather than insert at the caret).
+ if et.set_text_contents(value).await.unwrap_or(false) {
+ return Ok(());
+ }
+ // Some toolkits reject SetTextContents but accept an insert at the
+ // caret offset; clear-then-insert as a fallback.
+ let off = match proxies.text().await {
+ Ok(tp) => tp.caret_offset().await.unwrap_or(0),
+ Err(_) => 0,
+ };
+ let len = value.chars().count() as i32;
+ if et.insert_text(off, value, len).await.unwrap_or(false) {
+ return Ok(());
+ }
}
- // Some toolkits reject SetTextContents but accept an insert at the
- // caret offset; clear-then-insert as a fallback.
- let off = match proxies.text().await {
- Ok(tp) => tp.caret_offset().await.unwrap_or(0),
- Err(_) => 0,
- };
- let len = value.chars().count() as i32;
- if et.insert_text(off, value, len).await.unwrap_or(false) {
+ if target.has_value {
+ let v: f64 = value
+ .parse()
+ .map_err(|_| anyhow!("value '{value}' is not numeric for a Value element"))?;
+ proxies
+ .value()
+ .await
+ .map_err(|e| anyhow!("Value unavailable: {e}"))?
+ .set_current_value(v)
+ .await
+ .map_err(|e| anyhow!("setCurrentValue failed: {e}"))?;
return Ok(());
}
- }
- if target.has_value {
- let v: f64 = value
- .parse()
- .map_err(|_| anyhow!("value '{value}' is not numeric for a Value element"))?;
- proxies
- .value()
- .await
- .map_err(|e| anyhow!("Value unavailable: {e}"))?
- .set_current_value(v)
- .await
- .map_err(|e| anyhow!("setCurrentValue failed: {e}"))?;
- return Ok(());
- }
- Err(anyhow!("element {idx} exposes neither EditableText nor Value"))
- }, || Err(anyhow!("set_value timed out for pid {pid} (app unresponsive to AT-SPI)")))
+ Err(anyhow!(
+ "element {idx} exposes neither EditableText nor Value"
+ ))
+ },
+ || {
+ Err(anyhow!(
+ "set_value timed out for pid {pid} (app unresponsive to AT-SPI)"
+ ))
+ },
+ )
}
pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> {
- bounded(async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
- .await?
- .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
- let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
- let target = action_nodes
- .get(idx)
- .ok_or_else(|| anyhow!("element {idx} not found"))?;
- if !target.has_component {
- return Err(anyhow!("element {idx} exposes no Component interface"));
- }
- let comp = target
- .acc
- .proxies()
- .await
- .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?
- .component()
- .await
- .map_err(|e| anyhow!("Component unavailable: {e}"))?;
- // Prefer WINDOW coords + a deterministic screen offset — fixes GTK4,
- // whose CoordType::Screen collapses every element to (0,0). Fall back to
- // Screen on Wayland / when no X11 window resolves (offset is None).
- match window_to_screen_offset(pid, 0) {
- Some((ox, oy)) => {
- let (x, y, w, h) = comp
- .get_extents(CoordType::Window)
- .await
- .map_err(|e| anyhow!("getExtents failed: {e}"))?;
- Ok((x + ox, y + oy, w.max(0) as u32, h.max(0) as u32))
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = collect_visited(&conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
+ let target = action_nodes
+ .get(idx)
+ .ok_or_else(|| anyhow!("element {idx} not found"))?;
+ if !target.has_component {
+ return Err(anyhow!("element {idx} exposes no Component interface"));
}
- None => {
- let (x, y, w, h) = comp
- .get_extents(CoordType::Screen)
- .await
- .map_err(|e| anyhow!("getExtents failed: {e}"))?;
- Ok((x, y, w.max(0) as u32, h.max(0) as u32))
+ let comp = target
+ .acc
+ .proxies()
+ .await
+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?
+ .component()
+ .await
+ .map_err(|e| anyhow!("Component unavailable: {e}"))?;
+ // Prefer WINDOW coords + a deterministic screen offset — fixes GTK4,
+ // whose CoordType::Screen collapses every element to (0,0). Fall back to
+ // Screen on Wayland / when no X11 window resolves (offset is None).
+ match window_to_screen_offset(pid, 0) {
+ Some((ox, oy)) => {
+ let (x, y, w, h) = comp
+ .get_extents(CoordType::Window)
+ .await
+ .map_err(|e| anyhow!("getExtents failed: {e}"))?;
+ Ok((x + ox, y + oy, w.max(0) as u32, h.max(0) as u32))
+ }
+ None => {
+ let (x, y, w, h) = comp
+ .get_extents(CoordType::Screen)
+ .await
+ .map_err(|e| anyhow!("getExtents failed: {e}"))?;
+ Ok((x, y, w.max(0) as u32, h.max(0) as u32))
+ }
}
- }
- }, || Err(anyhow!("get_element_bounds timed out for pid {pid} (app unresponsive to AT-SPI)")))
+ },
+ || {
+ Err(anyhow!(
+ "get_element_bounds timed out for pid {pid} (app unresponsive to AT-SPI)"
+ ))
+ },
+ )
}
/// Real on-screen origin (root-relative top-left) of an X11 window, or `None`
/// if it can't be resolved. Mirrors `list_windows`' geometry path.
fn x11_window_origin(xid: u64) -> Option<(i32, i32)> {
- use x11rb::connection::Connection;
use x11rb::protocol::xproto::*;
use x11rb::rust_connection::RustConnection;
@@ -1267,7 +1556,6 @@ fn x11_window_origin(xid: u64) -> Option<(i32, i32)> {
/// server-side-decorated windows that don't set the property — which is also
/// how we tell GTK4-CSD apart from everyone else.
fn gtk_frame_extents(xid: u64) -> Option<(i32, i32)> {
- use x11rb::connection::Connection;
use x11rb::protocol::xproto::*;
use x11rb::rust_connection::RustConnection;
@@ -1370,75 +1658,88 @@ fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> {
///
/// Returns `(element_index, x, y, width, height)` tuples.
pub fn get_all_element_bounds(pid: u32, xid: u64) -> Result<Vec<(usize, i32, i32, u32, u32)>> {
- bounded(async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
- .await?
- .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
-
- // Query WINDOW-relative extents and add a deterministic screen offset
- // (X11 window origin + GTK4 CSD inset). This fixes GTK4 — whose
- // CoordType::Screen reports every element at (0,0) — by using the
- // distinct per-widget WINDOW coords instead. On Wayland / when no X11
- // window resolves, `offset` is None and we keep the legacy Screen path
- // so non-X11 behaviour is unchanged.
- let offset = window_to_screen_offset(pid, xid);
- let coord = if offset.is_some() { CoordType::Window } else { CoordType::Screen };
- let (offset_x, offset_y) = offset.unwrap_or((0, 0));
- if let Some((ox, oy)) = offset {
- dlog!("element bounds: WINDOW coords + screen offset ({ox},{oy})");
- }
-
- let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
- // Each element costs ~3 D-Bus round-trips (proxies + component +
- // GetExtents). Big trees (geany exposes ~787 nodes) would grind for
- // minutes and time out callers, so cap the walk; pre-order means the
- // first nodes are the window chrome / toolbars that are actually
- // visible, which is what bounds consumers (overlays, targeting) need.
- const MAX_BOUNDS_NODES: usize = 150;
- // Hard wall-clock budget for the whole collection: on pathological
- // trees individual D-Bus calls each burn up to CALL_TIMEOUT (geany's
- // unrealized nodes did exactly that), so a per-node cap alone can
- // still add up to minutes. Return whatever was collected in time.
- let deadline = std::time::Instant::now() + Duration::from_secs(20);
- let mut out = Vec::with_capacity(action_nodes.len().min(MAX_BOUNDS_NODES));
- for (idx, node) in action_nodes.iter().enumerate().take(MAX_BOUNDS_NODES) {
- if std::time::Instant::now() >= deadline {
- dlog!("get_all_element_bounds: 20s budget exhausted at node {idx}; returning {} bound(s)", out.len());
- break;
- }
- if !node.has_component {
- continue;
- }
- let proxies = match call(node.acc.proxies()).await {
- Some(Ok(p)) => p,
- _ => continue,
- };
- let comp = match call(proxies.component()).await {
- Some(Ok(c)) => c,
- _ => continue,
+ bounded(
+ async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let visited = collect_visited(&conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+
+ // Query WINDOW-relative extents and add a deterministic screen offset
+ // (X11 window origin + GTK4 CSD inset). This fixes GTK4 — whose
+ // CoordType::Screen reports every element at (0,0) — by using the
+ // distinct per-widget WINDOW coords instead. On Wayland / when no X11
+ // window resolves, `offset` is None and we keep the legacy Screen path
+ // so non-X11 behaviour is unchanged.
+ let offset = window_to_screen_offset(pid, xid);
+ let coord = if offset.is_some() {
+ CoordType::Window
+ } else {
+ CoordType::Screen
};
- if let Some(Ok((x, y, w, h))) = call(comp.get_extents(coord)).await {
- // Unrealized widgets (e.g. items inside closed menus/popovers)
- // report GetExtents as the i32::MIN sentinel and/or a degenerate
- // 0x0 / 1x1 size. Emitting those poisons downstream consumers
- // (overlay renderers, click targeting), so keep only elements
- // with plausible on-screen geometry. (Validate the raw extents,
- // before applying the screen offset, so the sentinel check still
- // catches unrealized widgets.)
- if x == i32::MIN || y == i32::MIN || x < -16384 || y < -16384 || w <= 1 || h <= 1 {
+ let (offset_x, offset_y) = offset.unwrap_or((0, 0));
+ if let Some((ox, oy)) = offset {
+ dlog!("element bounds: WINDOW coords + screen offset ({ox},{oy})");
+ }
+
+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
+ // Each element costs ~3 D-Bus round-trips (proxies + component +
+ // GetExtents). Big trees (geany exposes ~787 nodes) would grind for
+ // minutes and time out callers, so cap the walk; pre-order means the
+ // first nodes are the window chrome / toolbars that are actually
+ // visible, which is what bounds consumers (overlays, targeting) need.
+ const MAX_BOUNDS_NODES: usize = 150;
+ // Hard wall-clock budget for the whole collection: on pathological
+ // trees individual D-Bus calls each burn up to CALL_TIMEOUT (geany's
+ // unrealized nodes did exactly that), so a per-node cap alone can
+ // still add up to minutes. Return whatever was collected in time.
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ let mut out = Vec::with_capacity(action_nodes.len().min(MAX_BOUNDS_NODES));
+ for (idx, node) in action_nodes.iter().enumerate().take(MAX_BOUNDS_NODES) {
+ if std::time::Instant::now() >= deadline {
+ dlog!("get_all_element_bounds: 20s budget exhausted at node {idx}; returning {} bound(s)", out.len());
+ break;
+ }
+ if !node.has_component {
continue;
}
- out.push((idx, x + offset_x, y + offset_y, w as u32, h as u32));
+ let proxies = match call(node.acc.proxies()).await {
+ Some(Ok(p)) => p,
+ _ => continue,
+ };
+ let comp = match call(proxies.component()).await {
+ Some(Ok(c)) => c,
+ _ => continue,
+ };
+ if let Some(Ok((x, y, w, h))) = call(comp.get_extents(coord)).await {
+ // Unrealized widgets (e.g. items inside closed menus/popovers)
+ // report GetExtents as the i32::MIN sentinel and/or a degenerate
+ // 0x0 / 1x1 size. Emitting those poisons downstream consumers
+ // (overlay renderers, click targeting), so keep only elements
+ // with plausible on-screen geometry. (Validate the raw extents,
+ // before applying the screen offset, so the sentinel check still
+ // catches unrealized widgets.)
+ if x == i32::MIN
+ || y == i32::MIN
+ || x < -16384
+ || y < -16384
+ || w <= 1
+ || h <= 1
+ {
+ continue;
+ }
+ out.push((idx, x + offset_x, y + offset_y, w as u32, h as u32));
+ }
}
- }
- Ok(out)
- }, || {
- dlog!("get_all_element_bounds timed out for pid {pid}; returning no bounds");
- Ok(Vec::new())
- })
+ Ok(out)
+ },
+ || {
+ dlog!("get_all_element_bounds timed out for pid {pid}; returning no bounds");
+ Ok(Vec::new())
+ },
+ )
}
#[cfg(test)]
@@ -1463,9 +1764,9 @@ mod coord_tests {
#[test]
fn click_target_smallest_active_over_enclosing_panel() {
let frames = vec![
- (0usize, 0, 0, 400, 600, false), // panel
- (3usize, 80, 320, 64, 44, false), // button "7"
- (5usize, 150, 320, 64, 44, false), // button "8"
+ (0usize, 0, 0, 400, 600, false), // panel
+ (3usize, 80, 320, 64, 44, false), // button "7"
+ (5usize, 150, 320, 64, 44, false), // button "8"
];
assert_eq!(select_click_target(&frames, 100, 340), Some(3));
assert_eq!(select_click_target(&frames, 180, 340), Some(5));
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
index 1483a9a18f..e00a528494 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
@@ -46,7 +46,6 @@ fn capture_via_import(xid: u64) -> Result<Vec<u8>> {
}
fn capture_via_xgetimage(xid: u64) -> Result<(String, u32, u32)> {
- use x11rb::connection::Connection;
use x11rb::protocol::xproto::*;
use x11rb::rust_connection::RustConnection;
@@ -57,13 +56,17 @@ fn capture_via_xgetimage(xid: u64) -> Result<(String, u32, u32)> {
let w = geom.width as u32;
let h = geom.height as u32;
- let img = conn.get_image(
- ImageFormat::Z_PIXMAP,
- window,
- 0, 0,
- w as u16, h as u16,
- !0u32,
- )?.reply()?;
+ let img = conn
+ .get_image(
+ ImageFormat::Z_PIXMAP,
+ window,
+ 0,
+ 0,
+ w as u16,
+ h as u16,
+ !0u32,
+ )?
+ .reply()?;
// The raw data is BGRA or BGRX depending on depth.
// Encode as a minimal PNG.
@@ -71,7 +74,7 @@ fn capture_via_xgetimage(xid: u64) -> Result<(String, u32, u32)> {
let (bpp, has_alpha) = match img.depth {
32 => (4usize, true),
24 => (4usize, false),
- _ => bail!("Unsupported depth: {}", img.depth),
+ _ => bail!("Unsupported depth: {}", img.depth),
};
// Convert to RGBA.
@@ -156,9 +159,14 @@ pub(crate) fn screenshot_display_bytes_x11() -> Result<Vec<u8>> {
crate::no_display_hint()
);
}
- let img = conn.get_image(ImageFormat::Z_PIXMAP, root, 0, 0, w as u16, h as u16, !0u32)?.reply()?;
+ let img = conn
+ .get_image(ImageFormat::Z_PIXMAP, root, 0, 0, w as u16, h as u16, !0u32)?
+ .reply()?;
let bytes = img.data;
- let bpp = match img.depth { 32 | 24 => 4usize, _ => anyhow::bail!("Unsupported depth") };
+ let bpp = match img.depth {
+ 32 | 24 => 4usize,
+ _ => anyhow::bail!("Unsupported depth"),
+ };
let mut rgba = Vec::with_capacity((w * h * 4) as usize);
for chunk in bytes.chunks_exact(bpp) {
let (b, g, r) = (chunk[0], chunk[1], chunk[2]);
@@ -197,4 +205,3 @@ pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> Result<Vec<u8>> {
pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result<Vec<u8>> {
cua_driver_core::image_utils::crosshair_png_bytes(png_bytes, cx, cy)
}
-
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
index 57eb3f0045..258c392cdb 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
@@ -58,7 +58,9 @@ impl DeliveryMode {
Self::parse(args.get("delivery_mode").and_then(|v| v.as_str()))
}
- pub fn is_foreground(self) -> bool { matches!(self, Self::Foreground) }
+ pub fn is_foreground(self) -> bool {
+ matches!(self, Self::Foreground)
+ }
}
/// JSON-schema fragment for the `delivery_mode` field. Include this in every
@@ -93,20 +95,29 @@ pub enum BackgroundUnavailable {
/// No libei backend (built without `portal-libei`, or the portal session
/// was denied / unavailable). Input has no actuator at all.
NoLibeiBackend,
+ /// X11/Chromium does not accept a key chord addressed to an unfocused
+ /// renderer without briefly moving focus, which background delivery forbids.
+ ChromiumHotkey,
}
impl BackgroundUnavailable {
fn code(self) -> &'static str {
match self {
Self::NoLibeiBackend => "background_unavailable",
+ Self::ChromiumHotkey => "background_unavailable",
}
}
fn detail(self) -> &'static str {
match self {
- Self::NoLibeiBackend =>
+ Self::NoLibeiBackend => {
"no libei input backend on this Wayland compositor (built without \
portal-libei, or the xdg-desktop-portal RemoteDesktop session was \
- unavailable/denied): synthetic input has no actuator",
+ unavailable/denied): synthetic input has no actuator"
+ }
+ Self::ChromiumHotkey => {
+ "Chromium/Electron does not accept a key chord addressed to an \
+ unfocused renderer through X11 background injection"
+ }
}
}
}
@@ -114,7 +125,9 @@ impl BackgroundUnavailable {
/// Build the structured `background_unavailable` error returned when a
/// `delivery_mode:"background"` injection has no Wayland backend. Mirrors the
/// Windows silent-drop error so callers branch on `code` identically.
-pub fn background_unavailable_error(reason: BackgroundUnavailable) -> cua_driver_core::protocol::ToolResult {
+pub fn background_unavailable_error(
+ reason: BackgroundUnavailable,
+) -> cua_driver_core::protocol::ToolResult {
let detail = reason.detail();
cua_driver_core::protocol::ToolResult::error(format!(
"Background delivery is not available: {detail}. Either call bring_to_front \
@@ -137,26 +150,52 @@ mod tests {
#[test]
fn delivery_mode_parses_known_values() {
let j = |s: &str| serde_json::json!({"delivery_mode": s});
- assert_eq!(DeliveryMode::from_args(&j("background")), DeliveryMode::Background);
- assert_eq!(DeliveryMode::from_args(&j("foreground")), DeliveryMode::Foreground);
+ assert_eq!(
+ DeliveryMode::from_args(&j("background")),
+ DeliveryMode::Background
+ );
+ assert_eq!(
+ DeliveryMode::from_args(&j("foreground")),
+ DeliveryMode::Foreground
+ );
// Case-insensitive, matching macOS / Windows.
- assert_eq!(DeliveryMode::from_args(&j("Foreground")), DeliveryMode::Foreground);
+ assert_eq!(
+ DeliveryMode::from_args(&j("Foreground")),
+ DeliveryMode::Foreground
+ );
}
#[test]
fn delivery_mode_defaults_to_background() {
// Missing field, garbage value, null, and the removed legacy "auto" all
// resolve to Background — the no-foreground-by-default contract.
- assert_eq!(DeliveryMode::from_args(&serde_json::json!({})), DeliveryMode::Background);
- assert_eq!(DeliveryMode::from_args(&serde_json::json!({"delivery_mode": "garbage"})), DeliveryMode::Background);
- assert_eq!(DeliveryMode::from_args(&serde_json::json!({"delivery_mode": "auto"})), DeliveryMode::Background);
- assert_eq!(DeliveryMode::from_args(&serde_json::json!({"delivery_mode": null})), DeliveryMode::Background);
+ assert_eq!(
+ DeliveryMode::from_args(&serde_json::json!({})),
+ DeliveryMode::Background
+ );
+ assert_eq!(
+ DeliveryMode::from_args(&serde_json::json!({"delivery_mode": "garbage"})),
+ DeliveryMode::Background
+ );
+ assert_eq!(
+ DeliveryMode::from_args(&serde_json::json!({"delivery_mode": "auto"})),
+ DeliveryMode::Background
+ );
+ assert_eq!(
+ DeliveryMode::from_args(&serde_json::json!({"delivery_mode": null})),
+ DeliveryMode::Background
+ );
}
#[test]
fn delivery_mode_schema_advertises_two_modes() {
let s = delivery_mode_schema();
- let en: Vec<&str> = s["enum"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect();
+ let en: Vec<&str> = s["enum"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .filter_map(|v| v.as_str())
+ .collect();
assert_eq!(en, vec!["background", "foreground"]);
assert_eq!(s["default"], "background");
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
index 0e0d771ff8..6f5ced3118 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
@@ -16,6 +16,8 @@
pub mod delivery;
use anyhow::{anyhow, bail, Context, Result};
+use evdev::uinput::VirtualDevice;
+use evdev::{AttributeSet, EventType, InputEvent, Key, RelativeAxisType};
use std::collections::HashMap;
use std::ffi::{CStr, CString};
use std::fs;
@@ -24,8 +26,6 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::thread::sleep;
use std::time::Duration;
-use evdev::uinput::VirtualDevice;
-use evdev::{AttributeSet, EventType, InputEvent, Key, RelativeAxisType};
use x11rb::connection::Connection;
use x11rb::protocol::xproto::*;
use x11rb::rust_connection::RustConnection;
@@ -65,7 +65,12 @@ fn path_cumulative(path: &[(i32, i32)]) -> (Vec<f64>, f64) {
/// waypoints. Evaluated with meval (sin/cos/^/etc.); non-finite outputs
/// (ln of a negative, 1/0, …) are dropped. Errors on a bad expression or
/// fewer than 2 finite points.
-pub fn sample_function(expr: &str, x_from: f64, x_to: f64, samples: u64) -> Result<Vec<(f64, f64)>> {
+pub fn sample_function(
+ expr: &str,
+ x_from: f64,
+ x_to: f64,
+ samples: u64,
+) -> Result<Vec<(f64, f64)>> {
let parsed: meval::Expr = expr
.parse()
.map_err(|e| anyhow!("invalid fn '{expr}': {e}"))?;
@@ -93,10 +98,11 @@ fn point_on_path(path: &[(i32, i32)], cum: &[f64], total: f64, t: f64) -> (i32,
return *path.last().unwrap();
}
let d = t.clamp(0.0, 1.0) * total;
- let mut i = match cum.binary_search_by(|v| v.partial_cmp(&d).unwrap_or(std::cmp::Ordering::Less)) {
- Ok(i) => i,
- Err(i) => i.saturating_sub(1),
- };
+ let mut i =
+ match cum.binary_search_by(|v| v.partial_cmp(&d).unwrap_or(std::cmp::Ordering::Less)) {
+ Ok(i) => i,
+ Err(i) => i.saturating_sub(1),
+ };
if i >= path.len() - 1 {
i = path.len() - 2;
}
@@ -110,12 +116,13 @@ fn point_on_path(path: &[(i32, i32)], cum: &[f64], total: f64, t: f64) -> (i32,
#[derive(Clone, Copy, Debug)]
struct MasterPointerIds {
pointer_id: i32,
- keyboard_id: i32,
- slave_pointer_id: i32,
+ _keyboard_id: i32,
+ _slave_pointer_id: i32,
}
static MPX_POINTERS: OnceLock<Mutex<HashMap<String, MasterPointerIds>>> = OnceLock::new();
-static UINPUT_POINTERS: OnceLock<Mutex<HashMap<String, Arc<Mutex<VirtualDevice>>>>> = OnceLock::new();
+static UINPUT_POINTERS: OnceLock<Mutex<HashMap<String, Arc<Mutex<VirtualDevice>>>>> =
+ OnceLock::new();
static XLIB_THREADS_READY: OnceLock<Result<(), String>> = OnceLock::new();
static MPX_NAME_COUNTER: AtomicU64 = AtomicU64::new(1);
@@ -163,11 +170,10 @@ fn open_display() -> Result<*mut x11::xlib::Display> {
Ok(display)
}
-fn xi2_query_devices(
- display: *mut x11::xlib::Display,
-) -> Result<Vec<(i32, i32, String)>> {
+fn xi2_query_devices(display: *mut x11::xlib::Display) -> Result<Vec<(i32, i32, String)>> {
let mut count = 0;
- let ptr = unsafe { x11::xinput2::XIQueryDevice(display, x11::xinput2::XIAllDevices, &mut count) };
+ let ptr =
+ unsafe { x11::xinput2::XIQueryDevice(display, x11::xinput2::XIAllDevices, &mut count) };
if ptr.is_null() {
bail!("XIQueryDevice returned null");
}
@@ -177,7 +183,9 @@ fn xi2_query_devices(
let name = if info.name.is_null() {
String::new()
} else {
- unsafe { CStr::from_ptr(info.name) }.to_string_lossy().into_owned()
+ unsafe { CStr::from_ptr(info.name) }
+ .to_string_lossy()
+ .into_owned()
};
out.push((info.deviceid, info._use, name));
}
@@ -190,7 +198,9 @@ fn x_server_vendor(display: *mut x11::xlib::Display) -> String {
if ptr.is_null() {
return String::new();
}
- unsafe { CStr::from_ptr(ptr) }.to_string_lossy().into_owned()
+ unsafe { CStr::from_ptr(ptr) }
+ .to_string_lossy()
+ .into_owned()
}
fn supports_parallel_pointer_injection(display: *mut x11::xlib::Display) -> Result<()> {
@@ -308,7 +318,8 @@ pub fn real_pointer_input_available() -> bool {
let Ok(display) = open_display() else {
return false;
};
- let supported = supports_parallel_pointer_injection(display).is_ok() && !is_xvfb_process_running();
+ let supported =
+ supports_parallel_pointer_injection(display).is_ok() && !is_xvfb_process_running();
unsafe { x11::xlib::XCloseDisplay(display) };
supported
}
@@ -363,8 +374,10 @@ fn ensure_master_pointer(cursor_id: &str) -> Result<MasterPointerIds> {
}
}
- let pointer_id = pointer_id.ok_or_else(|| anyhow!("failed to locate created master pointer for '{cursor_id}'"))?;
- let keyboard_id = keyboard_id.ok_or_else(|| anyhow!("failed to locate created master keyboard for '{cursor_id}'"))?;
+ let pointer_id = pointer_id
+ .ok_or_else(|| anyhow!("failed to locate created master pointer for '{cursor_id}'"))?;
+ let keyboard_id = keyboard_id
+ .ok_or_else(|| anyhow!("failed to locate created master keyboard for '{cursor_id}'"))?;
let device_name = slave_pointer_name(&base);
let uinput_device = create_uinput_pointer(&device_name)?;
@@ -373,8 +386,15 @@ fn ensure_master_pointer(cursor_id: &str) -> Result<MasterPointerIds> {
set_flat_pointer_accel(display, slave_pointer_id);
unsafe { x11::xlib::XCloseDisplay(display) };
- let ids = MasterPointerIds { pointer_id, keyboard_id, slave_pointer_id };
- mpx_pointers().lock().unwrap().insert(cursor_id.to_owned(), ids);
+ let ids = MasterPointerIds {
+ pointer_id,
+ _keyboard_id: keyboard_id,
+ _slave_pointer_id: slave_pointer_id,
+ };
+ mpx_pointers()
+ .lock()
+ .unwrap()
+ .insert(cursor_id.to_owned(), ids);
uinput_pointers()
.lock()
.unwrap()
@@ -407,7 +427,9 @@ pub fn forget_master_pointer(cursor_id: &str) {
}
}
- let (Some(return_pointer), Some(return_keyboard)) = (virtual_core_pointer, virtual_core_keyboard) else {
+ let (Some(return_pointer), Some(return_keyboard)) =
+ (virtual_core_pointer, virtual_core_keyboard)
+ else {
unsafe { x11::xlib::XCloseDisplay(display) };
return;
};
@@ -441,13 +463,11 @@ fn create_uinput_pointer(name: &str) -> Result<VirtualDevice> {
rel_axes.insert(RelativeAxisType::REL_WHEEL);
rel_axes.insert(RelativeAxisType::REL_HWHEEL);
- Ok(
- evdev::uinput::VirtualDeviceBuilder::new()?
- .name(name)
- .with_keys(&keys)?
- .with_relative_axes(&rel_axes)?
- .build()?,
- )
+ Ok(evdev::uinput::VirtualDeviceBuilder::new()?
+ .name(name)
+ .with_keys(&keys)?
+ .with_relative_axes(&rel_axes)?
+ .build()?)
}
fn wait_for_slave_pointer_id(display: *mut x11::xlib::Display, device_name: &str) -> Result<i32> {
@@ -465,7 +485,11 @@ fn wait_for_slave_pointer_id(display: *mut x11::xlib::Display, device_name: &str
}
}
-fn attach_slave_to_master(display: *mut x11::xlib::Display, slave_pointer_id: i32, master_pointer_id: i32) -> Result<()> {
+fn attach_slave_to_master(
+ display: *mut x11::xlib::Display,
+ slave_pointer_id: i32,
+ master_pointer_id: i32,
+) -> Result<()> {
let mut change = x11::xinput2::XIAnyHierarchyChangeInfo::default();
unsafe {
let attach = change.attach();
@@ -537,7 +561,12 @@ fn set_flat_pointer_accel(display: *mut x11::xlib::Display, slave_pointer_id: i3
}
}
-fn warp_master_pointer(display: *mut x11::xlib::Display, ids: MasterPointerIds, x: i32, y: i32) -> Result<()> {
+fn warp_master_pointer(
+ display: *mut x11::xlib::Display,
+ ids: MasterPointerIds,
+ x: i32,
+ y: i32,
+) -> Result<()> {
let root = unsafe { x11::xlib::XDefaultRootWindow(display) };
let rc = unsafe {
x11::xinput2::XIWarpPointer(
@@ -622,7 +651,7 @@ fn install_shield_grab(
device_id,
button as std::os::raw::c_int,
window,
- 0, // cursor: None
+ 0, // cursor: None
x11::xinput2::XIGrabModeSync, // freeze the pointer on press
x11::xinput2::XIGrabModeAsync, // leave the paired keyboard alone
x11::xlib::False, // owner_events: deliver to us
@@ -638,14 +667,26 @@ fn install_shield_grab(
Ok(())
}
-fn remove_shield_grab(display: *mut x11::xlib::Display, device_id: i32, window: x11::xlib::Window, button: u8) {
+fn remove_shield_grab(
+ display: *mut x11::xlib::Display,
+ device_id: i32,
+ window: x11::xlib::Window,
+ button: u8,
+) {
let mut mods = x11::xinput2::XIGrabModifiers {
modifiers: XI_ANY_MODIFIER,
status: 0,
};
unsafe {
let prev = x11::xlib::XSetErrorHandler(Some(ignore_x_error));
- x11::xinput2::XIUngrabButton(display, device_id, button as std::os::raw::c_int, window, 1, &mut mods);
+ x11::xinput2::XIUngrabButton(
+ display,
+ device_id,
+ button as std::os::raw::c_int,
+ window,
+ 1,
+ &mut mods,
+ );
x11::xlib::XSync(display, 0);
x11::xlib::XSetErrorHandler(prev);
}
@@ -687,7 +728,12 @@ fn replay_shielded_presses(
let time = unsafe { (*de).time };
if pending_devices.remove(&device_id) {
unsafe {
- x11::xinput2::XIAllowEvents(display, device_id, x11::xinput2::XIReplayDevice, time);
+ x11::xinput2::XIAllowEvents(
+ display,
+ device_id,
+ x11::xinput2::XIReplayDevice,
+ time,
+ );
x11::xlib::XSync(display, 0);
}
}
@@ -698,11 +744,7 @@ fn replay_shielded_presses(
fn ewmh_active_window(display: *mut x11::xlib::Display) -> Option<x11::xlib::Window> {
unsafe {
- let atom = x11::xlib::XInternAtom(
- display,
- c"_NET_ACTIVE_WINDOW".as_ptr(),
- x11::xlib::True,
- );
+ let atom = x11::xlib::XInternAtom(display, c"_NET_ACTIVE_WINDOW".as_ptr(), x11::xlib::True);
if atom == 0 {
return None;
}
@@ -761,12 +803,8 @@ fn x_server_time(display: *mut x11::xlib::Display) -> x11::xlib::Time {
x11::xlib::XSync(display, 0);
let mut time: x11::xlib::Time = x11::xlib::CurrentTime;
let mut ev: x11::xlib::XEvent = std::mem::zeroed();
- while x11::xlib::XCheckWindowEvent(
- display,
- win,
- x11::xlib::PropertyChangeMask,
- &mut ev,
- ) != 0
+ while x11::xlib::XCheckWindowEvent(display, win, x11::xlib::PropertyChangeMask, &mut ev)
+ != 0
{
if ev.get_type() == x11::xlib::PropertyNotify {
time = ev.property.time;
@@ -784,11 +822,7 @@ fn ewmh_activate_window(
current_active: x11::xlib::Window,
) {
unsafe {
- let atom = x11::xlib::XInternAtom(
- display,
- c"_NET_ACTIVE_WINDOW".as_ptr(),
- x11::xlib::True,
- );
+ let atom = x11::xlib::XInternAtom(display, c"_NET_ACTIVE_WINDOW".as_ptr(), x11::xlib::True);
if atom == 0 {
return;
}
@@ -799,7 +833,8 @@ fn ewmh_activate_window(
ev.message_type = atom;
ev.format = 32;
ev.data.set_long(0, 2); // source indication: pager/tool
- ev.data.set_long(1, x_server_time(display) as std::os::raw::c_long);
+ ev.data
+ .set_long(1, x_server_time(display) as std::os::raw::c_long);
ev.data.set_long(2, current_active as std::os::raw::c_long);
x11::xlib::XSendEvent(
display,
@@ -835,7 +870,9 @@ pub fn with_x11_foreground<T>(
}
let prior = ewmh_active_window(display);
ewmh_activate_window(display, xid as x11::xlib::Window, prior.unwrap_or(0));
- unsafe { x11::xlib::XSync(display, 0); }
+ unsafe {
+ x11::xlib::XSync(display, 0);
+ }
if settle_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(settle_ms));
}
@@ -863,9 +900,13 @@ pub fn with_x11_foreground<T>(
// Restore the prior active window (brief swap, like macOS/Windows).
if let Some(p) = prior {
ewmh_activate_window(display, p, xid as x11::xlib::Window);
- unsafe { x11::xlib::XSync(display, 0); }
+ unsafe {
+ x11::xlib::XSync(display, 0);
+ }
+ }
+ unsafe {
+ x11::xlib::XCloseDisplay(display);
}
- unsafe { x11::xlib::XCloseDisplay(display); }
result
}
@@ -876,11 +917,21 @@ pub fn with_x11_foreground<T>(
pub fn x11_activate_window_persistent(xid: u64) -> Result<Option<u64>> {
let display = unsafe { x11::xlib::XOpenDisplay(ptr::null()) };
if display.is_null() {
- bail!("cannot activate window: no X display (DISPLAY={:?})", std::env::var("DISPLAY").ok());
+ bail!(
+ "cannot activate window: no X display (DISPLAY={:?})",
+ std::env::var("DISPLAY").ok()
+ );
}
let prior = ewmh_active_window(display).map(|w| w as u64);
- ewmh_activate_window(display, xid as x11::xlib::Window, prior.unwrap_or(0) as x11::xlib::Window);
- unsafe { x11::xlib::XSync(display, 0); x11::xlib::XCloseDisplay(display); }
+ ewmh_activate_window(
+ display,
+ xid as x11::xlib::Window,
+ prior.unwrap_or(0) as x11::xlib::Window,
+ );
+ unsafe {
+ x11::xlib::XSync(display, 0);
+ x11::xlib::XCloseDisplay(display);
+ }
Ok(prior)
}
@@ -895,17 +946,29 @@ fn button_code(button: u8) -> Result<Key> {
fn emit_button(device: &mut VirtualDevice, button: u8, press: bool) -> Result<()> {
let code = button_code(button)?;
- device.emit(&[InputEvent::new(EventType::KEY, code.0, if press { 1 } else { 0 })])?;
+ device.emit(&[InputEvent::new(
+ EventType::KEY,
+ code.0,
+ if press { 1 } else { 0 },
+ )])?;
Ok(())
}
fn emit_relative_motion(device: &mut VirtualDevice, dx: i32, dy: i32) -> Result<()> {
let mut events = Vec::with_capacity(2);
if dx != 0 {
- events.push(InputEvent::new(EventType::RELATIVE, RelativeAxisType::REL_X.0, dx));
+ events.push(InputEvent::new(
+ EventType::RELATIVE,
+ RelativeAxisType::REL_X.0,
+ dx,
+ ));
}
if dy != 0 {
- events.push(InputEvent::new(EventType::RELATIVE, RelativeAxisType::REL_Y.0, dy));
+ events.push(InputEvent::new(
+ EventType::RELATIVE,
+ RelativeAxisType::REL_Y.0,
+ dy,
+ ));
}
if events.is_empty() {
return Ok(());
@@ -927,9 +990,7 @@ fn emit_scroll(device: &mut VirtualDevice, horizontal: bool, value: i32) -> Resu
Ok(())
}
-pub fn send_parallel_virtual_pointer_drags(
- drags: &[(String, VirtualPointerDrag)],
-) -> Result<()> {
+pub fn send_parallel_virtual_pointer_drags(drags: &[(String, VirtualPointerDrag)]) -> Result<()> {
let display = open_display()?;
supports_parallel_pointer_injection(display)?;
let xi_opcode = xinput_opcode(display);
@@ -1160,9 +1221,8 @@ pub fn send_virtual_pointer_click(cursor_id: &str, click: &VirtualPointerClick)
.get(cursor_id)
.cloned()
.ok_or_else(|| anyhow!("missing uinput pointer for '{cursor_id}'"))?;
- let opcode = xi_opcode.ok_or_else(|| {
- anyhow!("no-focus-steal click requires XInput/XI2 shield grabs")
- })?;
+ let opcode = xi_opcode
+ .ok_or_else(|| anyhow!("no-focus-steal click requires XInput/XI2 shield grabs"))?;
let window = click.target_window as x11::xlib::Window;
install_shield_grab(display, ids.pointer_id, window, click.button)
@@ -1332,8 +1392,7 @@ fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) {
if attempt >= 2 {
if let Some(now_win) = now {
unsafe {
- let prev_handler =
- x11::xlib::XSetErrorHandler(Some(ignore_x_error));
+ let prev_handler = x11::xlib::XSetErrorHandler(Some(ignore_x_error));
x11::xlib::XSetInputFocus(
display,
now_win,
@@ -1569,7 +1628,11 @@ pub fn send_drag(
let (conn, _) = connect_x11_for_input()?;
let root = conn.setup().roots[0].root;
let steps = steps.max(1);
- let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };
+ let step_delay_ms = if steps > 1 {
+ duration_ms / steps as u64
+ } else {
+ duration_ms
+ };
let press_target = resolve_event_target(&conn, xid, from_x, from_y)?;
// ButtonPress at start.
@@ -1578,9 +1641,13 @@ pub fn send_drag(
detail: button,
sequence: 0,
time: x11rb::CURRENT_TIME,
- root, event: press_target.window, child: x11rb::NONE,
- root_x: press_target.root_x, root_y: press_target.root_y,
- event_x: press_target.local_x, event_y: press_target.local_y,
+ root,
+ event: press_target.window,
+ child: x11rb::NONE,
+ root_x: press_target.root_x,
+ root_y: press_target.root_y,
+ event_x: press_target.local_x,
+ event_y: press_target.local_y,
state: KeyButMask::from(0u16),
same_screen: true,
};
@@ -1599,9 +1666,13 @@ pub fn send_drag(
detail: Motion::NORMAL,
sequence: 0,
time: x11rb::CURRENT_TIME,
- root, event: target.window, child: x11rb::NONE,
- root_x: target.root_x, root_y: target.root_y,
- event_x: target.local_x, event_y: target.local_y,
+ root,
+ event: target.window,
+ child: x11rb::NONE,
+ root_x: target.root_x,
+ root_y: target.root_y,
+ event_x: target.local_x,
+ event_y: target.local_y,
state: button_state_mask(button),
same_screen: true,
};
@@ -1619,13 +1690,22 @@ pub fn send_drag(
detail: button,
sequence: 0,
time: x11rb::CURRENT_TIME,
- root, event: release_target.window, child: x11rb::NONE,
- root_x: release_target.root_x, root_y: release_target.root_y,
- event_x: release_target.local_x, event_y: release_target.local_y,
+ root,
+ event: release_target.window,
+ child: x11rb::NONE,
+ root_x: release_target.root_x,
+ root_y: release_target.root_y,
+ event_x: release_target.local_x,
+ event_y: release_target.local_y,
state: button_state_mask(button),
same_screen: true,
};
- conn.send_event(false, release_target.window, EventMask::BUTTON_RELEASE, &release)?;
+ conn.send_event(
+ false,
+ release_target.window,
+ EventMask::BUTTON_RELEASE,
+ &release,
+ )?;
conn.flush()?;
Ok(())
}
@@ -1670,7 +1750,9 @@ pub fn send_motion(xid: u64, x: i32, y: i32, button: Option<u8>) -> Result<()> {
root_y: target.root_y,
event_x: target.local_x,
event_y: target.local_y,
- state: button.map(button_state_mask).unwrap_or_else(|| KeyButMask::from(0u16)),
+ state: button
+ .map(button_state_mask)
+ .unwrap_or_else(|| KeyButMask::from(0u16)),
same_screen: true,
};
conn.send_event(false, target.window, EventMask::POINTER_MOTION, &motion)?;
@@ -1721,15 +1803,24 @@ pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Re
let Some((keycode, needs_shift)) = char_to_keycode_shift(&mapping, ch as u32) else {
continue;
};
- let state = if needs_shift { KeyButMask::SHIFT } else { KeyButMask::from(0u16) };
+ let state = if needs_shift {
+ KeyButMask::SHIFT
+ } else {
+ KeyButMask::from(0u16)
+ };
let press = KeyPressEvent {
response_type: KEY_PRESS_EVENT,
detail: keycode,
sequence: 0,
time: x11rb::CURRENT_TIME,
- root, event: window, child: x11rb::NONE,
- root_x: 0, root_y: 0, event_x: 0, event_y: 0,
+ root,
+ event: window,
+ child: x11rb::NONE,
+ root_x: 0,
+ root_y: 0,
+ event_x: 0,
+ event_y: 0,
state,
same_screen: true,
};
@@ -1738,8 +1829,13 @@ pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Re
detail: keycode,
sequence: 0,
time: x11rb::CURRENT_TIME,
- root, event: window, child: x11rb::NONE,
- root_x: 0, root_y: 0, event_x: 0, event_y: 0,
+ root,
+ event: window,
+ child: x11rb::NONE,
+ root_x: 0,
+ root_y: 0,
+ event_x: 0,
+ event_y: 0,
state,
same_screen: true,
};
@@ -1937,8 +2033,28 @@ pub fn send_click_xtest_desktop(x: i32, y: i32, button: u8, count: usize) -> Res
/// Send a named key press to a window.
pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {
+ send_key_to_target(xid, None, key, modifiers)
+}
+
+/// Send a named key to the deepest child at window-local coordinates without
+/// activating the window. Coordinate keyboard actions use this so embedded
+/// Chromium renderers receive the event on their input surface rather than on
+/// the native top-level wrapper.
+pub fn send_key_at(xid: u64, x: i32, y: i32, key: &str, modifiers: &[&str]) -> Result<()> {
+ send_key_to_target(xid, Some((x, y)), key, modifiers)
+}
+
+fn send_key_to_target(
+ xid: u64,
+ point: Option<(i32, i32)>,
+ key: &str,
+ modifiers: &[&str],
+) -> Result<()> {
let (conn, _) = connect_x11_for_input()?;
- let window = xid as u32;
+ let target = point
+ .map(|(x, y)| resolve_event_target(&conn, xid, x, y))
+ .transpose()?;
+ let window = target.map(|target| target.window).unwrap_or(xid as u32);
let root = conn.setup().roots[0].root;
// Resolve the named key to a keysym, then to a keycode. On sparse/headless
@@ -1949,39 +2065,66 @@ pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {
let keysym = key_name_to_keysym(key)?;
let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?;
let (keycode, remap_guard) = keycode_for_keysym(&conn, &mapping, keysym, key)?;
- let state = modifiers_to_state(modifiers);
- let press = KeyPressEvent {
- response_type: KEY_PRESS_EVENT,
- detail: keycode,
- sequence: 0,
- time: x11rb::CURRENT_TIME,
- root,
- event: window,
- child: x11rb::NONE,
- root_x: 0, root_y: 0,
- event_x: 0, event_y: 0,
- state,
- same_screen: true,
- };
+ // XSendEvent's state mask describes modifiers for one key event, but does
+ // not update Chromium's internal modifier state by itself. Emit the
+ // modifier transitions as part of the background chord so web handlers see
+ // the same ordered sequence as physical input without activating the
+ // target window.
+ let mut remap_guards = Vec::new();
+ let mut modifier_keycodes = Vec::new();
+ for modifier in modifiers {
+ let modifier_keysym = key_name_to_keysym(modifier)?;
+ let (modifier_keycode, guard) =
+ keycode_for_keysym(&conn, &mapping, modifier_keysym, modifier)?;
+ if let Some(guard) = guard {
+ remap_guards.push(guard);
+ }
+ modifier_keycodes.push((modifier_keycode, modifiers_to_state(&[*modifier])));
+ }
- let release = KeyReleaseEvent {
- response_type: KEY_RELEASE_EVENT,
- detail: keycode,
- sequence: 0,
- time: x11rb::CURRENT_TIME,
- root,
- event: window,
- child: x11rb::NONE,
- root_x: 0, root_y: 0,
- event_x: 0, event_y: 0,
- state,
- same_screen: true,
+ let send_key_event = |response_type, detail, state, event_mask| {
+ let event = KeyPressEvent {
+ response_type,
+ detail,
+ sequence: 0,
+ time: x11rb::CURRENT_TIME,
+ root,
+ event: window,
+ child: x11rb::NONE,
+ root_x: target.map(|target| target.root_x).unwrap_or(0),
+ root_y: target.map(|target| target.root_y).unwrap_or(0),
+ event_x: target.map(|target| target.local_x).unwrap_or(0),
+ event_y: target.map(|target| target.local_y).unwrap_or(0),
+ state,
+ same_screen: true,
+ };
+ conn.send_event(false, window, event_mask, &event)
};
- conn.send_event(false, window, EventMask::KEY_PRESS, &press)?;
+ let mut state_bits = 0u16;
+ for &(modifier_keycode, modifier_mask) in &modifier_keycodes {
+ send_key_event(
+ KEY_PRESS_EVENT,
+ modifier_keycode,
+ KeyButMask::from(state_bits),
+ EventMask::KEY_PRESS,
+ )?;
+ state_bits |= u16::from(modifier_mask);
+ }
+ let state = KeyButMask::from(state_bits);
+ send_key_event(KEY_PRESS_EVENT, keycode, state, EventMask::KEY_PRESS)?;
sleep(Duration::from_millis(KEY_DELAY_MS));
- conn.send_event(false, window, EventMask::KEY_RELEASE, &release)?;
+ send_key_event(KEY_RELEASE_EVENT, keycode, state, EventMask::KEY_RELEASE)?;
+ for &(modifier_keycode, modifier_mask) in modifier_keycodes.iter().rev() {
+ send_key_event(
+ KEY_RELEASE_EVENT,
+ modifier_keycode,
+ KeyButMask::from(state_bits),
+ EventMask::KEY_RELEASE,
+ )?;
+ state_bits &= !u16::from(modifier_mask);
+ }
conn.flush()?;
// If we borrowed a spare keycode for this keysym, give the target client a
@@ -1990,7 +2133,7 @@ pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {
// eagerly would race delivery. A server round-trip (which only returns once
// our queued requests have been processed) plus a short settle keeps that
// race closed; the guard then reinstates the original keysyms on drop.
- if remap_guard.is_some() {
+ if remap_guard.is_some() || !remap_guards.is_empty() {
let _ = conn.get_input_focus()?.reply();
sleep(Duration::from_millis(KEY_DELAY_MS));
}
@@ -2039,19 +2182,41 @@ fn key_name_to_keysym(key: &str) -> Result<u32> {
"down" => 0xFF54,
"left" => 0xFF51,
"right" => 0xFF53,
- "f1" => 0xFFBE, "f2" => 0xFFBF, "f3" => 0xFFC0, "f4" => 0xFFC1,
- "f5" => 0xFFC2, "f6" => 0xFFC3, "f7" => 0xFFC4, "f8" => 0xFFC5,
- "f9" => 0xFFC6, "f10" => 0xFFC7, "f11" => 0xFFC8, "f12" => 0xFFC9,
- "shift" => 0xFFE1, "ctrl" | "control" => 0xFFE3, "alt" => 0xFFE9, "super" | "meta" | "win" => 0xFFEB,
- "capslock" => 0xFFE5, "numlock" => 0xFF7F,
+ "f1" => 0xFFBE,
+ "f2" => 0xFFBF,
+ "f3" => 0xFFC0,
+ "f4" => 0xFFC1,
+ "f5" => 0xFFC2,
+ "f6" => 0xFFC3,
+ "f7" => 0xFFC4,
+ "f8" => 0xFFC5,
+ "f9" => 0xFFC6,
+ "f10" => 0xFFC7,
+ "f11" => 0xFFC8,
+ "f12" => 0xFFC9,
+ "shift" => 0xFFE1,
+ "ctrl" | "control" => 0xFFE3,
+ "alt" => 0xFFE9,
+ "super" | "meta" | "win" => 0xFFEB,
+ "capslock" => 0xFFE5,
+ "numlock" => 0xFF7F,
// Common X keysym names for punctuation. The single-char branch below
// already resolves the literal glyph ("+", "=", "*", "/"), but callers
// that speak the X keysym-name vocabulary may pass the spelled-out name.
// For the ASCII range the keysym value equals the codepoint.
- "plus" => 0x2B, "minus" | "dash" => 0x2D, "equal" | "equals" => 0x3D,
- "asterisk" | "star" => 0x2A, "slash" => 0x2F, "backslash" => 0x5C,
- "period" | "dot" => 0x2E, "comma" => 0x2C, "semicolon" => 0x3B,
- "colon" => 0x3A, "underscore" => 0x5F, "parenleft" => 0x28, "parenright" => 0x29,
+ "plus" => 0x2B,
+ "minus" | "dash" => 0x2D,
+ "equal" | "equals" => 0x3D,
+ "asterisk" | "star" => 0x2A,
+ "slash" => 0x2F,
+ "backslash" => 0x5C,
+ "period" | "dot" => 0x2E,
+ "comma" => 0x2C,
+ "semicolon" => 0x3B,
+ "colon" => 0x3A,
+ "underscore" => 0x5F,
+ "parenleft" => 0x28,
+ "parenright" => 0x29,
s if s.len() == 1 => s.chars().next().unwrap() as u32,
_ => anyhow::bail!("Unknown key: {key}"),
};
@@ -2185,7 +2350,10 @@ pub fn inject_tk_send(text: &str) -> Result<bool> {
// Tcl's `send` command: `send <target-app-name> <tcl-command>`.
// We target "cua-tk-target" (the name the test app registers with) and
// insert at the entry widget's current cursor position.
- let tcl_text = text.replace("\\", "\\\\").replace("{", "\\{").replace("}", "\\}");
+ let tcl_text = text
+ .replace("\\", "\\\\")
+ .replace("{", "\\{")
+ .replace("}", "\\}");
// Tk's `send` is synchronous: it blocks the sender until the *target's* Tcl
// event loop services the request and replies. If the target is wedged, or
@@ -2220,10 +2388,11 @@ exit 0"#,
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
- .spawn() {
- Ok(c) => c,
- Err(_) => return Ok(false),
- };
+ .spawn()
+ {
+ Ok(c) => c,
+ Err(_) => return Ok(false),
+ };
if let Some(mut stdin) = child.stdin.take() {
// Ignore write errors: if wish already exited we observe it via wait().
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
index 2aad60f3c0..80544f5157 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
@@ -21,12 +21,12 @@ use std::collections::{HashMap, HashSet};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
+#[cfg(target_os = "linux")]
+use cursor_overlay::ZOrderEnforcer;
use cursor_overlay::{
CursorConfig, CursorKey, KeyedOverlayCommand, OverlayCommand, OverlayMsg, Palette,
RenderStateCore,
};
-#[cfg(target_os = "linux")]
-use cursor_overlay::ZOrderEnforcer;
// ── Global channel ────────────────────────────────────────────────────────
@@ -127,7 +127,10 @@ pub fn send_command_for(key: CursorKey, cmd: OverlayCommand) {
if key.is_empty() {
return;
}
- let msg = OverlayMsg::Cmd(KeyedOverlayCommand { key: key.clone(), cmd: cmd.clone() });
+ let msg = OverlayMsg::Cmd(KeyedOverlayCommand {
+ key: key.clone(),
+ cmd: cmd.clone(),
+ });
if let Some(tx) = CMD_TX.get() {
let _ = tx.try_send(msg.clone());
}
@@ -163,7 +166,9 @@ pub fn is_enabled() -> bool {
}
pub fn is_enabled_for(key: &str) -> bool {
- RENDER.lock().ok()
+ RENDER
+ .lock()
+ .ok()
.and_then(|g| {
g.as_ref().and_then(|m| {
m.cursors
@@ -180,15 +185,23 @@ pub fn current_position() -> (f64, f64) {
}
pub fn current_position_for(key: &str) -> (f64, f64) {
- RENDER.lock().ok()
- .and_then(|g| g.as_ref().and_then(|m| m.cursors.get(key)).map(|rs| rs.core.pos))
+ RENDER
+ .lock()
+ .ok()
+ .and_then(|g| {
+ g.as_ref()
+ .and_then(|m| m.cursors.get(key))
+ .map(|rs| rs.core.pos)
+ })
.unwrap_or((-200.0, -200.0))
}
fn seed_start_if_sentinel(key: &CursorKey, target_x: f64, target_y: f64) -> bool {
const SEED_OFFSET: f64 = 140.0;
let mut guard = RENDER.lock().unwrap();
- let Some(map) = guard.as_mut() else { return false };
+ let Some(map) = guard.as_mut() else {
+ return false;
+ };
if map.ended.contains(key) {
return false;
}
@@ -236,11 +249,14 @@ pub async fn animate_cursor_to_for(key: CursorKey, x: f64, y: f64) {
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
arrival_register(key.clone(), tx);
- send_command_for(key, OverlayCommand::MoveTo {
- x,
- y,
- end_heading_radians: std::f64::consts::FRAC_PI_4,
- });
+ send_command_for(
+ key,
+ OverlayCommand::MoveTo {
+ x,
+ y,
+ end_heading_radians: std::f64::consts::FRAC_PI_4,
+ },
+ );
let _ = rx.await;
}
@@ -330,10 +346,10 @@ impl RenderState {
fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMsg>) {
use x11rb::connection::Connection;
use x11rb::protocol::shape::{ConnectionExt as ShapeConnectionExt, SK, SO};
+ use x11rb::protocol::xproto::ConnectionExt as XprotoConnectionExt;
use x11rb::protocol::xproto::{
AtomEnum, ColormapAlloc, CreateWindowAux, EventMask, PropMode, WindowClass,
};
- use x11rb::protocol::xproto::ConnectionExt as XprotoConnectionExt;
use x11rb::wrapper::ConnectionExt as WrapperConnectionExt;
// Connect to X11.
@@ -346,9 +362,9 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
};
let screen = &conn.setup().roots[screen_num];
- let root = screen.root;
- let scr_w = screen.width_in_pixels as u32;
- let scr_h = screen.height_in_pixels as u32;
+ let root = screen.root;
+ let scr_w = screen.width_in_pixels as u32;
+ let scr_h = screen.height_in_pixels as u32;
// Update render state with screen size.
{
@@ -361,13 +377,17 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
// Find 32-bit ARGB visual for compositing.
// Falls back to the default visual if XComposite 32-bit isn't available.
- let (visual_id, depth, colormap) = find_argb_visual(&conn, screen)
- .unwrap_or((screen.root_visual, screen.root_depth, screen.default_colormap));
+ let (visual_id, depth, colormap) = find_argb_visual(&conn, screen).unwrap_or((
+ screen.root_visual,
+ screen.root_depth,
+ screen.default_colormap,
+ ));
// Create a matching colormap if we got a non-default visual.
let colormap = if visual_id != screen.root_visual {
let cm = conn.generate_id().unwrap();
- conn.create_colormap(ColormapAlloc::NONE, cm, root, visual_id).ok();
+ conn.create_colormap(ColormapAlloc::NONE, cm, root, visual_id)
+ .ok();
cm
} else {
colormap
@@ -385,24 +405,32 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
.event_mask(EventMask::NO_EVENT);
conn.create_window(
- depth, win, root,
- 0, 0, scr_w as u16, scr_h as u16,
+ depth,
+ win,
+ root,
+ 0,
+ 0,
+ scr_w as u16,
+ scr_h as u16,
0,
WindowClass::INPUT_OUTPUT,
visual_id,
&win_aux,
- ).ok();
+ )
+ .ok();
// Set window title (identifies our overlay, matches Windows convention).
// `Cua.` namespace mirrors the Windows class-name + install-path
// convention; was `TropeCUA.` (leaked codename from an early C# ref).
let title = format!("Cua.AgentCursorOverlay.{}", cfg.cursor_id);
conn.change_property8(
- PropMode::REPLACE, win,
+ PropMode::REPLACE,
+ win,
AtomEnum::WM_NAME,
AtomEnum::STRING,
title.as_bytes(),
- ).ok();
+ )
+ .ok();
// Make the window fully click-through using the Shape extension (empty input region).
// This is the X11 equivalent of WS_EX_TRANSPARENT on Windows. Note that
@@ -413,9 +441,11 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
SK::INPUT,
x11rb::protocol::xproto::ClipOrdering::UNSORTED,
win,
- 0, 0,
+ 0,
+ 0,
&[],
- ).ok();
+ )
+ .ok();
conn.map_window(win).ok();
conn.flush().ok();
@@ -428,7 +458,7 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
loop {
let now = Instant::now();
- let dt = now.duration_since(last_tick).as_secs_f64().min(0.05);
+ let dt = now.duration_since(last_tick).as_secs_f64().min(0.05);
last_tick = now;
// Drain commands and tick.
@@ -518,7 +548,9 @@ struct X11ZOrderEnforcer<'a, C: x11rb::connection::Connection> {
#[cfg(target_os = "linux")]
impl<'a, C: x11rb::connection::Connection> ZOrderEnforcer for X11ZOrderEnforcer<'a, C> {
fn reassert(&self, target: Option<u64>) {
- use x11rb::protocol::xproto::{ConfigureWindowAux, ConnectionExt as XprotoConnectionExt, StackMode};
+ use x11rb::protocol::xproto::{
+ ConfigureWindowAux, ConnectionExt as XprotoConnectionExt, StackMode,
+ };
// Per the ZOrderEnforcer trait contract, a stale `target` (window
// gone) should fall back to the `None` behavior — top of the
@@ -554,13 +586,15 @@ impl<'a, C: x11rb::connection::Connection> ZOrderEnforcer for X11ZOrderEnforcer<
#[cfg(target_os = "linux")]
fn find_argb_visual(
- conn: &impl x11rb::connection::Connection,
+ _conn: &impl x11rb::connection::Connection,
screen: &x11rb::protocol::xproto::Screen,
) -> Option<(u32, u8, u32)> {
use x11rb::protocol::xproto::VisualClass;
// Walk all depth entries looking for a 32-bit ARGB visual.
for depth_entry in &screen.allowed_depths {
- if depth_entry.depth != 32 { continue; }
+ if depth_entry.depth != 32 {
+ continue;
+ }
for visual in &depth_entry.visuals {
if visual.class == VisualClass::TRUE_COLOR {
return Some((visual.visual_id, 32, screen.default_colormap));
@@ -576,13 +610,16 @@ fn find_argb_visual(
fn paint_x11(
conn: &impl x11rb::connection::Connection,
win: u32,
- w: u32, h: u32,
+ w: u32,
+ h: u32,
depth: u8,
_visual_id: u32,
pm: &tiny_skia::Pixmap,
) {
use x11rb::protocol::xproto::{ConnectionExt as XprotoConnectionExt, CreateGCAux, ImageFormat};
- if pm.width() == 0 || pm.height() == 0 { return; }
+ if pm.width() == 0 || pm.height() == 0 {
+ return;
+ }
// Create a GC for the window if we don't have one.
// (Simplified: we recreate it every frame which is safe but not optimal.)
@@ -591,7 +628,9 @@ fn paint_x11(
Err(_) => return,
};
let gc_aux = CreateGCAux::new();
- if conn.create_gc(gc_id, win, &gc_aux).is_err() { return; }
+ if conn.create_gc(gc_id, win, &gc_aux).is_err() {
+ return;
+ }
// Convert RGBA premult → BGRA premult for X11.
let src = pm.data();
@@ -608,8 +647,10 @@ fn paint_x11(
ImageFormat::Z_PIXMAP,
win,
gc_id,
- w as u16, h as u16,
- 0, 0,
+ w as u16,
+ h as u16,
+ 0,
+ 0,
0,
depth,
&bgra,
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 5f3c058d31..4a5b528569 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -24,7 +24,13 @@ pub struct DriverConfig {
}
impl Default for DriverConfig {
- fn default() -> Self { Self { capture_mode: "ax".into(), capture_scope: "window".into(), max_image_dimension: 1568 } }
+ fn default() -> Self {
+ Self {
+ capture_mode: "ax".into(),
+ capture_scope: "window".into(),
+ max_image_dimension: 1568,
+ }
+ }
}
/// Load `DriverConfig` from `~/.cua-driver/config.json`, falling back to
@@ -34,14 +40,21 @@ impl Default for DriverConfig {
/// invocations — matching the macOS daemon's startup load. See #2008.
pub fn load_driver_config() -> DriverConfig {
let mut cfg = DriverConfig::default();
- if let Some(v) = pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned)) {
+ if let Some(v) =
+ pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned))
+ {
cfg.capture_mode = v;
}
- if let Some(v) = pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned)) {
+ if let Some(v) =
+ pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned))
+ {
cfg.capture_scope = v;
}
- if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64()) {
- if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; }
+ if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64())
+ {
+ if let Ok(v32) = u32::try_from(v) {
+ cfg.max_image_dimension = v32;
+ }
}
cfg
}
@@ -51,10 +64,20 @@ pub struct ResizeRegistry {
}
impl ResizeRegistry {
- pub fn new() -> Self { Self { ratios: std::sync::Mutex::new(Default::default()) } }
- pub fn set_ratio(&self, pid: u32, ratio: f64) { self.ratios.lock().unwrap().insert(pid, ratio); }
- pub fn clear_ratio(&self, pid: u32) { self.ratios.lock().unwrap().remove(&pid); }
- pub fn ratio(&self, pid: u32) -> Option<f64> { self.ratios.lock().unwrap().get(&pid).copied() }
+ pub fn new() -> Self {
+ Self {
+ ratios: std::sync::Mutex::new(Default::default()),
+ }
+ }
+ pub fn set_ratio(&self, pid: u32, ratio: f64) {
+ self.ratios.lock().unwrap().insert(pid, ratio);
+ }
+ pub fn clear_ratio(&self, pid: u32) {
+ self.ratios.lock().unwrap().remove(&pid);
+ }
+ pub fn ratio(&self, pid: u32) -> Option<f64> {
+ self.ratios.lock().unwrap().get(&pid).copied()
+ }
}
/// Per-process zoom context — stores padded crop origin and resize scale from
@@ -70,7 +93,10 @@ pub struct ZoomContext {
impl ZoomContext {
pub fn zoom_to_window(&self, px: f64, py: f64) -> (f64, f64) {
- (self.origin_x + px * self.scale_inv, self.origin_y + py * self.scale_inv)
+ (
+ self.origin_x + px * self.scale_inv,
+ self.origin_y + py * self.scale_inv,
+ )
}
}
@@ -79,9 +105,17 @@ pub struct ZoomRegistry {
}
impl ZoomRegistry {
- pub fn new() -> Self { Self { inner: std::sync::Mutex::new(Default::default()) } }
- pub fn set(&self, pid: u32, ctx: ZoomContext) { self.inner.lock().unwrap().insert(pid, ctx); }
- pub fn get(&self, pid: u32) -> Option<ZoomContext> { self.inner.lock().unwrap().get(&pid).copied() }
+ pub fn new() -> Self {
+ Self {
+ inner: std::sync::Mutex::new(Default::default()),
+ }
+ }
+ pub fn set(&self, pid: u32, ctx: ZoomContext) {
+ self.inner.lock().unwrap().insert(pid, ctx);
+ }
+ pub fn get(&self, pid: u32) -> Option<ZoomContext> {
+ self.inner.lock().unwrap().get(&pid).copied()
+ }
}
pub struct ToolState {
@@ -95,7 +129,6 @@ pub struct ToolState {
#[derive(Clone, Debug)]
pub struct MouseHoldState {
- pub cursor_id: String,
pub pid: u32,
pub xid: u64,
pub button: u8,
@@ -126,7 +159,8 @@ impl Tool for ListAppsTool {
fn def(&self) -> &ToolDef {
LIST_APPS_DEF.get_or_init(|| ToolDef {
name: "list_apps".into(),
- description: "List Linux apps — both currently running and installed-but-not-running — \
+ description:
+ "List Linux apps — both currently running and installed-but-not-running — \
with per-app state flags:\n\n\
- running: is a process for this app live? (pid is 0 when false)\n\
- active: reserved (Linux X11/Wayland focus model differs from frontmost-app); \
@@ -145,9 +179,13 @@ impl Tool for ListAppsTool {
filtered. A `.desktop` file whose launcher matches a running process \
(by basename) is merged into a single entry with `running: true`.\n\n\
Use this for \"is X installed?\" as well as \"is X running?\". For per-window \
- state — visibility, geometry, titles — call list_windows instead.".into(),
+ state — visibility, geometry, titles — call list_windows instead."
+ .into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
@@ -172,16 +210,23 @@ impl Tool for ListAppsTool {
}
}
- let mut consumed: std::collections::HashSet<usize> =
- std::collections::HashSet::new();
+ let mut consumed: std::collections::HashSet<usize> = std::collections::HashSet::new();
let mut out = Vec::new();
for p in &procs {
- let key_source = if !p.cmdline.is_empty() { &p.cmdline } else { &p.name };
+ let key_source = if !p.cmdline.is_empty() {
+ &p.cmdline
+ } else {
+ &p.name
+ };
let basename = exec_basename(key_source);
- if basename.is_empty() { continue }
+ if basename.is_empty() {
+ continue;
+ }
let candidates = by_exe.get(&basename).map(|v| v.as_slice()).unwrap_or(&[]);
let merged = disambiguate_installed_match(candidates, &installed, key_source);
- if let Some(idx) = merged { consumed.insert(idx); }
+ if let Some(idx) = merged {
+ consumed.insert(idx);
+ }
let (name, bundle_id, launch_path, kind, last_used) = match merged {
Some(idx) => {
let a = &installed[idx];
@@ -194,7 +239,11 @@ impl Tool for ListAppsTool {
)
}
None => (
- if !p.name.is_empty() { p.name.clone() } else { basename.clone() },
+ if !p.name.is_empty() {
+ p.name.clone()
+ } else {
+ basename.clone()
+ },
None,
None,
None,
@@ -214,7 +263,9 @@ impl Tool for ListAppsTool {
}));
}
for (i, app) in installed.iter().enumerate() {
- if consumed.contains(&i) { continue }
+ if consumed.contains(&i) {
+ continue;
+ }
out.push(json!({
"pid": 0,
"bundle_id": app.bundle_id.clone(),
@@ -228,17 +279,25 @@ impl Tool for ListAppsTool {
}));
}
out
- }).await.unwrap_or_default();
+ })
+ .await
+ .unwrap_or_default();
- let running_count = apps.iter().filter(|a| a["running"].as_bool().unwrap_or(false)).count();
+ let running_count = apps
+ .iter()
+ .filter(|a| a["running"].as_bool().unwrap_or(false))
+ .count();
let total = apps.len();
let installed_only = total - running_count;
let mut lines = vec![format!(
"✅ Found {total} app(s): {running_count} running, {installed_only} installed-not-running."
)];
- for app in apps.iter().filter(|a| a["running"].as_bool().unwrap_or(false)) {
+ for app in apps
+ .iter()
+ .filter(|a| a["running"].as_bool().unwrap_or(false))
+ {
let name = app["name"].as_str().unwrap_or("?");
- let pid = app["pid"].as_u64().unwrap_or(0);
+ let pid = app["pid"].as_u64().unwrap_or(0);
lines.push(format!("- {name} (pid {pid})"));
}
// Unified `apps` array + legacy `processes` alias for older callers.
@@ -265,13 +324,20 @@ fn disambiguate_installed_match(
installed: &[crate::installed_apps::InstalledApp],
proc_key_source: &str,
) -> Option<usize> {
- if candidates.is_empty() { return None; }
- if candidates.len() == 1 { return Some(candidates[0]); }
+ if candidates.is_empty() {
+ return None;
+ }
+ if candidates.len() == 1 {
+ return Some(candidates[0]);
+ }
// Look for an exact path match against the cmdline's first token.
let proc_path = proc_key_source.split_whitespace().next().unwrap_or("");
if !proc_path.is_empty() {
- if let Some(&idx) = candidates.iter().find(|&&i| installed[i].launch_path == proc_path) {
+ if let Some(&idx) = candidates
+ .iter()
+ .find(|&&i| installed[i].launch_path == proc_path)
+ {
return Some(idx);
}
}
@@ -297,7 +363,9 @@ fn exec_basename(s: &str) -> String {
// Take the first whitespace-separated token that looks like a binary,
// skipping `env`-style prefixes and `K=V` assignments.
for tok in s.split_whitespace() {
- if tok == "env" { continue }
+ if tok == "env" {
+ continue;
+ }
if tok.contains('=') && !tok.starts_with('/') && !tok.starts_with('-') {
// `FOO=bar` env-var assignment — skip.
continue;
@@ -334,13 +402,19 @@ impl Tool for ListWindowsTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let filter_pid = args.opt_u64("pid").map(|v| v as u32);
- let windows = tokio::task::spawn_blocking(move || crate::wayland::list_windows_dispatch(filter_pid)).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::wayland::list_windows_dispatch(filter_pid))
+ .await
+ .unwrap_or_default();
let mut lines = vec![format!("Found {} windows:", windows.len())];
for w in &windows {
- lines.push(format!(" [xid={}] pid={:?} \"{}\" {}x{}+{}+{}",
- w.xid, w.pid, w.title, w.width, w.height, w.x, w.y));
+ lines.push(format!(
+ " [xid={}] pid={:?} \"{}\" {}x{}+{}+{}",
+ w.xid, w.pid, w.title, w.width, w.height, w.x, w.y
+ ));
}
- let structured = json!({ "windows": windows.iter().map(window_record_json).collect::<Vec<_>>() });
+ let structured =
+ json!({ "windows": windows.iter().map(window_record_json).collect::<Vec<_>>() });
ToolResult::text(lines.join("\n")).with_structured(structured)
}
}
@@ -391,7 +465,9 @@ mod list_windows_tests {
let rec = window_record_json(&w);
// Canonical cross-platform shape: nested `bounds` object.
- let bounds = rec.get("bounds").expect("record must carry a `bounds` object");
+ let bounds = rec
+ .get("bounds")
+ .expect("record must carry a `bounds` object");
assert_eq!(bounds["x"], json!(10));
assert_eq!(bounds["y"], json!(20));
assert_eq!(bounds["width"], json!(300));
@@ -466,8 +542,14 @@ impl Tool for GetWindowStateTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
- let xid = match args.require_u64("window_id") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
+ let xid = match args.require_u64("window_id") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let max_dim = {
let cfg = self.state.config.read().unwrap();
cfg.max_image_dimension
@@ -497,8 +579,14 @@ impl Tool for GetWindowStateTool {
});
// Optional caps — when omitted, the AT-SPI walker uses its built-in
// defaults (#22865).
- let max_elements = args.get("max_elements").and_then(|v| v.as_u64()).map(|v| v.max(1) as usize);
- let max_depth = args.get("max_depth").and_then(|v| v.as_u64()).map(|v| v.max(1) as usize);
+ let max_elements = args
+ .get("max_elements")
+ .and_then(|v| v.as_u64())
+ .map(|v| v.max(1) as usize);
+ let max_depth = args
+ .get("max_depth")
+ .and_then(|v| v.as_u64())
+ .map(|v| v.max(1) as usize);
// Always walk the AT-SPI tree; capture the screenshot by default. The
// tree+screenshot pair is the default so the agent grounds on both and
@@ -509,7 +597,13 @@ impl Tool for GetWindowStateTool {
let state = self.state.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
- let tree_result = Some(crate::atspi::walk_tree_bounded(pid, xid, query.as_deref(), max_elements, max_depth));
+ let tree_result = Some(crate::atspi::walk_tree_bounded(
+ pid,
+ xid,
+ query.as_deref(),
+ max_elements,
+ max_depth,
+ ));
// Best-effort per-element screen bounds (AT-SPI Component.GetExtents).
// Tolerant: an empty/missing map never fails the call.
let bounds = crate::atspi::get_all_element_bounds(pid, xid).unwrap_or_default();
@@ -522,7 +616,9 @@ impl Tool for GetWindowStateTool {
let screenshot = if should_capture {
match crate::wayland::screenshot_dispatch(xid) {
Ok(raw) => {
- let orig_w = crate::capture::png_dimensions_pub(&raw).map(|(w, _)| w).unwrap_or(0);
+ let orig_w = crate::capture::png_dimensions_pub(&raw)
+ .map(|(w, _)| w)
+ .unwrap_or(0);
let png = crate::capture::resize_png_if_needed(&raw, max_dim)?;
let (w, h) = crate::capture::png_dimensions_pub(&png)?;
let original_w = if w < orig_w { Some(orig_w) } else { None };
@@ -540,7 +636,8 @@ impl Tool for GetWindowStateTool {
None
};
Ok((tree_result, screenshot, bounds))
- }).await;
+ })
+ .await;
match result {
Ok(Ok((tree_opt, shot_opt, bounds))) => {
@@ -548,9 +645,15 @@ impl Tool for GetWindowStateTool {
let mut structured = json!({ "window_id": xid, "pid": pid });
if let Some(tr) = tree_opt {
- let count = tr.nodes.iter().filter(|n| n.element_index.is_some()).count();
+ let count = tr
+ .nodes
+ .iter()
+ .filter(|n| n.element_index.is_some())
+ .count();
let header = format!("window_id={xid} pid={pid} elements={count}\n\n");
- content.push(cua_driver_core::protocol::Content::text(header + &tr.tree_markdown));
+ content.push(cua_driver_core::protocol::Content::text(
+ header + &tr.tree_markdown,
+ ));
state.element_cache.update(pid, xid, &tr.nodes);
structured["element_count"] = json!(count);
structured["tree_markdown"] = json!(tr.tree_markdown);
@@ -619,11 +722,10 @@ impl Tool for GetWindowStateTool {
.collect();
structured["elements"] = json!(elements);
// Surface 6: snapshot id mirror for debug correlation.
- structured["snapshot_id"] = json!(
- cua_driver_core::element_token::token_for(snapshot_id, 0)
+ structured["snapshot_id"] =
+ json!(cua_driver_core::element_token::token_for(snapshot_id, 0)
.trim_end_matches(":0")
- .to_string()
- );
+ .to_string());
structured["_note"] = json!(
"Prefer `elements` — `tree_markdown` will continue to work \
but new fields will only be added to the structured side. \
@@ -660,7 +762,9 @@ impl Tool for GetWindowStateTool {
if let Some((b64_opt, file_path, w, h, orig_w)) = shot_opt {
if let Some(ow) = orig_w {
- if w > 0 { state.resize_registry.set_ratio(pid, ow as f64 / w as f64); }
+ if w > 0 {
+ state.resize_registry.set_ratio(pid, ow as f64 / w as f64);
+ }
} else {
state.resize_registry.clear_ratio(pid);
}
@@ -680,7 +784,11 @@ impl Tool for GetWindowStateTool {
}
}
- ToolResult { content, is_error: None, structured_content: Some(structured) }
+ ToolResult {
+ content,
+ is_error: None,
+ structured_content: Some(structured),
+ }
}
Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")),
Err(e) => ToolResult::error(format!("Task error: {e}")),
@@ -722,53 +830,51 @@ impl Tool for LaunchAppTool {
return ToolResult::error("Provide at least one of: launch_path, name, or urls.");
}
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(
- String,
- Option<u32>,
- String,
- )> {
- // Open URLs via xdg-open.
- if !urls.is_empty() {
- for url in &urls {
- std::process::Command::new("xdg-open").arg(url).spawn()?;
- }
- return Ok((
- format!("Opened {} URL(s) via xdg-open.", urls.len()),
- None,
- "xdg-open".to_owned(),
- ));
- }
- // launch_path > name. Both go through the same direct-exec path
- // (so XDG `Exec=` commands round-trip), but launch_path is the
- // canonical form preferred by list_apps callers.
- let command = launch_path_opt.as_deref().or(name_opt.as_deref());
- if let Some(cmd) = command {
- let mut parts = cmd.split_whitespace();
- let prog = parts.next().unwrap_or(cmd);
- let rest: Vec<&str> = parts.collect();
- match std::process::Command::new(prog).args(&rest).spawn() {
- Ok(child) => {
- let pid = child.id();
- return Ok((
- format!("✅ Launched {cmd} (pid {pid}) in background."),
- Some(pid),
- cmd.to_owned(),
- ));
+ let result = tokio::task::spawn_blocking(
+ move || -> anyhow::Result<(String, Option<u32>, String)> {
+ // Open URLs via xdg-open.
+ if !urls.is_empty() {
+ for url in &urls {
+ std::process::Command::new("xdg-open").arg(url).spawn()?;
}
- Err(_) => {
- // Fall back to xdg-open for .desktop app names. xdg-open may
- // spawn a helper and exit, so do not claim its pid is the app pid.
- std::process::Command::new("xdg-open").arg(cmd).spawn()?;
- return Ok((
- format!("Opened '{cmd}' via xdg-open."),
- None,
- cmd.to_owned(),
- ));
+ return Ok((
+ format!("Opened {} URL(s) via xdg-open.", urls.len()),
+ None,
+ "xdg-open".to_owned(),
+ ));
+ }
+ // launch_path > name. Both go through the same direct-exec path
+ // (so XDG `Exec=` commands round-trip), but launch_path is the
+ // canonical form preferred by list_apps callers.
+ let command = launch_path_opt.as_deref().or(name_opt.as_deref());
+ if let Some(cmd) = command {
+ let mut parts = cmd.split_whitespace();
+ let prog = parts.next().unwrap_or(cmd);
+ let rest: Vec<&str> = parts.collect();
+ match std::process::Command::new(prog).args(&rest).spawn() {
+ Ok(child) => {
+ let pid = child.id();
+ return Ok((
+ format!("✅ Launched {cmd} (pid {pid}) in background."),
+ Some(pid),
+ cmd.to_owned(),
+ ));
+ }
+ Err(_) => {
+ // Fall back to xdg-open for .desktop app names. xdg-open may
+ // spawn a helper and exit, so do not claim its pid is the app pid.
+ std::process::Command::new("xdg-open").arg(cmd).spawn()?;
+ return Ok((
+ format!("Opened '{cmd}' via xdg-open."),
+ None,
+ cmd.to_owned(),
+ ));
+ }
}
}
- }
- unreachable!()
- })
+ unreachable!()
+ },
+ )
.await;
match result {
@@ -815,9 +921,11 @@ impl Tool for LaunchAppTool {
/// Looks up element bounds via pyatspi subprocess, finds the owning window
/// (via `xid_hint` or the first window for `pid`), then converts screen-absolute
/// → window-local coords via X11 translate_coordinates.
-fn resolve_element_local_coords(pid: u32, idx: usize, xid_hint: Option<u64>)
- -> anyhow::Result<(u64, f64, f64)>
-{
+fn resolve_element_local_coords(
+ pid: u32,
+ idx: usize,
+ xid_hint: Option<u64>,
+) -> anyhow::Result<(u64, f64, f64)> {
let (bx, by, bw, bh) = crate::atspi::get_element_bounds(pid, idx)?;
let screen_cx = bx as f64 + bw as f64 / 2.0;
let screen_cy = by as f64 + bh as f64 / 2.0;
@@ -826,7 +934,9 @@ fn resolve_element_local_coords(pid: u32, idx: usize, xid_hint: Option<u64>)
x
} else {
crate::x11::list_windows(Some(pid))
- .into_iter().next().map(|w| w.xid)
+ .into_iter()
+ .next()
+ .map(|w| w.xid)
.ok_or_else(|| anyhow::anyhow!("No windows for pid {pid}"))?
};
@@ -835,7 +945,9 @@ fn resolve_element_local_coords(pid: u32, idx: usize, xid_hint: Option<u64>)
use x11rb::rust_connection::RustConnection;
let (conn, screen_num) = RustConnection::connect(None)?;
let root = conn.setup().roots[screen_num].root;
- let reply = conn.translate_coordinates(xid as u32, root, 0, 0)?.reply()?;
+ let reply = conn
+ .translate_coordinates(xid as u32, root, 0, 0)?
+ .reply()?;
let local_x = screen_cx - reply.dst_x as f64;
let local_y = screen_cy - reply.dst_y as f64;
Ok((xid, local_x, local_y))
@@ -853,7 +965,9 @@ fn window_local_to_screen(xid: u64, x: f64, y: f64) -> anyhow::Result<(f64, f64)
let (conn, screen_num) = RustConnection::connect(None)?;
let root = conn.setup().roots[screen_num].root;
- let reply = conn.translate_coordinates(xid as u32, root, 0, 0)?.reply()?;
+ let reply = conn
+ .translate_coordinates(xid as u32, root, 0, 0)?
+ .reply()?;
Ok((reply.dst_x as f64 + x, reply.dst_y as f64 + y))
}
@@ -983,9 +1097,7 @@ fn is_chromium_embedder(pid: u32) -> bool {
fn argv_is_chromium_helper(p: u32) -> bool {
match fs::read(format!("/proc/{p}/cmdline")) {
Ok(raw) => String::from_utf8_lossy(&raw).split('\0').any(|arg| {
- arg == "--type=renderer"
- || arg == "--type=zygote"
- || arg == "--type=gpu-process"
+ arg == "--type=renderer" || arg == "--type=zygote" || arg == "--type=gpu-process"
}),
Err(_) => false,
}
@@ -1049,7 +1161,9 @@ fn window_screen_center(xid: u64) -> anyhow::Result<(i32, i32)> {
let (conn, screen_num) = RustConnection::connect(None)?;
let root = conn.setup().roots[screen_num].root;
let geom = conn.get_geometry(xid as u32)?.reply()?;
- let trans = conn.translate_coordinates(xid as u32, root, 0, 0)?.reply()?;
+ let trans = conn
+ .translate_coordinates(xid as u32, root, 0, 0)?
+ .reply()?;
Ok((
trans.dst_x as i32 + geom.width as i32 / 2,
trans.dst_y as i32 + geom.height as i32 / 2,
@@ -1135,7 +1249,11 @@ fn mouse_hold_json(cursor_id: &str, hold: Option<&MouseHoldState>) -> Value {
}
}
-fn held_target_mismatch(args: &Value, cursor_id: &str, hold: &MouseHoldState) -> Option<ToolResult> {
+fn held_target_mismatch(
+ args: &Value,
+ cursor_id: &str,
+ hold: &MouseHoldState,
+) -> Option<ToolResult> {
match args.opt_u32("pid") {
Ok(Some(pid)) if pid != hold.pid => {
return Some(
@@ -1227,7 +1345,8 @@ fn process_name(pid: u32) -> Option<String> {
}
let status = fs::read_to_string(format!("/proc/{pid}/status")).ok()?;
- status.lines()
+ status
+ .lines()
.find(|l| l.starts_with("Name:"))
.map(|l| l[5..].trim().to_owned())
}
@@ -1243,7 +1362,8 @@ fn is_terminal_process(pid: u32) -> bool {
}
fn terminal_descendant_ttys(pid: u32) -> Vec<PathBuf> {
- let mut parent_to_children: std::collections::HashMap<u32, Vec<u32>> = std::collections::HashMap::new();
+ let mut parent_to_children: std::collections::HashMap<u32, Vec<u32>> =
+ std::collections::HashMap::new();
let proc_dir = std::path::Path::new("/proc");
let entries = match fs::read_dir(proc_dir) {
Ok(entries) => entries,
@@ -1261,11 +1381,15 @@ fn terminal_descendant_ttys(pid: u32) -> Vec<PathBuf> {
Ok(status) => status,
Err(_) => continue,
};
- let parent_pid = status.lines()
+ let parent_pid = status
+ .lines()
.find(|l| l.starts_with("PPid:"))
.and_then(|l| l[5..].trim().parse::<u32>().ok());
if let Some(parent_pid) = parent_pid {
- parent_to_children.entry(parent_pid).or_default().push(child_pid);
+ parent_to_children
+ .entry(parent_pid)
+ .or_default()
+ .push(child_pid);
}
}
@@ -1414,7 +1538,11 @@ impl Tool for ClickTool {
"click: unknown button \"{button_raw}\" — expected one of left, right, middle."
));
}
- let button = parse_mouse_button(if button_raw.is_empty() { "left" } else { &button_raw });
+ let button = parse_mouse_button(if button_raw.is_empty() {
+ "left"
+ } else {
+ &button_raw
+ });
let sx = args.f64_or("x", 0.0) as i32;
let sy = args.f64_or("y", 0.0) as i32;
let n = args.u64_or("count", 1) as usize;
@@ -1433,13 +1561,18 @@ impl Tool for ClickTool {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Sent screen-absolute click at ({sx},{sy}) (desktop scope)."
))
- .with_structured(json!({ "path": "xtest_desktop", "verified": false, "effect": "unverifiable" })),
+ .with_structured(
+ json!({ "path": "xtest_desktop", "verified": false, "effect": "unverifiable" }),
+ ),
Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")),
Err(e) => ToolResult::error(format!("task error: {e}")),
};
}
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let count = args.u64_or("count", 1) as usize;
// Surface 5: reject unknown buttons so a typo can't silently fall through
// to a left-click. Empty string keeps back-compat with old clients.
@@ -1449,14 +1582,18 @@ impl Tool for ClickTool {
"click: unknown button \"{button_str_raw}\" — expected one of left, right, middle."
));
}
- let button_str = if button_str_raw.is_empty() { "left" } else { button_str_raw.as_str() };
+ let button_str = if button_str_raw.is_empty() {
+ "left"
+ } else {
+ button_str_raw.as_str()
+ };
let button = parse_mouse_button(button_str);
// Surface 6: element_token / element_index precedence resolution.
// We resolve before the legacy `opt_u64("element_index")` branch
// so a token-only call (no integer arg) still takes the element path.
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id");
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid as i32,
@@ -1469,13 +1606,15 @@ impl Tool for ClickTool {
Err(e) => return e,
};
let elem_idx_resolved: Option<usize> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let window_id_resolved: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
+ window_id.map(|v| v as u64)
+ }
cua_driver_core::element_token::ResolvedElement::None => window_id_arg,
};
@@ -1489,7 +1628,12 @@ impl Tool for ClickTool {
let (xid, sx, sy) = tokio::task::spawn_blocking(move || -> (u64, f64, f64) {
let (cx, cy) = element_screen_center(pid, idx).unwrap_or((0.0, 0.0));
let xid = xid_hint
- .or_else(|| crate::x11::list_windows(Some(pid)).into_iter().next().map(|w| w.xid))
+ .or_else(|| {
+ crate::x11::list_windows(Some(pid))
+ .into_iter()
+ .next()
+ .map(|w| w.xid)
+ })
.unwrap_or(0);
(xid, cx, cy)
})
@@ -1512,15 +1656,16 @@ impl Tool for ClickTool {
// AT-SPI rung also reports whether the actuated element looked like a
// silent no-op (passive role / no advertised action) — see
// perform_action — so the response can flag `effect: "suspected_noop"`.
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(&'static str, bool)> {
- if let Ok((_action, suspected_noop)) = crate::atspi::perform_action(pid, idx) {
- return Ok(("ax", suspected_noop));
- }
- let (xid2, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?;
- crate::input::send_click(xid2, lx as i32, ly as i32, count, button)?;
- Ok(("x11_pixel", false))
- })
- .await;
+ let result =
+ tokio::task::spawn_blocking(move || -> anyhow::Result<(&'static str, bool)> {
+ if let Ok((_action, suspected_noop)) = crate::atspi::perform_action(pid, idx) {
+ return Ok(("ax", suspected_noop));
+ }
+ let (xid2, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?;
+ crate::input::send_click(xid2, lx as i32, ly as i32, count, button)?;
+ Ok(("x11_pixel", false))
+ })
+ .await;
return match result {
// An element click is never driver-verifiable (no read-back) —
// verified:false; the caller confirms via screenshot. `effect` is
@@ -1554,10 +1699,16 @@ impl Tool for ClickTool {
let mut y = args.f64_or("y", 0.0);
if from_zoom {
match self.state.zoom_registry.get(pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; }
- None => return ToolResult::error(
- format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")
- ),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(x, y);
+ x = wx;
+ y = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
x *= ratio;
@@ -1660,8 +1811,13 @@ impl Tool for ClickTool {
} else {
inject(false)
}
- }).await;
- let mode_label = if delivery.is_foreground() { "foreground" } else { "background" };
+ })
+ .await;
+ let mode_label = if delivery.is_foreground() {
+ "foreground"
+ } else {
+ "background"
+ };
match result {
// A pixel/coordinate click is never driver-verifiable (no read-back) —
// verified:false, effect:"unverifiable"; the caller confirms via
@@ -1702,10 +1858,20 @@ async fn focus_by_pixel(
"pid": pid, "x": x, "y": y,
"delivery_mode": if foreground { "foreground" } else { "background" },
});
- if let Some(wid) = window_id { click_args["window_id"] = json!(wid); }
- if let Some(s) = session { click_args["session"] = json!(s); }
- if from_zoom { click_args["from_zoom"] = json!(true); }
- let focus = ClickTool { state: state.clone() }.invoke(click_args).await;
+ if let Some(wid) = window_id {
+ click_args["window_id"] = json!(wid);
+ }
+ if let Some(s) = session {
+ click_args["session"] = json!(s);
+ }
+ if from_zoom {
+ click_args["from_zoom"] = json!(true);
+ }
+ let focus = ClickTool {
+ state: state.clone(),
+ }
+ .invoke(click_args)
+ .await;
if focus.is_error == Some(true) {
return Err(ToolResult::error(format!(
"focus pixel-click at ({x:.0},{y:.0}) failed."
@@ -1749,7 +1915,10 @@ impl Tool for TypeTextTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let pid = args.u64_or("pid", 0) as u32;
- let text_raw = match args.require_str("text") { Ok(v) => v, Err(e) => return e };
+ let text_raw = match args.require_str("text") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// Strip trailing agent-protocol closing tags — see
// cua_driver_core::text_sanitize docs for rationale.
let text = cua_driver_core::text_sanitize::strip_trailing_agent_protocol_tags(&text_raw)
@@ -1768,8 +1937,11 @@ impl Tool for TypeTextTool {
Err(e) => return e,
};
let (resolved_elem_idx, resolved_window_id) = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, window_id, .. } =>
- (Some(*element_index), window_id.map(|v| v as u64)),
+ cua_driver_core::element_token::ResolvedElement::Element {
+ element_index,
+ window_id,
+ ..
+ } => (Some(*element_index), window_id.map(|v| v as u64)),
cua_driver_core::element_token::ResolvedElement::None => (None, None),
};
let xid_opt = resolved_window_id.or_else(|| args.opt_u64("window_id"));
@@ -1778,10 +1950,17 @@ impl Tool for TypeTextTool {
let xid = match xid_opt {
Some(x) => x,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.xid,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -1792,19 +1971,29 @@ impl Tool for TypeTextTool {
// can't, then fall through to the focused-element type path below (it
// escalates AT-SPI → key events and lands once focused). Reuses ClickTool's
// exact coordinate translation + delivery_mode.
- if let (Some(cx), Some(cy)) =
- (args.get("x").and_then(|v| v.as_f64()), args.get("y").and_then(|v| v.as_f64()))
- {
+ if let (Some(cx), Some(cy)) = (
+ args.get("x").and_then(|v| v.as_f64()),
+ args.get("y").and_then(|v| v.as_f64()),
+ ) {
if resolved_elem_idx.is_some() {
return ToolResult::error(
- "Pass either element_index (ax) or x,y (px) to type_text, not both."
+ "Pass either element_index (ax) or x,y (px) to type_text, not both.",
);
}
let fg = crate::input::delivery::DeliveryMode::from_args(&args).is_foreground();
let from_zoom = args.bool_or("from_zoom", false);
if let Err(e) = focus_by_pixel(
- &self.state, pid, Some(xid), cx, cy, fg, args.opt_str("session"), from_zoom,
- ).await {
+ &self.state,
+ pid,
+ Some(xid),
+ cx,
+ cy,
+ fg,
+ args.opt_str("session"),
+ from_zoom,
+ )
+ .await
+ {
return e;
}
// resolved_elem_idx stays None → the type path below writes to the now-
@@ -1817,12 +2006,17 @@ impl Tool for TypeTextTool {
let text_len = text.chars().count();
let text_w = text.clone();
let result =
- tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w)).await;
+ tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w))
+ .await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Typed {text_len} character(s) (focus-free via EIS compositor)."
))
- .with_structured(type_text_structured("key_events", text_len, false)),
+ .with_structured(type_text_structured(
+ "key_events",
+ text_len,
+ false,
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -1840,7 +2034,11 @@ impl Tool for TypeTextTool {
Ok(Ok(())) => ToolResult::text(format!(
"Typed {text_len} character(s) (via Wayland virtual-keyboard)."
))
- .with_structured(type_text_structured("key_events", text_len, false)),
+ .with_structured(type_text_structured(
+ "key_events",
+ text_len,
+ false,
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -1855,9 +2053,10 @@ impl Tool for TypeTextTool {
// synthesis. Either way the structured response reports
// `path: "key_events"` so callers can verify the route taken.
let pid_is_terminal = is_terminal_process(pid);
- let wm_class_is_terminal = tokio::task::spawn_blocking(move || {
- crate::terminal::is_terminal_window(xid)
- }).await.unwrap_or(false);
+ let wm_class_is_terminal =
+ tokio::task::spawn_blocking(move || crate::terminal::is_terminal_window(xid))
+ .await
+ .unwrap_or(false);
if pid_is_terminal || wm_class_is_terminal {
let text_len = text.chars().count();
let text_t = text.clone();
@@ -1869,12 +2068,17 @@ impl Tool for TypeTextTool {
return Ok(());
}
crate::input::send_type_text_xtest(&text_t)
- }).await;
+ })
+ .await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Typed {text_len} character(s) (terminal emulator: pty/XTest key events)."
))
- .with_structured(type_text_structured("key_events", text_len, false)),
+ .with_structured(type_text_structured(
+ "key_events",
+ text_len,
+ false,
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -1906,7 +2110,10 @@ impl Tool for TypeTextTool {
// nothing focused (None) falls through to the existing AT-SPI-first flow.
let focus_kind = tokio::task::spawn_blocking(move || {
crate::atspi::focused_is_editable(pid).ok().flatten()
- }).await.ok().flatten();
+ })
+ .await
+ .ok()
+ .flatten();
if focus_kind == Some(false) {
let text_f = text.clone();
let result = tokio::task::spawn_blocking(move || {
@@ -1918,12 +2125,17 @@ impl Tool for TypeTextTool {
// stay empty. The click that gave this widget focus already put it
// under the X input focus, so XTest-to-focus lands correctly.
crate::input::send_type_text_xtest(&text_f)
- }).await;
+ })
+ .await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Typed {text_len} character(s) into the focused widget."
))
- .with_structured(type_text_structured("key_events", text_len, false)),
+ .with_structured(type_text_structured(
+ "key_events",
+ text_len,
+ false,
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -1931,9 +2143,9 @@ impl Tool for TypeTextTool {
// Try AT-SPI EditableText first (focus-free, works for Qt6/GTK4).
let text_clone = text.clone();
- let atspi_result = tokio::task::spawn_blocking(move || {
- crate::atspi::type_into_editable(pid, &text_clone)
- }).await;
+ let atspi_result =
+ tokio::task::spawn_blocking(move || crate::atspi::type_into_editable(pid, &text_clone))
+ .await;
match atspi_result {
Ok(Ok(())) => {
@@ -1964,12 +2176,15 @@ impl Tool for TypeTextTool {
crate::input::send_focus_out(xid)?;
result
- }).await;
+ })
+ .await;
match qt5_result {
Ok(Ok(())) => {
return type_text_ax_confirm_result(
- pid, text_len, "via AT-SPI with focus workaround",
+ pid,
+ text_len,
+ "via AT-SPI with focus workaround",
);
}
_ => {
@@ -2012,8 +2227,13 @@ impl Tool for TypeTextTool {
}
crate::input::send_type_text(xid, &text)?;
Ok("key_events")
- }).await;
- let mode_label = if delivery.is_foreground() { "foreground" } else { "background" };
+ })
+ .await;
+ let mode_label = if delivery.is_foreground() {
+ "foreground"
+ } else {
+ "background"
+ };
match result {
// Read-back verdict: the AT-SPI EditableText.insertText path ("ax") is
// the driver-verifiable rung on Linux — the a11y layer confirms the
@@ -2024,7 +2244,9 @@ impl Tool for TypeTextTool {
// through the Electron/Chromium AX-echo suppression (mirrors macOS).
// Every other rung is already verified:false.
Ok(Ok("ax")) => type_text_ax_confirm_result(
- pid, text_len, &format!("via X11, delivery_mode={mode_label}"),
+ pid,
+ text_len,
+ &format!("via X11, delivery_mode={mode_label}"),
),
Ok(Ok(path)) => ToolResult::text(format!(
"Typed {text_len} character(s) (via X11, delivery_mode={mode_label})."
@@ -2070,7 +2292,10 @@ impl Tool for PressKeyTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let pid = args.u64_or("pid", 0) as u32;
- let key = match args.require_str("key") { Ok(v) => v, Err(e) => return e };
+ let key = match args.require_str("key") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let mods: Vec<String> = args.str_array("modifiers");
// Surface 6: resolve element_token / element_index for the window-id
@@ -2078,7 +2303,7 @@ impl Tool for PressKeyTool {
// the resolved window_id — element_index itself is not used to
// address an AX node here (no focus-grab path).
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id");
+ let window_id_arg = args.opt_u64("window_id");
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid as i32,
@@ -2091,17 +2316,25 @@ impl Tool for PressKeyTool {
Err(e) => return e,
};
let xid_opt = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64).or(window_id_arg),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
+ window_id.map(|v| v as u64).or(window_id_arg)
+ }
cua_driver_core::element_token::ResolvedElement::None => window_id_arg,
};
let xid = match xid_opt {
Some(x) => x,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.xid,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -2111,33 +2344,69 @@ impl Tool for PressKeyTool {
// background path (the focus-click already handled fronting when fg). Pass x,y
// (no element_index) for Chromium/Electron surfaces the AX path can't focus.
let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
- let px_focused = {
+
+ // An element-addressed keypress needs to establish the target's
+ // focus before the window-level X11 key event is sent. AT-SPI's
+ // primary action is the focus-free route for editable web controls;
+ // resolving the element only for its window ID silently loses the key
+ // on Chromium inputs whose window is backgrounded.
+ if let Some(element_index) = element_index_arg {
+ let focused = tokio::task::spawn_blocking(move || {
+ crate::atspi::focus_element(pid, element_index)
+ })
+ .await;
+ match focused {
+ Ok(Ok(true)) => {}
+ Ok(Ok(false)) => {
+ return ToolResult::error(format!(
+ "AT-SPI Component.GrabFocus returned false for element {element_index}"
+ ))
+ }
+ Ok(Err(e)) => return ToolResult::error(e.to_string()),
+ Err(e) => return ToolResult::error(format!("Task error: {e}")),
+ }
+ }
+
+ let px_target = {
let px = args.get("x").and_then(|v| v.as_f64());
let py = args.get("y").and_then(|v| v.as_f64());
if let (Some(cx), Some(cy)) = (px, py) {
if element_index_arg.is_some() {
return ToolResult::error(
- "Pass either element_index (ax) or x,y (px) to press_key, not both."
+ "Pass either element_index (ax) or x,y (px) to press_key, not both.",
);
}
let from_zoom = args.bool_or("from_zoom", false);
if let Err(e) = focus_by_pixel(
- &self.state, pid, Some(xid), cx, cy, delivery.is_foreground(),
- args.opt_str("session"), from_zoom,
- ).await {
+ &self.state,
+ pid,
+ Some(xid),
+ cx,
+ cy,
+ delivery.is_foreground(),
+ args.opt_str("session"),
+ from_zoom,
+ )
+ .await
+ {
return e;
}
- true
- } else { false }
+ Some((cx.round() as i32, cy.round() as i32))
+ } else {
+ None
+ }
};
// EIS nested compositor: focus-free named-key into window_id.
if crate::wayland::is_inject_mode() {
let key_w = key.clone();
let result =
- tokio::task::spawn_blocking(move || crate::wayland::inject_press_key(xid, &key_w)).await;
+ tokio::task::spawn_blocking(move || crate::wayland::inject_press_key(xid, &key_w))
+ .await;
return match result {
- Ok(Ok(())) => ToolResult::text(format!("Pressed key '{key}' (focus-free via EIS compositor).")),
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Pressed key '{key}' (focus-free via EIS compositor)."
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -2146,7 +2415,8 @@ impl Tool for PressKeyTool {
// Native Wayland: send the key to the focused surface via virtual-keyboard.
if crate::wayland::is_wayland() {
let key_w = key.clone();
- let result = tokio::task::spawn_blocking(move || crate::wayland::press_key(&key_w)).await;
+ let result =
+ tokio::task::spawn_blocking(move || crate::wayland::press_key(&key_w)).await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Pressed key '{key}' (via Wayland virtual-keyboard)."
@@ -2159,7 +2429,7 @@ impl Tool for PressKeyTool {
let key_for_task = key.clone();
// px-focus already clicked (and fronted, when fg) the target → deliver via
// the plain background path. Otherwise honor the requested delivery_mode.
- let deliver_fg = delivery.is_foreground() && !px_focused;
+ let deliver_fg = delivery.is_foreground() && px_target.is_none();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
if mods.is_empty() && key_for_task.eq_ignore_ascii_case("enter") {
if inject_terminal_input(pid, xid, "\n")? {
@@ -2177,12 +2447,23 @@ impl Tool for PressKeyTool {
crate::input::send_key_xtest(&key_for_task, &m)
});
}
- crate::input::send_key(xid, &key_for_task, &m)
- }).await;
- let mode_label = if deliver_fg { "foreground" } else { "background" };
+ if let Some((x, y)) = px_target {
+ crate::input::send_key_at(xid, x, y, &key_for_task, &m)
+ } else {
+ crate::input::send_key(xid, &key_for_task, &m)
+ }
+ })
+ .await;
+ let mode_label = if deliver_fg {
+ "foreground"
+ } else {
+ "background"
+ };
match result {
- Ok(Ok(())) => ToolResult::text(format!("Pressed key '{key}' (delivery_mode={mode_label})."))
- .with_structured(json!({ "verified": false, "delivery_mode": mode_label })),
+ Ok(Ok(())) => {
+ ToolResult::text(format!("Pressed key '{key}' (delivery_mode={mode_label})."))
+ .with_structured(json!({ "verified": false, "delivery_mode": mode_label }))
+ }
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -2192,8 +2473,19 @@ impl Tool for PressKeyTool {
// ── hotkey ────────────────────────────────────────────────────────────────────
fn is_modifier(k: &str) -> bool {
- matches!(k.to_lowercase().as_str(),
- "ctrl" | "control" | "shift" | "alt" | "super" | "meta" | "cmd" | "command" | "win" | "windows")
+ matches!(
+ k.to_lowercase().as_str(),
+ "ctrl"
+ | "control"
+ | "shift"
+ | "alt"
+ | "super"
+ | "meta"
+ | "cmd"
+ | "command"
+ | "win"
+ | "windows"
+ )
}
pub struct HotkeyTool {
@@ -2233,17 +2525,27 @@ impl Tool for HotkeyTool {
let xid = match xid_opt {
Some(x) => x,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.xid,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
// Parse keys array (preferred) or fall back to legacy key+modifiers.
let (key, mods) = if let Some(arr) = args.get("keys").and_then(|v| v.as_array()) {
- let keys: Vec<String> = arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect();
+ let keys: Vec<String> = arr
+ .iter()
+ .filter_map(|v| v.as_str().map(str::to_owned))
+ .collect();
let modifiers: Vec<String> = keys.iter().filter(|k| is_modifier(k)).cloned().collect();
let non_mods: Vec<String> = keys.iter().filter(|k| !is_modifier(k)).cloned().collect();
if non_mods.is_empty() {
@@ -2254,7 +2556,9 @@ impl Tool for HotkeyTool {
let mods: Vec<String> = args.str_array("modifiers");
(k, mods)
} else {
- return ToolResult::error("Provide 'keys' array (e.g. [\"ctrl\",\"c\"]) or 'key'+'modifiers' parameters.");
+ return ToolResult::error(
+ "Provide 'keys' array (e.g. [\"ctrl\",\"c\"]) or 'key'+'modifiers' parameters.",
+ );
};
let key_display = format!("{}+{}", mods.join("+"), key);
@@ -2262,25 +2566,49 @@ impl Tool for HotkeyTool {
let mods_for_wayland = mods.clone();
let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
+ // X11 can address an unfocused native window with synthetic events, but
+ // Chromium's renderer only processes modifier chords for its focused
+ // input surface. Do not report the XSendEvent attempt as success: the
+ // explicit foreground rung is the only truthful fallback unless an
+ // EIS compositor can target the surface without focus.
+ if !delivery.is_foreground()
+ && is_chromium_embedder(pid)
+ && !crate::wayland::is_inject_mode()
+ {
+ return crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::ChromiumHotkey,
+ );
+ }
+
// ── px form: pixel-click to focus, then the combo acts on the focused field ──
// (e.g. Ctrl+V into a Chromium input). Reuses click's translation +
// delivery_mode; after it, deliver the combo via the plain background path
// (the focus-click already fronted when fg).
- let px_focused = {
+ let px_target = {
let px = args.get("x").and_then(|v| v.as_f64());
let py = args.get("y").and_then(|v| v.as_f64());
if let (Some(cx), Some(cy)) = (px, py) {
let from_zoom = args.bool_or("from_zoom", false);
if let Err(e) = focus_by_pixel(
- &self.state, pid, Some(xid), cx, cy, delivery.is_foreground(),
- args.opt_str("session"), from_zoom,
- ).await {
+ &self.state,
+ pid,
+ Some(xid),
+ cx,
+ cy,
+ delivery.is_foreground(),
+ args.opt_str("session"),
+ from_zoom,
+ )
+ .await
+ {
return e;
}
- true
- } else { false }
+ Some((cx.round() as i32, cy.round() as i32))
+ } else {
+ None
+ }
};
- let deliver_fg = delivery.is_foreground() && !px_focused;
+ let deliver_fg = delivery.is_foreground() && px_target.is_none();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
if crate::wayland::is_wayland() {
@@ -2301,12 +2629,23 @@ impl Tool for HotkeyTool {
crate::input::send_key_xtest(&key, &m)
});
}
- crate::input::send_key(xid, &key, &m)
- }).await;
- let mode_label = if deliver_fg { "foreground" } else { "background" };
+ if let Some((x, y)) = px_target {
+ crate::input::send_key_at(xid, x, y, &key, &m)
+ } else {
+ crate::input::send_key(xid, &key, &m)
+ }
+ })
+ .await;
+ let mode_label = if deliver_fg {
+ "foreground"
+ } else {
+ "background"
+ };
match result {
- Ok(Ok(())) => ToolResult::text(format!("Pressed {key_display} on pid {pid} (delivery_mode={mode_label})."))
- .with_structured(json!({ "verified": false, "delivery_mode": mode_label })),
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Pressed {key_display} on pid {pid} (delivery_mode={mode_label})."
+ ))
+ .with_structured(json!({ "verified": false, "delivery_mode": mode_label })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -2340,8 +2679,14 @@ impl Tool for SetValueTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
- let value = match args.require_str("value") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
+ let value = match args.require_str("value") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// Surface 6: element_token / element_index precedence resolution.
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid as i32,
@@ -2354,11 +2699,12 @@ impl Tool for SetValueTool {
Err(e) => return e,
};
let idx = match resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => element_index,
- cua_driver_core::element_token::ResolvedElement::None =>
- return ToolResult::error(
- "set_value requires element_index or element_token to address the target element."
- ),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ element_index
+ }
+ cua_driver_core::element_token::ResolvedElement::None => return ToolResult::error(
+ "set_value requires element_index or element_token to address the target element.",
+ ),
};
let value_for_task = value.clone();
// Pulse the agent cursor onto the target element before writing, so a
@@ -2378,9 +2724,9 @@ impl Tool for SetValueTool {
y: sy,
});
}
- let result = tokio::task::spawn_blocking(move || {
- crate::atspi::set_value(pid, idx, &value_for_task)
- }).await;
+ let result =
+ tokio::task::spawn_blocking(move || crate::atspi::set_value(pid, idx, &value_for_task))
+ .await;
match result {
Ok(Ok(())) => ToolResult::text(format!("Set value of element [{idx}] to '{value}'.")),
Ok(Err(e)) => ToolResult::error(e.to_string()),
@@ -2423,10 +2769,15 @@ impl Tool for ScrollTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
- use cua_driver_core::tool_args::ArgsExt;
let cursor_id = resolve_cursor_key(&args);
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
- let direction = match args.require_str("direction") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
+ let direction = match args.require_str("direction") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let amount = args.u64_or("amount", 3).clamp(1, 50) as usize;
// Surface 6: resolve element_token / element_index. The Linux
// scroll implementation today doesn't actually pre-focus the
@@ -2444,8 +2795,9 @@ impl Tool for ScrollTool {
Err(e) => return e,
};
let xid_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
+ .map(|v| v as u64)
+ .or_else(|| args.opt_u64("window_id")),
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
@@ -2453,23 +2805,88 @@ impl Tool for ScrollTool {
let xid = match xid_opt {
Some(x) => x,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.xid,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
+ }
+ }
+ };
+
+ let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
+ if !delivery.is_foreground() {
+ if let cua_driver_core::element_token::ResolvedElement::Element {
+ element_index, ..
+ } = &resolved
+ {
+ let idx = *element_index;
+ let direction_for_ax = direction.clone();
+ let ax_result = tokio::task::spawn_blocking(move || {
+ crate::atspi::scroll_element(pid, idx, &direction_for_ax, amount)
+ })
+ .await;
+ if matches!(ax_result, Ok(Ok(()))) {
+ return ToolResult::text(format!(
+ "Scrolled {direction} {amount} ticks via AT-SPI (delivery_mode:background)."
+ ))
+ .with_structured(json!({
+ "path": "atspi",
+ "verified": false,
+ "delivery_mode": "background"
+ }));
+ }
+ }
+ }
+
+ // An element-addressed scroll must land over the element. The old
+ // fallback used (0, 0) in the window, which can report success while
+ // Chromium/GTK routes the wheel to the toplevel instead of the
+ // requested scroll region.
+ let element_point = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ let idx = *element_index;
+ match tokio::task::spawn_blocking(move || {
+ resolve_element_local_coords(pid, idx, Some(xid))
+ })
+ .await
+ {
+ Ok(Ok((_resolved_xid, local_x, local_y))) => {
+ let screen = window_local_to_screen(xid, local_x, local_y).ok();
+ Some(((local_x, local_y), screen))
+ }
+ Ok(Err(e)) => {
+ return ToolResult::error(format!(
+ "Could not resolve scroll element [{idx}] coordinates: {e}"
+ ))
+ }
+ Err(e) => {
+ return ToolResult::error(format!(
+ "Scroll element coordinate task failed: {e}"
+ ))
+ }
}
}
+ cua_driver_core::element_token::ResolvedElement::None => None,
};
// X11 scroll buttons: 4=up, 5=down, 6=left, 7=right
// Note: "page" scroll is still per-click on X11; send more ticks for page.
let button: u8 = match direction.as_str() {
- "up" => 4, "left" => 6, "right" => 7, _ => 5,
+ "up" => 4,
+ "left" => 6,
+ "right" => 7,
+ _ => 5,
};
let direction_for_wayland = direction.clone();
let amount_u32 = amount as u32;
let cursor_id_for_task = cursor_id.clone();
- let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
if crate::wayland::is_wayland() {
return crate::wayland::scroll(xid, &direction_for_wayland, amount_u32);
@@ -2477,44 +2894,55 @@ impl Tool for ScrollTool {
// foreground: activate the window, then scroll, then restore — for
// surfaces that only route wheel events to the active window.
let x11_scroll = || -> anyhow::Result<()> {
- // X11: synthetic Button4-7 XSendEvents are dropped by XInput2
- // toolkits (GTK never scrolls). On a real Xorg host, drive a real
- // wheel detent through the MPX uinput pointer over the window's
- // center — libinput turns it into the XI2 smooth-scroll GTK reads —
- // without stealing focus. Falls back to the legacy XSendEvent
- // Button4-7 path on Xvfb / unsupported servers.
- // Button → axis/sign: 4=up(+v) 5=down(-v) 6=left(-h) 7=right(+h).
- if crate::input::real_pointer_input_available() {
- if let Ok((cx, cy)) = window_screen_center(xid) {
- let horizontal = matches!(button, 6 | 7);
- let ticks = match button {
- 4 | 7 => amount as i32,
- _ => -(amount as i32), // 5 (down) and 6 (left)
- };
- match crate::input::send_virtual_pointer_scroll(
- &cursor_id_for_task,
- &crate::input::VirtualPointerScroll {
- target_window: xid,
- x: cx,
- y: cy,
- horizontal,
- ticks,
- },
- ) {
- Ok(()) => return Ok(()),
- Err(e) => tracing::warn!("MPX scroll fell back to XSendEvent: {e}"),
+ // X11: synthetic Button4-7 XSendEvents are dropped by XInput2
+ // toolkits (GTK never scrolls). On a real Xorg host, drive a real
+ // wheel detent through the MPX uinput pointer over the window's
+ // center — libinput turns it into the XI2 smooth-scroll GTK reads —
+ // without stealing focus. Falls back to the legacy XSendEvent
+ // Button4-7 path on Xvfb / unsupported servers.
+ // Button → axis/sign: 4=up(+v) 5=down(-v) 6=left(-h) 7=right(+h).
+ if crate::input::real_pointer_input_available() {
+ let pointer_point = element_point
+ .and_then(|(_, screen)| screen)
+ .map(|(x, y)| (x as i32, y as i32))
+ .or_else(|| window_screen_center(xid).ok());
+ if let Some((cx, cy)) = pointer_point {
+ let horizontal = matches!(button, 6 | 7);
+ let ticks = match button {
+ 4 | 7 => amount as i32,
+ _ => -(amount as i32), // 5 (down) and 6 (left)
+ };
+ match crate::input::send_virtual_pointer_scroll(
+ &cursor_id_for_task,
+ &crate::input::VirtualPointerScroll {
+ target_window: xid,
+ x: cx,
+ y: cy,
+ horizontal,
+ ticks,
+ },
+ ) {
+ Ok(()) => return Ok(()),
+ Err(e) => tracing::warn!("MPX scroll fell back to XSendEvent: {e}"),
+ }
}
}
- }
- crate::input::send_click(xid, 0, 0, amount, button)
+ let (local_x, local_y) =
+ element_point.map(|(local, _)| local).unwrap_or((0.0, 0.0));
+ crate::input::send_click(xid, local_x as i32, local_y as i32, amount, button)
};
if delivery.is_foreground() {
crate::input::with_x11_foreground(xid, 80, x11_scroll)
} else {
x11_scroll()
}
- }).await;
- let mode_label = if delivery.is_foreground() { "foreground" } else { "background" };
+ })
+ .await;
+ let mode_label = if delivery.is_foreground() {
+ "foreground"
+ } else {
+ "background"
+ };
match result {
Ok(Ok(())) => ToolResult::text(format!(
"Scrolled {direction} {amount} ticks (delivery_mode={mode_label})."
@@ -2563,9 +2991,11 @@ impl Tool for DoubleClickTool {
})
}
async fn invoke(&self, args: Value) -> ToolResult {
- use cua_driver_core::tool_args::ArgsExt;
let cursor_id = resolve_cursor_key(&args);
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// Surface 6: element_token / element_index precedence.
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid as i32,
@@ -2578,20 +3008,23 @@ impl Tool for DoubleClickTool {
Err(e) => return e,
};
let elem_idx_resolved = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let window_id_resolved: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
+ window_id.map(|v| v as u64)
+ }
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
if let Some(idx) = elem_idx_resolved {
let xid_hint = window_id_resolved;
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {
resolve_element_local_coords(pid, idx, xid_hint)
- }).await;
+ })
+ .await;
return match result {
Ok(Ok((xid, lx, ly))) => {
if let Ok(Ok((sx, sy))) =
@@ -2615,9 +3048,12 @@ impl Tool for DoubleClickTool {
return crate::wayland::click(xid, lxi, lyi, 2, 1);
}
x11_pixel_click_no_focus_steal(&cursor_id_for_task, xid, lxi, lyi, 1, 2)
- }).await;
+ })
+ .await;
match click_result {
- Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked element [{idx}].")),
+ Ok(Ok(())) => {
+ ToolResult::text(format!("✅ Double-clicked element [{idx}]."))
+ }
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -2627,17 +3063,24 @@ impl Tool for DoubleClickTool {
};
}
let xid = match window_id_resolved {
- Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."),
+ Some(v) => v,
+ None => return ToolResult::error("Provide either element_index or window_id + x/y."),
};
let from_zoom = args.bool_or("from_zoom", false);
let mut x = args.f64_or("x", 0.0);
let mut y = args.f64_or("y", 0.0);
if from_zoom {
match self.state.zoom_registry.get(pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; }
- None => return ToolResult::error(
- format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")
- ),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(x, y);
+ x = wx;
+ y = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
x *= ratio;
@@ -2693,8 +3136,13 @@ impl Tool for DoubleClickTool {
});
}
x11_pixel_click_no_focus_steal(&cursor_id_for_task, xid, xi, yi, 1, 2)
- }).await;
- let mode_label = if delivery.is_foreground() { "foreground" } else { "background" };
+ })
+ .await;
+ let mode_label = if delivery.is_foreground() {
+ "foreground"
+ } else {
+ "background"
+ };
match result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Double-clicked at ({x:.1}, {y:.1}) (delivery_mode={mode_label})."
@@ -2738,9 +3186,11 @@ impl Tool for RightClickTool {
})
}
async fn invoke(&self, args: Value) -> ToolResult {
- use cua_driver_core::tool_args::ArgsExt;
let cursor_id = resolve_cursor_key(&args);
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// Surface 6: element_token / element_index precedence.
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid as i32,
@@ -2753,20 +3203,23 @@ impl Tool for RightClickTool {
Err(e) => return e,
};
let elem_idx_resolved = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let window_id_resolved: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
+ window_id.map(|v| v as u64)
+ }
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
if let Some(idx) = elem_idx_resolved {
let xid_hint = window_id_resolved;
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {
resolve_element_local_coords(pid, idx, xid_hint)
- }).await;
+ })
+ .await;
return match result {
Ok(Ok((xid, lx, ly))) => {
if let Ok(Ok((sx, sy))) =
@@ -2790,9 +3243,12 @@ impl Tool for RightClickTool {
return crate::wayland::click(xid, lxi, lyi, 1, 3);
}
x11_pixel_click_no_focus_steal(&cursor_id_for_task, xid, lxi, lyi, 3, 1)
- }).await;
+ })
+ .await;
match click_result {
- Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked element [{idx}].")),
+ Ok(Ok(())) => {
+ ToolResult::text(format!("✅ Right-clicked element [{idx}]."))
+ }
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -2802,17 +3258,24 @@ impl Tool for RightClickTool {
};
}
let xid = match window_id_resolved {
- Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."),
+ Some(v) => v,
+ None => return ToolResult::error("Provide either element_index or window_id + x/y."),
};
let from_zoom = args.bool_or("from_zoom", false);
let mut x = args.f64_or("x", 0.0);
let mut y = args.f64_or("y", 0.0);
if from_zoom {
match self.state.zoom_registry.get(pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; }
- None => return ToolResult::error(
- format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")
- ),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(x, y);
+ x = wx;
+ y = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
x *= ratio;
@@ -2867,9 +3330,14 @@ impl Tool for RightClickTool {
});
}
x11_pixel_click_no_focus_steal(&cursor_id_for_task, xid, xi, yi, 3, 1)
- }).await;
- let mode_label = if delivery.is_foreground() { "foreground" } else { "background" };
- match result {
+ })
+ .await;
+ let mode_label = if delivery.is_foreground() {
+ "foreground"
+ } else {
+ "background"
+ };
+ match result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Right-clicked at ({x:.1}, {y:.1}) (delivery_mode={mode_label})."
))
@@ -2915,37 +3383,63 @@ impl Tool for DragTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
let cursor_id = resolve_cursor_key(&args);
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let xid = match args.opt_u64("window_id") {
- Some(v) => v, None => return ToolResult::error("window_id is required on Linux."),
+ Some(v) => v,
+ None => return ToolResult::error("window_id is required on Linux."),
};
let coerce = |key: &str| -> Option<f64> {
- args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64))
+ args.opt_f64(key)
+ .or_else(|| args.opt_i64(key).map(|i| i as f64))
+ };
+ let mut from_x = match coerce("from_x") {
+ Some(v) => v,
+ None => return ToolResult::error("Missing: from_x"),
+ };
+ let mut from_y = match coerce("from_y") {
+ Some(v) => v,
+ None => return ToolResult::error("Missing: from_y"),
+ };
+ let mut to_x = match coerce("to_x") {
+ Some(v) => v,
+ None => return ToolResult::error("Missing: to_x"),
+ };
+ let mut to_y = match coerce("to_y") {
+ Some(v) => v,
+ None => return ToolResult::error("Missing: to_y"),
};
- let mut from_x = match coerce("from_x") { Some(v) => v, None => return ToolResult::error("Missing: from_x") };
- let mut from_y = match coerce("from_y") { Some(v) => v, None => return ToolResult::error("Missing: from_y") };
- let mut to_x = match coerce("to_x") { Some(v) => v, None => return ToolResult::error("Missing: to_x") };
- let mut to_y = match coerce("to_y") { Some(v) => v, None => return ToolResult::error("Missing: to_y") };
let duration_ms = args.u64_or("duration_ms", 500);
- let steps = args.u64_or("steps", 20) as usize;
- let button_str = args.str_or("button", "left");
+ let steps = args.u64_or("steps", 20) as usize;
+ let button_str = args.str_or("button", "left");
let button = parse_mouse_button(button_str.as_str());
- let from_zoom = args.bool_or("from_zoom", false);
+ let from_zoom = args.bool_or("from_zoom", false);
if from_zoom {
match self.state.zoom_registry.get(pid) {
Some(ctx) => {
let (wx, wy) = ctx.zoom_to_window(from_x, from_y);
let (wx2, wy2) = ctx.zoom_to_window(to_x, to_y);
- from_x = wx; from_y = wy; to_x = wx2; to_y = wy2;
+ from_x = wx;
+ from_y = wy;
+ to_x = wx2;
+ to_y = wy2;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
}
- None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")),
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
- from_x *= ratio; from_y *= ratio;
- to_x *= ratio; to_y *= ratio;
+ from_x *= ratio;
+ from_y *= ratio;
+ to_x *= ratio;
+ to_y *= ratio;
}
crate::overlay::send_command_for(
@@ -2956,11 +3450,16 @@ impl Tool for DragTool {
tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await
{
overlay_glide_to_for(&cursor_id, sx_from, sy_from).await;
- self.state.cursor_registry.update_position(&cursor_id, sx_from, sy_from);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_id, sx_from, sy_from);
overlay_snap_to_for(&cursor_id, sx_from, sy_from, None);
crate::overlay::send_command_for(
cursor_id.clone(),
- cursor_overlay::OverlayCommand::ClickPulse { x: sx_from, y: sy_from },
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx_from,
+ y: sy_from,
+ },
);
}
crate::overlay::send_command_for(
@@ -2977,7 +3476,8 @@ impl Tool for DragTool {
let steps_u32 = steps as u32;
let drag_result = tokio::task::spawn_blocking(move || {
crate::wayland::drag(xid, fxi, fyi, txi, tyi, steps_u32, button)
- }).await;
+ })
+ .await;
crate::overlay::send_command_for(
cursor_id.clone(),
cursor_overlay::OverlayCommand::SetPressed(false),
@@ -2994,8 +3494,14 @@ impl Tool for DragTool {
}
let press_result = tokio::task::spawn_blocking(move || {
- crate::input::send_button_down(xid, from_x.round() as i32, from_y.round() as i32, button)
- }).await;
+ crate::input::send_button_down(
+ xid,
+ from_x.round() as i32,
+ from_y.round() as i32,
+ button,
+ )
+ })
+ .await;
let mut result: anyhow::Result<()> = match press_result {
Ok(Ok(())) => Ok(()),
Ok(Err(e)) => Err(e),
@@ -3003,7 +3509,11 @@ impl Tool for DragTool {
};
if result.is_ok() {
- let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };
+ let step_delay_ms = if steps > 1 {
+ duration_ms / steps as u64
+ } else {
+ duration_ms
+ };
let mut prev_x = from_x;
let mut prev_y = from_y;
for i in 1..=steps {
@@ -3011,27 +3521,37 @@ impl Tool for DragTool {
let ix = from_x + (to_x - from_x) * t;
let iy = from_y + (to_y - from_y) * t;
let motion_result = tokio::task::spawn_blocking(move || {
- crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button))
- }).await;
+ crate::input::send_motion(
+ xid,
+ ix.round() as i32,
+ iy.round() as i32,
+ Some(button),
+ )
+ })
+ .await;
match motion_result {
Ok(Ok(())) => {
if let Ok(Ok((sx, sy))) =
- tokio::task::spawn_blocking(move || window_local_to_screen(xid, ix, iy)).await
- {
- let heading = if (ix - prev_x).abs() > f64::EPSILON
- || (iy - prev_y).abs() > f64::EPSILON
+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, ix, iy))
+ .await
{
- Some((iy - prev_y).atan2(ix - prev_x))
- } else {
- None
- };
- self.state.cursor_registry.update_position(&cursor_id, sx, sy);
- overlay_move_to_for(&cursor_id, sx, sy, heading);
- }
- prev_x = ix;
- prev_y = iy;
- if step_delay_ms > 0 {
- tokio::time::sleep(std::time::Duration::from_millis(step_delay_ms)).await;
+ let heading = if (ix - prev_x).abs() > f64::EPSILON
+ || (iy - prev_y).abs() > f64::EPSILON
+ {
+ Some((iy - prev_y).atan2(ix - prev_x))
+ } else {
+ None
+ };
+ self.state
+ .cursor_registry
+ .update_position(&cursor_id, sx, sy);
+ overlay_move_to_for(&cursor_id, sx, sy, heading);
+ }
+ prev_x = ix;
+ prev_y = iy;
+ if step_delay_ms > 0 {
+ tokio::time::sleep(std::time::Duration::from_millis(step_delay_ms))
+ .await;
}
}
Ok(Err(e)) => {
@@ -3048,7 +3568,8 @@ impl Tool for DragTool {
let release_result = tokio::task::spawn_blocking(move || {
crate::input::send_button_up(xid, to_x.round() as i32, to_y.round() as i32, button)
- }).await;
+ })
+ .await;
if result.is_ok() {
result = match release_result {
Ok(Ok(())) => Ok(()),
@@ -3065,7 +3586,9 @@ impl Tool for DragTool {
if let Ok(Ok((sx_to, sy_to))) =
tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y)).await
{
- self.state.cursor_registry.update_position(&cursor_id, sx_to, sy_to);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_id, sx_to, sy_to);
overlay_snap_to_for(
&cursor_id,
sx_to,
@@ -3121,14 +3644,24 @@ impl Tool for MouseButtonDownTool {
async fn invoke(&self, args: Value) -> ToolResult {
let cursor_id = resolve_cursor_key(&args);
- if let Some(held) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() {
+ if let Some(held) = self
+ .state
+ .mouse_hold
+ .lock()
+ .unwrap()
+ .get(&cursor_id)
+ .cloned()
+ {
return ToolResult::error(format!(
"Cursor '{cursor_id}' already has a held mouse button. Call mouse_button_up first."
))
.with_structured(mouse_hold_json(&cursor_id, Some(&held)));
}
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let xid = match args.opt_u64("window_id") {
Some(v) => v,
None => return ToolResult::error("window_id is required on Linux."),
@@ -3139,8 +3672,16 @@ impl Tool for MouseButtonDownTool {
let mut y = args.f64_or("y", 0.0);
if args.bool_or("from_zoom", false) {
match self.state.zoom_registry.get(pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; }
- None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(x, y);
+ x = wx;
+ y = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
x *= ratio;
@@ -3151,7 +3692,9 @@ impl Tool for MouseButtonDownTool {
cursor_id.clone(),
cursor_overlay::OverlayCommand::PinAbove(xid),
);
- if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await {
+ if let Ok(Ok((sx, sy))) =
+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await
+ {
overlay_glide_to_for(&cursor_id, sx, sy).await;
crate::overlay::send_command_for(
cursor_id.clone(),
@@ -3166,18 +3709,34 @@ impl Tool for MouseButtonDownTool {
// the existing input::send_button_down behaviour.
let result = if crate::wayland::is_wayland() {
let cid = cursor_id.clone();
- tokio::task::spawn_blocking(move || crate::wayland::persistent_vptr::press(&cid, xid, xi, yi, button)).await
+ tokio::task::spawn_blocking(move || {
+ crate::wayland::persistent_vptr::press(&cid, xid, xi, yi, button)
+ })
+ .await
} else {
- tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button)).await
+ tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button))
+ .await
};
match result {
Ok(Ok(())) => {
- let hold = MouseHoldState { cursor_id: cursor_id.clone(), pid, xid, button, x, y };
- self.state.mouse_hold.lock().unwrap().insert(cursor_id.clone(), hold.clone());
+ let hold = MouseHoldState {
+ pid,
+ xid,
+ button,
+ x,
+ y,
+ };
+ self.state
+ .mouse_hold
+ .lock()
+ .unwrap()
+ .insert(cursor_id.clone(), hold.clone());
if let Ok(Ok((sx, sy))) =
tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await
{
- self.state.cursor_registry.update_position(&cursor_id, sx, sy);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_id, sx, sy);
overlay_snap_to_for(&cursor_id, sx, sy, None);
crate::overlay::send_command_for(
cursor_id.clone(),
@@ -3190,10 +3749,16 @@ impl Tool for MouseButtonDownTool {
))
.with_structured(mouse_hold_json(&cursor_id, Some(&hold)))
}
- Ok(Err(e)) => ToolResult::error(e.to_string())
- .with_structured(mouse_hold_json(&cursor_id, self.state.mouse_hold.lock().unwrap().get(&cursor_id))),
- Err(e) => ToolResult::error(format!("Task error: {e}"))
- .with_structured(mouse_hold_json(&cursor_id, self.state.mouse_hold.lock().unwrap().get(&cursor_id))),
+ Ok(Err(e)) => ToolResult::error(e.to_string()).with_structured(mouse_hold_json(
+ &cursor_id,
+ self.state.mouse_hold.lock().unwrap().get(&cursor_id),
+ )),
+ Err(e) => {
+ ToolResult::error(format!("Task error: {e}")).with_structured(mouse_hold_json(
+ &cursor_id,
+ self.state.mouse_hold.lock().unwrap().get(&cursor_id),
+ ))
+ }
}
}
}
@@ -3228,7 +3793,14 @@ impl Tool for MouseDragTool {
async fn invoke(&self, args: Value) -> ToolResult {
let cursor_id = resolve_cursor_key(&args);
- let Some(mut hold) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() else {
+ let Some(mut hold) = self
+ .state
+ .mouse_hold
+ .lock()
+ .unwrap()
+ .get(&cursor_id)
+ .cloned()
+ else {
return ToolResult::error(format!(
"No mouse button is currently held for cursor '{cursor_id}'. Call mouse_button_down first."
))
@@ -3242,9 +3814,18 @@ impl Tool for MouseDragTool {
let mut to_y = args.f64_or("y", 0.0);
if args.bool_or("from_zoom", false) {
match self.state.zoom_registry.get(hold.pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(to_x, to_y); to_x = wx; to_y = wy; }
- None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {}. Call zoom first.", hold.pid))
- .with_structured(mouse_hold_json(&cursor_id, Some(&hold))),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(to_x, to_y);
+ to_x = wx;
+ to_y = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {}. Call zoom first.",
+ hold.pid
+ ))
+ .with_structured(mouse_hold_json(&cursor_id, Some(&hold)))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid) {
to_x *= ratio;
@@ -3261,9 +3842,13 @@ impl Tool for MouseDragTool {
cursor_id.clone(),
cursor_overlay::OverlayCommand::PinAbove(xid),
);
- if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await {
+ if let Ok(Ok((sx, sy))) =
+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await
+ {
overlay_glide_to_for(&cursor_id, sx, sy).await;
- self.state.cursor_registry.update_position(&cursor_id, sx, sy);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_id, sx, sy);
overlay_snap_to_for(&cursor_id, sx, sy, None);
crate::overlay::send_command_for(
cursor_id.clone(),
@@ -3272,7 +3857,11 @@ impl Tool for MouseDragTool {
}
let button = hold.button;
- let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };
+ let step_delay_ms = if steps > 1 {
+ duration_ms / steps as u64
+ } else {
+ duration_ms
+ };
let mut result: anyhow::Result<()> = Ok(());
let mut prev_x = from_x;
let mut prev_y = from_y;
@@ -3287,24 +3876,40 @@ impl Tool for MouseDragTool {
let cid_inner = cursor_id.clone();
let move_result = if is_wl {
tokio::task::spawn_blocking(move || {
- crate::wayland::persistent_vptr::move_to(&cid_inner, ix.round() as i32, iy.round() as i32)
- }).await
+ crate::wayland::persistent_vptr::move_to(
+ &cid_inner,
+ ix.round() as i32,
+ iy.round() as i32,
+ )
+ })
+ .await
} else {
tokio::task::spawn_blocking(move || {
- crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button))
- }).await
+ crate::input::send_motion(
+ xid,
+ ix.round() as i32,
+ iy.round() as i32,
+ Some(button),
+ )
+ })
+ .await
};
match move_result {
Ok(Ok(())) => {
if let Ok(Ok((sx, sy))) =
- tokio::task::spawn_blocking(move || window_local_to_screen(xid, ix, iy)).await
+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, ix, iy))
+ .await
{
- let heading = if (ix - prev_x).abs() > f64::EPSILON || (iy - prev_y).abs() > f64::EPSILON {
+ let heading = if (ix - prev_x).abs() > f64::EPSILON
+ || (iy - prev_y).abs() > f64::EPSILON
+ {
Some((iy - prev_y).atan2(ix - prev_x))
} else {
None
};
- self.state.cursor_registry.update_position(&cursor_id, sx, sy);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_id, sx, sy);
overlay_move_to_for(&cursor_id, sx, sy, heading);
}
prev_x = ix;
@@ -3328,10 +3933,24 @@ impl Tool for MouseDragTool {
Ok(()) => {
hold.x = to_x;
hold.y = to_y;
- self.state.mouse_hold.lock().unwrap().insert(cursor_id.clone(), hold.clone());
- if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y)).await {
- self.state.cursor_registry.update_position(&cursor_id, sx, sy);
- overlay_snap_to_for(&cursor_id, sx, sy, Some((to_y - from_y).atan2(to_x - from_x)));
+ self.state
+ .mouse_hold
+ .lock()
+ .unwrap()
+ .insert(cursor_id.clone(), hold.clone());
+ if let Ok(Ok((sx, sy))) =
+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y))
+ .await
+ {
+ self.state
+ .cursor_registry
+ .update_position(&cursor_id, sx, sy);
+ overlay_snap_to_for(
+ &cursor_id,
+ sx,
+ sy,
+ Some((to_y - from_y).atan2(to_x - from_x)),
+ );
crate::overlay::send_command_for(
cursor_id.clone(),
cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },
@@ -3376,9 +3995,18 @@ impl Tool for MouseButtonUpTool {
async fn invoke(&self, args: Value) -> ToolResult {
let cursor_id = resolve_cursor_key(&args);
- let Some(mut hold) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() else {
- return ToolResult::error(format!("No mouse button is currently held for cursor '{cursor_id}'."))
- .with_structured(mouse_hold_json(&cursor_id, None));
+ let Some(hold) = self
+ .state
+ .mouse_hold
+ .lock()
+ .unwrap()
+ .get(&cursor_id)
+ .cloned()
+ else {
+ return ToolResult::error(format!(
+ "No mouse button is currently held for cursor '{cursor_id}'."
+ ))
+ .with_structured(mouse_hold_json(&cursor_id, None));
};
if let Some(err) = held_target_mismatch(&args, &cursor_id, &hold) {
return err;
@@ -3390,9 +4018,18 @@ impl Tool for MouseButtonUpTool {
let mut y = args.opt_f64("y").unwrap_or(hold.y);
if args.bool_or("from_zoom", false) {
match self.state.zoom_registry.get(hold.pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; }
- None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {}. Call zoom first.", hold.pid))
- .with_structured(mouse_hold_json(&cursor_id, Some(&hold))),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(x, y);
+ x = wx;
+ y = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {}. Call zoom first.",
+ hold.pid
+ ))
+ .with_structured(mouse_hold_json(&cursor_id, Some(&hold)))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid) {
x *= ratio;
@@ -3403,7 +4040,9 @@ impl Tool for MouseButtonUpTool {
cursor_id.clone(),
cursor_overlay::OverlayCommand::PinAbove(xid),
);
- if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await {
+ if let Ok(Ok((sx, sy))) =
+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await
+ {
overlay_glide_to_for(&cursor_id, sx, sy).await;
}
@@ -3415,18 +4054,22 @@ impl Tool for MouseButtonUpTool {
// (single logical drag rather than a click pair).
let result = if crate::wayland::is_wayland() {
let cid = cursor_id.clone();
- tokio::task::spawn_blocking(move || crate::wayland::persistent_vptr::release(&cid, button)).await
+ tokio::task::spawn_blocking(move || {
+ crate::wayland::persistent_vptr::release(&cid, button)
+ })
+ .await
} else {
- tokio::task::spawn_blocking(move || crate::input::send_button_up(xid, xi, yi, button)).await
+ tokio::task::spawn_blocking(move || crate::input::send_button_up(xid, xi, yi, button))
+ .await
};
match result {
Ok(Ok(())) => {
- hold.x = x;
- hold.y = y;
if let Ok(Ok((sx, sy))) =
tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await
{
- self.state.cursor_registry.update_position(&cursor_id, sx, sy);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_id, sx, sy);
overlay_snap_to_for(&cursor_id, sx, sy, None);
}
crate::overlay::send_command_for(
@@ -3470,7 +4113,8 @@ async fn parallel_drag_inject(args: &Value) -> ToolResult {
let Some(xid) = item.get("window_id").and_then(|v| v.as_u64()) else {
return ToolResult::error("each drag item requires window_id.");
};
- let local: Vec<(f64, f64)> = if let Some(pts) = item.get("path").and_then(|v| v.as_array()) {
+ let local: Vec<(f64, f64)> = if let Some(pts) = item.get("path").and_then(|v| v.as_array())
+ {
pts.iter()
.filter_map(|p| {
let a = p.as_array()?;
@@ -3481,23 +4125,45 @@ async fn parallel_drag_inject(args: &Value) -> ToolResult {
let g = |k: &str| item.get(k).and_then(|v| v.as_f64());
match (g("from_x"), g("from_y"), g("to_x"), g("to_y")) {
(Some(fx), Some(fy), Some(tx), Some(ty)) => vec![(fx, fy), (tx, ty)],
- _ => return ToolResult::error("each drag item requires path[] or from_x/from_y/to_x/to_y."),
+ _ => {
+ return ToolResult::error(
+ "each drag item requires path[] or from_x/from_y/to_x/to_y.",
+ )
+ }
}
};
if local.len() < 2 {
return ToolResult::error("drag path needs at least 2 points.");
}
- let steps = item.get("steps").and_then(|v| v.as_u64()).unwrap_or(60).clamp(1, 300) as usize;
- let x_button = parse_mouse_button(item.get("button").and_then(|v| v.as_str()).unwrap_or("left")) as u32;
- let app = match tokio::task::spawn_blocking(move || crate::wayland::app_id_for_window(xid)).await {
+ let steps = item
+ .get("steps")
+ .and_then(|v| v.as_u64())
+ .unwrap_or(60)
+ .clamp(1, 300) as usize;
+ let x_button = parse_mouse_button(
+ item.get("button")
+ .and_then(|v| v.as_str())
+ .unwrap_or("left"),
+ ) as u32;
+ let app = match tokio::task::spawn_blocking(move || crate::wayland::app_id_for_window(xid))
+ .await
+ {
Ok(Some(a)) => a,
_ => return ToolResult::error(format!("no Wayland app_id for window {xid}")),
};
- drags.push(crate::wayland::InjectDrag { app_id: app, idx: i, x_button, path: local, steps });
+ drags.push(crate::wayland::InjectDrag {
+ app_id: app,
+ idx: i,
+ x_button,
+ path: local,
+ steps,
+ });
}
let n = drags.len();
match tokio::task::spawn_blocking(move || crate::wayland::inject_parallel_drags(&drags)).await {
- Ok(Ok(())) => ToolResult::text(format!("Ran {n} concurrent drags (multi-cursor via EIS compositor).")),
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Ran {n} concurrent drags (multi-cursor via EIS compositor)."
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -3549,7 +4215,7 @@ impl Tool for ParallelMouseDragTool {
return ToolResult::error(
"parallel_mouse_drag requires the cua-compositor inject socket on Wayland \
(set CUA_INJECT_SOCKET to the cua-compositor control socket), \
- or run the target under X11."
+ or run the target under X11.",
);
}
match tokio::task::spawn_blocking(crate::input::check_parallel_pointer_support).await {
@@ -3578,7 +4244,9 @@ impl Tool for ParallelMouseDragTool {
// `path` of [x,y] points, a function `fn` (y = f(x) sampled over
// [x_from, x_to]), or a straight from→to segment.
let is_fn = item.get("fn").and_then(|v| v.as_str()).is_some();
- let local: Vec<(f64, f64)> = if let Some(pts) = item.get("path").and_then(|v| v.as_array()) {
+ let local: Vec<(f64, f64)> = if let Some(pts) =
+ item.get("path").and_then(|v| v.as_array())
+ {
let mut out = Vec::with_capacity(pts.len());
for p in pts {
let a = p.as_array();
@@ -3601,7 +4269,11 @@ impl Tool for ParallelMouseDragTool {
let Some(x_to) = item.get("x_to").and_then(|v| v.as_f64()) else {
return ToolResult::error("`fn` requires x_to.");
};
- let samples = item.get("samples").and_then(|v| v.as_u64()).unwrap_or(80).clamp(2, 400);
+ let samples = item
+ .get("samples")
+ .and_then(|v| v.as_u64())
+ .unwrap_or(80)
+ .clamp(2, 400);
match crate::input::sample_function(expr_str, x_from, x_to, samples) {
Ok(pts) => pts,
Err(e) => return ToolResult::error(e.to_string()),
@@ -3614,38 +4286,69 @@ impl Tool for ParallelMouseDragTool {
}
};
- let button = parse_mouse_button(item.get("button").and_then(|v| v.as_str()).unwrap_or("left"));
- let duration_ms = item.get("duration_ms").and_then(|v| v.as_u64())
+ let button = parse_mouse_button(
+ item.get("button")
+ .and_then(|v| v.as_str())
+ .unwrap_or("left"),
+ );
+ let duration_ms = item
+ .get("duration_ms")
+ .and_then(|v| v.as_u64())
.unwrap_or(if is_fn { 1500 } else { 500 });
// One translate gives the window origin; the path is a pure offset.
- let origin = match tokio::task::spawn_blocking(move || window_local_to_screen(xid, 0.0, 0.0)).await {
- Ok(Ok(o)) => o,
- Ok(Err(e)) => return ToolResult::error(e.to_string()),
- Err(e) => return ToolResult::error(format!("Task error: {e}")),
- };
- let path: Vec<(i32, i32)> = local.iter()
- .map(|(lx, ly)| ((origin.0 + lx).round() as i32, (origin.1 + ly).round() as i32))
+ let origin =
+ match tokio::task::spawn_blocking(move || window_local_to_screen(xid, 0.0, 0.0))
+ .await
+ {
+ Ok(Ok(o)) => o,
+ Ok(Err(e)) => return ToolResult::error(e.to_string()),
+ Err(e) => return ToolResult::error(format!("Task error: {e}")),
+ };
+ let path: Vec<(i32, i32)> = local
+ .iter()
+ .map(|(lx, ly)| {
+ (
+ (origin.0 + lx).round() as i32,
+ (origin.1 + ly).round() as i32,
+ )
+ })
.collect();
// Default sub-step count scaled to path length (smooth glide),
// overridable via `steps`.
- let total_len: f64 = path.windows(2)
- .map(|w| (((w[1].0 - w[0].0) as f64).powi(2) + ((w[1].1 - w[0].1) as f64).powi(2)).sqrt())
+ let total_len: f64 = path
+ .windows(2)
+ .map(|w| {
+ (((w[1].0 - w[0].0) as f64).powi(2) + ((w[1].1 - w[0].1) as f64).powi(2)).sqrt()
+ })
.sum();
- let steps = item.get("steps").and_then(|v| v.as_u64())
+ let steps = item
+ .get("steps")
+ .and_then(|v| v.as_u64())
.map(|s| (s as usize).clamp(1, 300))
.unwrap_or_else(|| ((total_len / 3.0).round() as usize).clamp(24, 300));
let start = path[0];
- self.state.cursor_registry.update_position(session, start.0 as f64, start.1 as f64);
- crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::PinAbove(xid));
- crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SnapTo {
- x: start.0 as f64,
- y: start.1 as f64,
- heading_radians: None,
- });
- crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(true));
+ self.state
+ .cursor_registry
+ .update_position(session, start.0 as f64, start.1 as f64);
+ crate::overlay::send_command_for(
+ session.to_owned(),
+ cursor_overlay::OverlayCommand::PinAbove(xid),
+ );
+ crate::overlay::send_command_for(
+ session.to_owned(),
+ cursor_overlay::OverlayCommand::SnapTo {
+ x: start.0 as f64,
+ y: start.1 as f64,
+ heading_radians: None,
+ },
+ );
+ crate::overlay::send_command_for(
+ session.to_owned(),
+ cursor_overlay::OverlayCommand::SetPressed(true),
+ );
drags.push((
session.to_owned(),
@@ -3660,37 +4363,62 @@ impl Tool for ParallelMouseDragTool {
}
let drags_for_task = drags.clone();
- let result = tokio::task::spawn_blocking(move || crate::input::send_parallel_virtual_pointer_drags(&drags_for_task)).await;
+ let result = tokio::task::spawn_blocking(move || {
+ crate::input::send_parallel_virtual_pointer_drags(&drags_for_task)
+ })
+ .await;
match result {
Ok(Ok(())) => {
for (session, drag) in &drags {
let n = drag.path.len();
let end = drag.path[n - 1];
let prev = drag.path[n.saturating_sub(2)];
- self.state.cursor_registry.update_position(session, end.0 as f64, end.1 as f64);
- crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SnapTo {
- x: end.0 as f64,
- y: end.1 as f64,
- heading_radians: Some(((end.1 - prev.1) as f64).atan2((end.0 - prev.0) as f64)),
- });
- crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false));
- crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::ClickPulse {
- x: end.0 as f64,
- y: end.1 as f64,
- });
+ self.state
+ .cursor_registry
+ .update_position(session, end.0 as f64, end.1 as f64);
+ crate::overlay::send_command_for(
+ session.to_owned(),
+ cursor_overlay::OverlayCommand::SnapTo {
+ x: end.0 as f64,
+ y: end.1 as f64,
+ heading_radians: Some(
+ ((end.1 - prev.1) as f64).atan2((end.0 - prev.0) as f64),
+ ),
+ },
+ );
+ crate::overlay::send_command_for(
+ session.to_owned(),
+ cursor_overlay::OverlayCommand::SetPressed(false),
+ );
+ crate::overlay::send_command_for(
+ session.to_owned(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: end.0 as f64,
+ y: end.1 as f64,
+ },
+ );
}
- ToolResult::text(format!("✅ Ran {} MPX drag gesture(s) concurrently.", drags.len()))
- .with_structured(json!({"count": drags.len()}))
+ ToolResult::text(format!(
+ "✅ Ran {} MPX drag gesture(s) concurrently.",
+ drags.len()
+ ))
+ .with_structured(json!({"count": drags.len()}))
}
Ok(Err(e)) => {
for (session, _) in &drags {
- crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false));
+ crate::overlay::send_command_for(
+ session.to_owned(),
+ cursor_overlay::OverlayCommand::SetPressed(false),
+ );
}
ToolResult::error(e.to_string())
}
Err(e) => {
for (session, _) in &drags {
- crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false));
+ crate::overlay::send_command_for(
+ session.to_owned(),
+ cursor_overlay::OverlayCommand::SetPressed(false),
+ );
}
ToolResult::error(format!("Task error: {e}"))
}
@@ -3710,9 +4438,13 @@ impl Tool for GetScreenSizeTool {
name: "get_screen_size".into(),
description: "Return the logical size of the main display in points plus its backing \
scale factor. Agents click in points; Retina displays have scale_factor 2.0. \
- Requires no TCC permissions.".into(),
+ Requires no TCC permissions."
+ .into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
@@ -3723,11 +4455,14 @@ impl Tool for GetScreenSizeTool {
// `xrandr --query` for true scale.
let (w, h) = x11_screen_size()?;
Ok::<(u32, u32, f64), anyhow::Error>((w, h, 1.0))
- }).await;
+ })
+ .await;
match result {
// Matches Swift text format 1:1.
- Ok(Ok((w, h, scale))) => ToolResult::text(format!("✅ Main display: {w}x{h} points @ {scale}x"))
- .with_structured(json!({ "width": w, "height": h, "scale_factor": scale })),
+ Ok(Ok((w, h, scale))) => {
+ ToolResult::text(format!("✅ Main display: {w}x{h} points @ {scale}x"))
+ .with_structured(json!({ "width": w, "height": h, "scale_factor": scale }))
+ }
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -3837,9 +4572,14 @@ impl Tool for GetDesktopStateTool {
None
};
use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
- let b64 = if written.is_some() { None } else { Some(B64.encode(&png)) };
+ let b64 = if written.is_some() {
+ None
+ } else {
+ Some(B64.encode(&png))
+ };
Ok((b64, shot_w, shot_h, screen_w, screen_h, written))
- }).await;
+ })
+ .await;
match result {
Ok(Ok((b64_opt, shot_w, shot_h, screen_w, screen_h, written))) => {
@@ -3865,7 +4605,11 @@ impl Tool for GetDesktopStateTool {
"✅ Desktop screenshot {shot_w}x{shot_h} (screen {screen_w}x{screen_h})"
)));
}
- ToolResult { content, is_error: None, structured_content: Some(structured) }
+ ToolResult {
+ content,
+ is_error: None,
+ structured_content: Some(structured),
+ }
}
Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")),
Err(e) => ToolResult::error(format!("Task error: {e}")),
@@ -3883,9 +4627,14 @@ impl Tool for GetCursorPositionTool {
fn def(&self) -> &ToolDef {
GCP_DEF.get_or_init(|| ToolDef {
name: "get_cursor_position".into(),
- description: "Return the current mouse cursor position in screen points (origin top-left).".into(),
+ description:
+ "Return the current mouse cursor position in screen points (origin top-left)."
+ .into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
@@ -3912,7 +4661,8 @@ impl Tool for GetCursorPositionTool {
let root = conn.setup().roots[screen_num].root;
let reply = conn.query_pointer(root)?.reply()?;
Ok::<(i32, i32), anyhow::Error>((reply.root_x as i32, reply.root_y as i32))
- }).await;
+ })
+ .await;
match result {
// Text format matches Swift `GetCursorPositionTool` 1:1.
Ok(Ok((x, y))) => ToolResult::text(format!("✅ Cursor at ({x}, {y})"))
@@ -3968,14 +4718,20 @@ impl Tool for MoveCursorTool {
let real_warp_note = if crate::wayland::is_wayland() {
let xi = x.round() as i32;
let yi = y.round() as i32;
- match tokio::task::spawn_blocking(move || crate::wayland::move_cursor_absolute(window_id, xi, yi)).await {
+ match tokio::task::spawn_blocking(move || {
+ crate::wayland::move_cursor_absolute(window_id, xi, yi)
+ })
+ .await
+ {
Ok(Ok(())) => " (real cursor warped via virtual-pointer)",
Ok(Err(_)) | Err(_) => " (overlay updated; real-cursor warp failed)",
}
} else {
""
};
- ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).{real_warp_note}"))
+ ToolResult::text(format!(
+ "Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).{real_warp_note}"
+ ))
}
}
@@ -4001,14 +4757,20 @@ impl Tool for SetAgentCursorEnabledTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let enabled = match args.require_bool("enabled") { Ok(v) => v, Err(e) => return e };
+ let enabled = match args.require_bool("enabled") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let cursor_id = resolve_cursor_key(&args);
self.state.cursor_registry.set_enabled(&cursor_id, enabled);
crate::overlay::send_command_for(
cursor_id.clone(),
cursor_overlay::OverlayCommand::SetEnabled(enabled),
);
- ToolResult::text(format!("Agent cursor '{cursor_id}' {}.", if enabled { "enabled" } else { "disabled" }))
+ ToolResult::text(format!(
+ "Agent cursor '{cursor_id}' {}.",
+ if enabled { "enabled" } else { "disabled" }
+ ))
}
}
@@ -4057,18 +4819,32 @@ impl Tool for SetAgentCursorMotionTool {
let icon_owned = icon.clone();
match tokio::task::spawn_blocking(move || {
cursor_overlay::resolve_cursor_icon(&icon_owned)
- }).await {
- Ok(Ok(resolution)) => shape_cmd = Some(cursor_overlay::OverlayCommand::from_cursor_icon(resolution)),
+ })
+ .await
+ {
+ Ok(Ok(resolution)) => {
+ shape_cmd = Some(cursor_overlay::OverlayCommand::from_cursor_icon(resolution))
+ }
Ok(Err(e)) => return ToolResult::error(format!("Invalid cursor_icon: {e}")),
Err(e) => return ToolResult::error(format!("Task error: {e}")),
}
}
self.state.cursor_registry.update_config(&cursor_id, |cfg| {
- if let Some(v) = args.opt_str("cursor_icon") { cfg.cursor_icon = Some(v); }
- if let Some(v) = args.opt_str("cursor_color") { cfg.cursor_color = Some(v); }
- if let Some(v) = args.opt_str("cursor_label") { cfg.cursor_label = Some(v); }
- if let Some(v) = args.opt_f64("cursor_size") { cfg.cursor_size = Some(v); }
- if let Some(v) = args.opt_f64("cursor_opacity") { cfg.cursor_opacity = Some(v.clamp(0.0, 1.0)); }
+ if let Some(v) = args.opt_str("cursor_icon") {
+ cfg.cursor_icon = Some(v);
+ }
+ if let Some(v) = args.opt_str("cursor_color") {
+ cfg.cursor_color = Some(v);
+ }
+ if let Some(v) = args.opt_str("cursor_label") {
+ cfg.cursor_label = Some(v);
+ }
+ if let Some(v) = args.opt_f64("cursor_size") {
+ cfg.cursor_size = Some(v);
+ }
+ if let Some(v) = args.opt_f64("cursor_opacity") {
+ cfg.cursor_opacity = Some(v.clamp(0.0, 1.0));
+ }
});
if let Some(cmd) = shape_cmd {
crate::overlay::send_command_for(cursor_id.clone(), cmd);
@@ -4165,7 +4941,6 @@ impl Tool for SetAgentCursorStyleTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
- use cua_driver_core::tool_args::ArgsExt;
let cursor_id = resolve_cursor_key(&args);
// image_path
@@ -4177,7 +4952,9 @@ impl Tool for SetAgentCursorStyleTool {
let path_owned = path.to_owned();
match tokio::task::spawn_blocking(move || {
cursor_overlay::CursorShape::load(&path_owned)
- }).await {
+ })
+ .await
+ {
Ok(Ok(shape)) => {
let path_for_cfg = path.to_owned();
self.state.cursor_registry.update_config(&cursor_id, |cfg| {
@@ -4185,7 +4962,9 @@ impl Tool for SetAgentCursorStyleTool {
});
Some(cursor_overlay::OverlayCommand::SetShape(Some(shape)))
}
- Ok(Err(e)) => return ToolResult::error(format!("Failed to load image_path: {e}")),
+ Ok(Err(e)) => {
+ return ToolResult::error(format!("Failed to load image_path: {e}"))
+ }
Err(e) => return ToolResult::error(format!("Task error: {e}")),
}
}
@@ -4194,34 +4973,36 @@ impl Tool for SetAgentCursorStyleTool {
};
// gradient_colors
- let gradient_colors: Vec<[u8; 4]> = if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {
- let mut out = vec![];
- for v in arr {
- if let Some(hex) = v.as_str() {
- match parse_hex_color(hex) {
- Some(c) => out.push(c),
- None => return ToolResult::error(format!("Invalid hex color: {hex}")),
+ let gradient_colors: Vec<[u8; 4]> =
+ if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {
+ let mut out = vec![];
+ for v in arr {
+ if let Some(hex) = v.as_str() {
+ match parse_hex_color(hex) {
+ Some(c) => out.push(c),
+ None => return ToolResult::error(format!("Invalid hex color: {hex}")),
+ }
}
}
- }
- out
- } else {
- vec![]
- };
+ out
+ } else {
+ vec![]
+ };
// bloom_color
- let bloom_color: Option<Option<[u8; 4]>> = if let Some(hex) = args.get("bloom_color").and_then(|v| v.as_str()) {
- if hex.is_empty() {
- Some(None)
- } else {
- match parse_hex_color(hex) {
- Some(c) => Some(Some(c)),
- None => return ToolResult::error(format!("Invalid bloom_color: {hex}")),
+ let bloom_color: Option<Option<[u8; 4]>> =
+ if let Some(hex) = args.get("bloom_color").and_then(|v| v.as_str()) {
+ if hex.is_empty() {
+ Some(None)
+ } else {
+ match parse_hex_color(hex) {
+ Some(c) => Some(Some(c)),
+ None => return ToolResult::error(format!("Invalid bloom_color: {hex}")),
+ }
}
- }
- } else {
- None
- };
+ } else {
+ None
+ };
// Dispatch to overlay
if let Some(cmd) = shape_cmd {
@@ -4239,21 +5020,36 @@ impl Tool for SetAgentCursorStyleTool {
);
}
- let grad_str = args.get("gradient_colors")
+ let grad_str = args
+ .get("gradient_colors")
.and_then(|v| v.as_array())
.map(|arr| {
- let strs: Vec<String> = arr.iter()
+ let strs: Vec<String> = arr
+ .iter()
.filter_map(|v| v.as_str().map(str::to_owned))
.collect();
format!("[{}]", strs.join(", "))
})
.unwrap_or_else(|| "(unchanged)".into());
- let bloom_str = args.get("bloom_color")
+ let bloom_str = args
+ .get("bloom_color")
.and_then(|v| v.as_str())
- .map(|s| if s.is_empty() { "(reverted)".to_owned() } else { s.to_owned() })
+ .map(|s| {
+ if s.is_empty() {
+ "(reverted)".to_owned()
+ } else {
+ s.to_owned()
+ }
+ })
.unwrap_or_else(|| "(unchanged)".into());
let img_str = image_path
- .map(|s| if s.is_empty() { "(reverted to default)".to_owned() } else { s.to_owned() })
+ .map(|s| {
+ if s.is_empty() {
+ "(reverted to default)".to_owned()
+ } else {
+ s.to_owned()
+ }
+ })
.unwrap_or_else(|| "(unchanged)".into());
ToolResult::text(format!(
@@ -4293,14 +5089,19 @@ impl Tool for CheckPermissionsTool {
name: "check_permissions".into(),
description: "Check required permissions for cua-driver-rs on Linux.".into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
// Check X11 connectivity (required for window enumeration and input injection).
let x11_ok = tokio::task::spawn_blocking(|| {
x11rb::rust_connection::RustConnection::connect(None).is_ok()
- }).await.unwrap_or(false);
+ })
+ .await
+ .unwrap_or(false);
// Check AT-SPI: not merely "is there a session bus?" but "does
// org.a11y.Bus actually answer on it?" — the previous env-var-or-
@@ -4362,22 +5163,24 @@ impl Tool for GetConfigTool {
name: "get_config".into(),
description: "Return current cua-driver-rs configuration.".into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
let cfg = self.state.config.read().unwrap();
let (pip_enabled, pip_geometry) = pip_preview::read_pip_keys_from_file();
- ToolResult::text("cua-driver-rs configuration")
- .with_structured(json!({
- "version": env!("CARGO_PKG_VERSION"),
- "platform": "linux",
- "capture_mode": cfg.capture_mode,
- "capture_scope": cfg.capture_scope,
- "max_image_dimension": cfg.max_image_dimension,
- "experimental_pip": pip_enabled,
- "experimental_pip_geometry": pip_geometry
- }))
+ ToolResult::text("cua-driver-rs configuration").with_structured(json!({
+ "version": env!("CARGO_PKG_VERSION"),
+ "platform": "linux",
+ "capture_mode": cfg.capture_mode,
+ "capture_scope": cfg.capture_scope,
+ "max_image_dimension": cfg.max_image_dimension,
+ "experimental_pip": pip_enabled,
+ "experimental_pip_geometry": pip_geometry
+ }))
}
}
@@ -4422,10 +5225,9 @@ impl Tool for SetConfigTool {
// Linux previously read only the legacy per-field keys below, so a
// `{"key":"max_image_dimension","value":800}` write was silently
// dropped (issue #1923). Dispatch on `key` to the same fields.
- if let (Some(key), Some(val)) = (
- args.get("key").and_then(|v| v.as_str()),
- args.get("value"),
- ) {
+ if let (Some(key), Some(val)) =
+ (args.get("key").and_then(|v| v.as_str()), args.get("value"))
+ {
match key {
"capture_mode" => match val.as_str() {
Some(s) => {
@@ -4488,7 +5290,9 @@ impl Tool for SetConfigTool {
}
// Legacy per-field shape.
if let Some(mode) = args.opt_str("capture_mode") {
- if let Err(e) = pip_preview::write_config_key("capture_mode", Value::String(mode.clone())) {
+ if let Err(e) =
+ pip_preview::write_config_key("capture_mode", Value::String(mode.clone()))
+ {
tracing::warn!("set_config: failed to persist capture_mode: {e}");
}
parts.push(format!("capture_mode={mode}"));
@@ -4496,9 +5300,13 @@ impl Tool for SetConfigTool {
}
if let Some(scope) = args.opt_str("capture_scope") {
if scope != "window" && scope != "desktop" {
- return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\"."));
+ return ToolResult::error(format!(
+ "`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\"."
+ ));
}
- if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(scope.clone())) {
+ if let Err(e) =
+ pip_preview::write_config_key("capture_scope", Value::String(scope.clone()))
+ {
tracing::warn!("set_config: failed to persist capture_scope: {e}");
}
parts.push(format!("capture_scope={scope}"));
@@ -4512,7 +5320,8 @@ impl Tool for SetConfigTool {
parts.push(format!("max_image_dimension={dim}"));
}
if let Some(enabled) = args.get("experimental_pip").and_then(|v| v.as_bool()) {
- if let Err(e) = pip_preview::write_config_key("experimental_pip", Value::Bool(enabled)) {
+ if let Err(e) = pip_preview::write_config_key("experimental_pip", Value::Bool(enabled))
+ {
return ToolResult::error(format!("failed to persist experimental_pip: {e}"));
}
parts.push(format!("experimental_pip={enabled} (next restart)"));
@@ -4523,8 +5332,13 @@ impl Tool for SetConfigTool {
"experimental_pip_geometry `{geom}` is not a valid WxH or WxH+X+Y string"
));
}
- if let Err(e) = pip_preview::write_config_key("experimental_pip_geometry", Value::String(geom.clone())) {
- return ToolResult::error(format!("failed to persist experimental_pip_geometry: {e}"));
+ if let Err(e) = pip_preview::write_config_key(
+ "experimental_pip_geometry",
+ Value::String(geom.clone()),
+ ) {
+ return ToolResult::error(format!(
+ "failed to persist experimental_pip_geometry: {e}"
+ ));
}
parts.push(format!("experimental_pip_geometry={geom} (next restart)"));
}
@@ -4534,14 +5348,13 @@ impl Tool for SetConfigTool {
format!("Config updated: {}", parts.join(", "))
};
let (pip_enabled, pip_geometry) = pip_preview::read_pip_keys_from_file();
- ToolResult::text(msg)
- .with_structured(json!({
- "capture_mode": cfg.capture_mode,
- "capture_scope": cfg.capture_scope,
- "max_image_dimension": cfg.max_image_dimension,
- "experimental_pip": pip_enabled,
- "experimental_pip_geometry": pip_geometry
- }))
+ ToolResult::text(msg).with_structured(json!({
+ "capture_mode": cfg.capture_mode,
+ "capture_scope": cfg.capture_scope,
+ "max_image_dimension": cfg.max_image_dimension,
+ "experimental_pip": pip_enabled,
+ "experimental_pip_geometry": pip_geometry
+ }))
}
}
@@ -4559,36 +5372,55 @@ impl Tool for GetAccessibilityTreeTool {
description: "Return a lightweight snapshot of the desktop: running processes and \
on-screen visible X11 windows with their bounds and owner pid.\n\n\
For the full AT-SPI subtree of a single window (with interactive element indices \
- you can click by), use get_window_state instead — this is a fast discovery read.".into(),
+ you can click by), use get_window_state instead — this is a fast discovery read."
+ .into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
let (procs, windows) = tokio::task::spawn_blocking(|| {
- (crate::proc_fs::list_processes(), crate::x11::list_windows(None))
- }).await.unwrap_or_default();
+ (
+ crate::proc_fs::list_processes(),
+ crate::x11::list_windows(None),
+ )
+ })
+ .await
+ .unwrap_or_default();
let mut lines = vec![format!(
"{} running process(es), {} visible window(s)",
- procs.len(), windows.len()
+ procs.len(),
+ windows.len()
)];
for p in &procs {
- let cmd = if p.cmdline.is_empty() { p.name.clone() } else { p.cmdline.clone() };
+ let cmd = if p.cmdline.is_empty() {
+ p.name.clone()
+ } else {
+ p.cmdline.clone()
+ };
lines.push(format!("- {} (pid {})", cmd, p.pid));
}
if !windows.is_empty() {
lines.push(String::new());
lines.push("Windows:".to_owned());
for w in &windows {
- let title = if w.title.is_empty() { "(no title)".to_owned() }
- else { format!("\"{}\"", w.title) };
+ let title = if w.title.is_empty() {
+ "(no title)".to_owned()
+ } else {
+ format!("\"{}\"", w.title)
+ };
lines.push(format!(
"- pid={:?} {} [window_id: {}] {}x{}+{}+{}",
w.pid, title, w.xid, w.width, w.height, w.x, w.y
));
}
- lines.push("→ Call get_window_state(pid, window_id) to inspect a window's UI.".to_owned());
+ lines.push(
+ "→ Call get_window_state(pid, window_id) to inspect a window's UI.".to_owned(),
+ );
}
let structured = json!({
@@ -4632,13 +5464,30 @@ impl Tool for ZoomTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let xid = match args.require_u64("window_id") { Ok(v) => v, Err(e) => return e };
+ let xid = match args.require_u64("window_id") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let pid = args.opt_u64("pid").map(|v| v as u32);
- let x1 = match args.opt_f64("x1") { Some(v) => v, None => return ToolResult::error("Missing x1") };
- let y1 = match args.opt_f64("y1") { Some(v) => v, None => return ToolResult::error("Missing y1") };
- let x2 = match args.opt_f64("x2") { Some(v) => v, None => return ToolResult::error("Missing x2") };
- let y2 = match args.opt_f64("y2") { Some(v) => v, None => return ToolResult::error("Missing y2") };
- if x2 <= x1 || y2 <= y1 { return ToolResult::error("x2 must be > x1 and y2 must be > y1"); }
+ let x1 = match args.opt_f64("x1") {
+ Some(v) => v,
+ None => return ToolResult::error("Missing x1"),
+ };
+ let y1 = match args.opt_f64("y1") {
+ Some(v) => v,
+ None => return ToolResult::error("Missing y1"),
+ };
+ let x2 = match args.opt_f64("x2") {
+ Some(v) => v,
+ None => return ToolResult::error("Missing x2"),
+ };
+ let y2 = match args.opt_f64("y2") {
+ Some(v) => v,
+ None => return ToolResult::error("Missing y2"),
+ };
+ if x2 <= x1 || y2 <= y1 {
+ return ToolResult::error("x2 must be > x1 and y2 must be > y1");
+ }
let state = self.state.clone();
let result = tokio::task::spawn_blocking(move || {
@@ -4648,16 +5497,20 @@ impl Tool for ZoomTool {
// X11-only path with a foreign-toplevel id.
let png = crate::wayland::screenshot_window_dispatch(xid)?;
cursor_overlay::capture_utils::crop_png_to_jpeg(&png, x1, y1, x2, y2, 500)
- }).await;
+ })
+ .await;
match result {
Ok(Ok(crop)) => {
if let Some(p) = pid {
- state.zoom_registry.set(p, ZoomContext {
- origin_x: crop.origin_x,
- origin_y: crop.origin_y,
- scale_inv: crop.scale_inv,
- });
+ state.zoom_registry.set(
+ p,
+ ZoomContext {
+ origin_x: crop.origin_x,
+ origin_y: crop.origin_y,
+ scale_inv: crop.scale_inv,
+ },
+ );
}
use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
let b64 = B64.encode(&crop.jpeg_bytes);
@@ -4666,7 +5519,9 @@ impl Tool for ZoomTool {
ToolResult {
content: vec![
Content::image_jpeg(b64),
- Content::text(format!("Zoom ({x1:.0},{y1:.0})–({x2:.0},{y2:.0}) → {w}×{h} px JPEG.")),
+ Content::text(format!(
+ "Zoom ({x1:.0},{y1:.0})–({x2:.0},{y2:.0}) → {w}×{h} px JPEG."
+ )),
],
is_error: None,
// Surface 7: `mime_type` mirrors the MCP image part's `mimeType`
@@ -4713,7 +5568,10 @@ impl Tool for TypeTextCharsTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let pid = args.u64_or("pid", 0) as u32;
- let text_raw = match args.require_str("text") { Ok(v) => v, Err(e) => return e };
+ let text_raw = match args.require_str("text") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// Same trailing-protocol-tag scrub as TypeTextTool — see
// cua_driver_core::text_sanitize for rationale.
let text = cua_driver_core::text_sanitize::strip_trailing_agent_protocol_tags(&text_raw)
@@ -4723,10 +5581,17 @@ impl Tool for TypeTextCharsTool {
let xid = match xid_opt {
Some(x) => x,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.xid,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -4747,9 +5612,12 @@ impl Tool for TypeTextCharsTool {
return Ok(());
}
crate::input::send_type_text_with_delay(xid, &text, delay_ms)
- }).await;
+ })
+ .await;
match result {
- Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) with {delay_ms}ms delay.")),
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Typed {text_len} character(s) with {delay_ms}ms delay."
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -4768,7 +5636,8 @@ impl Tool for KillAppTool {
name: "kill_app".into(),
description: "Force-terminate a process by pid (kill -9 equivalent on Linux). \
Use as escalation when the cooperative close path failed to make the process \
- exit. Unsaved state is lost — prefer the cooperative path first.".into(),
+ exit. Unsaved state is lost — prefer the cooperative path first."
+ .into(),
input_schema: json!({"type":"object","required":["pid"],"properties":{
"pid":{"type":"integer","description":"PID of the process to terminate."}
},"additionalProperties":false}),
@@ -4782,8 +5651,14 @@ impl Tool for KillAppTool {
async fn invoke(&self, args: Value) -> ToolResult {
let pid_i = match args.get("pid").and_then(|v| v.as_i64()) {
Some(p) if p > 0 && p <= i32::MAX as i64 => p as i32,
- Some(_) => return ToolResult::error("kill_app: `pid` must be a positive integer".to_string()),
- None => return ToolResult::error("kill_app: missing required integer field `pid`".to_string()),
+ Some(_) => {
+ return ToolResult::error("kill_app: `pid` must be a positive integer".to_string())
+ }
+ None => {
+ return ToolResult::error(
+ "kill_app: missing required integer field `pid`".to_string(),
+ )
+ }
};
// SAFETY: libc::kill is a thin syscall wrapper, no thread-safety concerns.
let rc = unsafe { libc::kill(pid_i, libc::SIGKILL) };
@@ -4861,17 +5736,23 @@ impl Tool for BringToFrontTool {
Some(x) => x,
None => {
let pid = args.u64_or("pid", 0) as u32;
- let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid)))
- .await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.xid,
- None => return ToolResult::error(format!(
+ None => {
+ return ToolResult::error(format!(
"bring_to_front: no window_id given and no windows found for pid {pid}."
- )),
+ ))
+ }
}
}
};
- let r = tokio::task::spawn_blocking(move || crate::input::x11_activate_window_persistent(xid)).await;
+ let r =
+ tokio::task::spawn_blocking(move || crate::input::x11_activate_window_persistent(xid))
+ .await;
match r {
Ok(Ok(prior)) => ToolResult::text(format!(
"✅ Brought window {xid} to front (X11 _NET_ACTIVE_WINDOW)."
@@ -4897,28 +5778,56 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
cua_driver_core::session::register_session_end_hook(move |session_id| {
cursor_registry.remove(session_id);
crate::overlay::remove_cursor(session_id.to_owned());
- state_for_session_end.mouse_hold.lock().unwrap().remove(session_id);
+ state_for_session_end
+ .mouse_hold
+ .lock()
+ .unwrap()
+ .remove(session_id);
crate::input::forget_master_pointer(session_id);
});
}
let mut r = ToolRegistry::new();
r.register(Box::new(ListAppsTool));
r.register(Box::new(ListWindowsTool));
- r.register(Box::new(GetWindowStateTool { state: state.clone() }));
+ r.register(Box::new(GetWindowStateTool {
+ state: state.clone(),
+ }));
r.register(Box::new(LaunchAppTool));
r.register(Box::new(KillAppTool));
r.register(Box::new(BringToFrontTool));
- r.register(Box::new(ClickTool { state: state.clone() }));
- r.register(Box::new(DoubleClickTool { state: state.clone() }));
- r.register(Box::new(RightClickTool { state: state.clone() }));
- r.register(Box::new(DragTool { state: state.clone() }));
- r.register(Box::new(MouseButtonDownTool { state: state.clone() }));
- r.register(Box::new(MouseDragTool { state: state.clone() }));
- r.register(Box::new(MouseButtonUpTool { state: state.clone() }));
- r.register(Box::new(ParallelMouseDragTool { state: state.clone() }));
- r.register(Box::new(TypeTextTool { state: state.clone() }));
- r.register(Box::new(PressKeyTool { state: state.clone() }));
- r.register(Box::new(HotkeyTool { state: state.clone() }));
+ r.register(Box::new(ClickTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(DoubleClickTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(RightClickTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(DragTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(MouseButtonDownTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(MouseDragTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(MouseButtonUpTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(ParallelMouseDragTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(TypeTextTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(PressKeyTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(HotkeyTool {
+ state: state.clone(),
+ }));
r.register(Box::new(SetValueTool));
r.register(Box::new(ScrollTool));
// `screenshot` removed - see the matching comment in
@@ -4928,28 +5837,46 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
r.register(Box::new(GetScreenSizeTool));
r.register(Box::new(GetDesktopStateTool));
r.register(Box::new(GetCursorPositionTool));
- r.register(Box::new(MoveCursorTool { state: state.clone() }));
- r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() }));
- r.register(Box::new(SetAgentCursorMotionTool { state: state.clone() }));
- r.register(Box::new(GetAgentCursorStateTool { state: state.clone() }));
- r.register(Box::new(SetAgentCursorStyleTool { state: state.clone() }));
+ r.register(Box::new(MoveCursorTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetAgentCursorEnabledTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetAgentCursorMotionTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(GetAgentCursorStateTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetAgentCursorStyleTool {
+ state: state.clone(),
+ }));
r.register(Box::new(CheckPermissionsTool));
// `health_report` — single-call cross-platform driver diagnostics.
// Stable schema_version="1" contract for downstream consumers. Linux skips
// tcc_* and bundle_identity with "not applicable on Linux".
- r.register(Box::new(cua_driver_core::health_report::HealthReportTool::new(
- std::sync::Arc::new(crate::health_report::LinuxHealthProvider),
- )));
- r.register(Box::new(GetConfigTool { state: state.clone() }));
- r.register(Box::new(SetConfigTool { state: state.clone() }));
+ r.register(Box::new(
+ cua_driver_core::health_report::HealthReportTool::new(std::sync::Arc::new(
+ crate::health_report::LinuxHealthProvider,
+ )),
+ ));
+ r.register(Box::new(GetConfigTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetConfigTool {
+ state: state.clone(),
+ }));
r.register(Box::new(GetAccessibilityTreeTool));
- r.register(Box::new(ZoomTool { state: state.clone() }));
+ r.register(Box::new(ZoomTool {
+ state: state.clone(),
+ }));
r.register(Box::new(TypeTextCharsTool));
// Cross-platform `page` tool definition lives in mcp-server; Linux plugs
// in its AT-SPI + CDP backend here.
- r.register(Box::new(cua_driver_core::page::PageTool::new(
- Arc::new(super::page::LinuxPageBackend::new()),
- )));
+ r.register(Box::new(cua_driver_core::page::PageTool::new(Arc::new(
+ super::page::LinuxPageBackend::new(),
+ ))));
r.register_recording_tools();
r.register_session_tools();
r
@@ -4965,7 +5892,9 @@ mod click_button_schema_tests {
/// pre-Surface-5; this freezes the schema shape so the contract can't drift.
#[test]
fn schema_advertises_button_enum_and_description() {
- let tool = ClickTool { state: super::ToolState::new() };
+ let tool = ClickTool {
+ state: super::ToolState::new(),
+ };
let d = tool.def();
let props = d.input_schema.get("properties").expect("properties");
let button = props.get("button").expect("button field present");
@@ -4986,7 +5915,10 @@ mod click_button_schema_tests {
.expect("button.description present");
let lc = desc.to_ascii_lowercase();
assert!(lc.contains("left"), "description should mention default");
- assert!(lc.contains("wayland"), "description should call out wayland fallback");
+ assert!(
+ lc.contains("wayland"),
+ "description should call out wayland fallback"
+ );
}
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
index 28f929e078..f214fa9e54 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
@@ -10,10 +10,10 @@
//! `wayland-protocols-wlr`; until then `screenshot_window_dispatch` returns a
//! typed error on pure Wayland.
+pub mod ext_screencopy;
+pub mod overlay;
pub mod persistent_vptr;
pub mod portal_screenshot;
-pub mod overlay;
-pub mod ext_screencopy;
pub mod shell_helper;
// `portal_screencast` (PipeWire per-window capture) and `libei` (GNOME/KDE
// input via xdg-desktop-portal RemoteDesktop) need libpipewire-0.3 and reis
@@ -23,9 +23,9 @@ pub mod shell_helper;
// Nix build (which already has modern PipeWire + libei from nixpkgs)
// enables it. Wlroots screencopy + virtual-pointer remain unconditional.
#[cfg(feature = "portal-libei")]
-pub mod portal_screencast;
-#[cfg(feature = "portal-libei")]
pub mod libei;
+#[cfg(feature = "portal-libei")]
+pub mod portal_screencast;
/// Whether this binary was compiled with the `portal-libei` feature — the
/// xdg-desktop-portal RemoteDesktop + libei input path. It is the ONLY input
@@ -137,7 +137,10 @@ fn wl_sockets(dir: &str) -> std::collections::HashSet<String> {
.flatten()
.flatten()
.filter_map(|e| e.file_name().into_string().ok())
- .filter(|n| n.strip_prefix("wayland-").is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit())))
+ .filter(|n| {
+ n.strip_prefix("wayland-")
+ .is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()))
+ })
.collect()
}
@@ -185,7 +188,9 @@ pub fn ensure_nested_session() {
break;
}
if std::time::Instant::now() >= deadline {
- tracing::error!("cua nested compositor '{comp}': no Wayland socket appeared");
+ tracing::error!(
+ "cua nested compositor '{comp}': no Wayland socket appeared"
+ );
break;
}
std::thread::sleep(std::time::Duration::from_millis(200));
@@ -245,24 +250,38 @@ impl Dispatch<wl_registry::WlRegistry, ()> for State {
_: &Connection,
qh: &QueueHandle<Self>,
) {
- if let wl_registry::Event::Global { name, interface, version } = event {
+ if let wl_registry::Event::Global {
+ name,
+ interface,
+ version,
+ } = event
+ {
if interface == ZwlrForeignToplevelManagerV1::interface().name {
let v = version.min(3);
- state.manager = Some(registry.bind::<ZwlrForeignToplevelManagerV1, _, _>(name, v, qh, ()));
+ state.manager =
+ Some(registry.bind::<ZwlrForeignToplevelManagerV1, _, _>(name, v, qh, ()));
} else if interface == WlSeat::interface().name {
let v = version.min(7);
state.seat = Some(registry.bind::<WlSeat, _, _>(name, v, qh, ()));
} else if interface == ZwlrVirtualPointerManagerV1::interface().name {
- state.vptr_manager =
- Some(registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(name, version.min(2), qh, ()));
+ state.vptr_manager = Some(registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(
+ name,
+ version.min(2),
+ qh,
+ (),
+ ));
} else if interface == WlOutput::interface().name {
let out = registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ());
if state.output.is_none() {
state.output = Some(out);
}
} else if interface == ZwlrScreencopyManagerV1::interface().name {
- state.scrcopy_manager =
- Some(registry.bind::<ZwlrScreencopyManagerV1, _, _>(name, version.min(3), qh, ()));
+ state.scrcopy_manager = Some(registry.bind::<ZwlrScreencopyManagerV1, _, _>(
+ name,
+ version.min(3),
+ qh,
+ (),
+ ));
} else if interface == WlShm::interface().name {
state.shm = Some(registry.bind::<WlShm, _, _>(name, version.min(1), qh, ()));
}
@@ -386,7 +405,12 @@ impl Dispatch<ZwlrScreencopyFrameV1, ()> for State {
_: &QueueHandle<Self>,
) {
match event {
- scrcopy_frame::Event::Buffer { format, width, height, stride } => {
+ scrcopy_frame::Event::Buffer {
+ format,
+ width,
+ height,
+ stride,
+ } => {
if let WEnum::Value(fmt) = format {
state.capture.format = Some(fmt as u32);
}
@@ -481,7 +505,15 @@ pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {
} else {
format!("{} [{}]", tl.title, tl.app_id)
};
- out.push(WindowInfo { xid: *id as u64, pid: None, title, x: 0, y: 0, width: 0, height: 0 });
+ out.push(WindowInfo {
+ xid: *id as u64,
+ pid: None,
+ title,
+ x: 0,
+ y: 0,
+ width: 0,
+ height: 0,
+ });
}
Ok(out)
}
@@ -507,7 +539,9 @@ pub fn screenshot_bytes() -> anyhow::Result<Vec<u8>> {
/// Shell out to `grim -t png -` — the wlroots reference screenshot tool. Kept
/// as the last-resort fallback for compositors that hide screencopy.
fn capture_via_grim() -> anyhow::Result<Vec<u8>> {
- let out = std::process::Command::new("grim").args(["-t", "png", "-"]).output()?;
+ let out = std::process::Command::new("grim")
+ .args(["-t", "png", "-"])
+ .output()?;
if !out.status.success() {
anyhow::bail!("grim failed: {}", String::from_utf8_lossy(&out.stderr));
}
@@ -614,7 +648,11 @@ fn capture_via_screencopy() -> anyhow::Result<Vec<u8>> {
let raw = unsafe { std::slice::from_raw_parts(mmap_ptr as *const u8, mmap_len) };
let mut rgba = Vec::with_capacity((w as usize) * (h as usize) * 4);
for row in 0..(h as usize) {
- let src_row = if state.capture.y_invert { (h as usize) - 1 - row } else { row };
+ let src_row = if state.capture.y_invert {
+ (h as usize) - 1 - row
+ } else {
+ row
+ };
let base = src_row * stride;
for col in 0..(w as usize) {
let px = &raw[base + col * 4..base + col * 4 + 4];
@@ -655,7 +693,10 @@ pub(crate) fn anon_shm(size: usize) -> anyhow::Result<(i32, *mut libc::c_void)>
let name = b"cua-scrcopy\0";
let fd = unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) };
if fd < 0 {
- return Err(anyhow::anyhow!("memfd_create failed: {}", std::io::Error::last_os_error()));
+ return Err(anyhow::anyhow!(
+ "memfd_create failed: {}",
+ std::io::Error::last_os_error()
+ ));
}
let rc = unsafe { libc::ftruncate(fd, size as libc::off_t) };
if rc != 0 {
@@ -788,8 +829,8 @@ pub fn screenshot_window_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
/// pointer op (click, scroll, drag) needs. Returned by [`open_vptr_session`].
pub struct VptrSession {
pub conn: Connection,
- pub queue: wayland_client::EventQueue<State>,
- pub state: State,
+ queue: wayland_client::EventQueue<State>,
+ state: State,
pub seat: WlSeat,
pub vptr: ZwlrVirtualPointerV1,
pub output_w: u32,
@@ -817,10 +858,9 @@ pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<Vptr
queue.roundtrip(&mut state)?;
}
- let seat = state
- .seat
- .clone()
- .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input"))?;
+ let seat = state.seat.clone().ok_or_else(|| {
+ anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input")
+ })?;
let mgr = state.vptr_manager.clone().ok_or_else(|| {
if PORTAL_LIBEI_ENABLED {
anyhow::anyhow!("compositor does not expose zwlr_virtual_pointer_manager_v1")
@@ -850,7 +890,15 @@ pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<Vptr
let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ());
let (output_w, output_h) = (state.output_w.max(1), state.output_h.max(1));
- Ok(VptrSession { conn, queue, state, seat, vptr, output_w, output_h })
+ Ok(VptrSession {
+ conn,
+ queue,
+ state,
+ seat,
+ vptr,
+ output_w,
+ output_h,
+ })
}
/// Map a cua/X11 pointer button (1=left / 2=middle / 3=right) to its evdev
@@ -953,7 +1001,8 @@ fn record_synth_cursor(x: i32, y: i32) {
/// the moment the user moves their physical mouse. Callers should surface
/// `source: "synthetic"` in the structured payload.
pub fn last_synth_cursor_pos() -> Option<(i32, i32)> {
- SYNTH_CURSOR_POS.get_or_init(|| std::sync::Mutex::new(None))
+ SYNTH_CURSOR_POS
+ .get_or_init(|| std::sync::Mutex::new(None))
.lock()
.ok()
.and_then(|g| *g)
@@ -1061,9 +1110,14 @@ pub fn type_text(text: &str) -> anyhow::Result<()> {
/// Press a single named key into the focused Wayland surface via `wtype -k`.
pub fn press_key(key: &str) -> anyhow::Result<()> {
let keysym = key_to_keysym(key);
- let out = std::process::Command::new("wtype").args(["-k", &keysym]).output()?;
+ let out = std::process::Command::new("wtype")
+ .args(["-k", &keysym])
+ .output()?;
if !out.status.success() {
- anyhow::bail!("wtype -k {keysym} failed: {}", String::from_utf8_lossy(&out.stderr));
+ anyhow::bail!(
+ "wtype -k {keysym} failed: {}",
+ String::from_utf8_lossy(&out.stderr)
+ );
}
Ok(())
}
@@ -1115,9 +1169,10 @@ fn partition_modifiers(keys: &[String]) -> anyhow::Result<(Vec<String>, String)>
_ => non_mods.push(k.clone()),
}
}
- let final_key = non_mods.last().cloned().ok_or_else(|| {
- anyhow::anyhow!("hotkey requires at least one non-modifier key")
- })?;
+ let final_key = non_mods
+ .last()
+ .cloned()
+ .ok_or_else(|| anyhow::anyhow!("hotkey requires at least one non-modifier key"))?;
Ok((mods, final_key))
}
@@ -1154,7 +1209,9 @@ fn key_to_keysym(key: &str) -> String {
/// The control socket path, when running against the EIS nested compositor.
pub fn inject_socket_path() -> Option<String> {
- std::env::var("CUA_INJECT_SOCKET").ok().filter(|s| !s.is_empty())
+ std::env::var("CUA_INJECT_SOCKET")
+ .ok()
+ .filter(|s| !s.is_empty())
}
/// True when input should be routed through the EIS compositor's control socket
@@ -1171,11 +1228,15 @@ fn inject_send(lines: &[String]) -> anyhow::Result<()> {
let mut stream = None;
for _ in 0..60 {
match UnixStream::connect(&path) {
- Ok(s) => { stream = Some(s); break; }
+ Ok(s) => {
+ stream = Some(s);
+ break;
+ }
Err(_) => std::thread::sleep(std::time::Duration::from_millis(50)),
}
}
- let mut s = stream.ok_or_else(|| anyhow::anyhow!("could not connect to inject socket {path}"))?;
+ let mut s =
+ stream.ok_or_else(|| anyhow::anyhow!("could not connect to inject socket {path}"))?;
let mut buf = String::new();
for l in lines {
buf.push_str(l);
@@ -1250,13 +1311,7 @@ pub fn inject_press_key(window_id: u64, key: &str) -> anyhow::Result<()> {
/// Focus-free click into the window's surface via the nested EIS compositor.
/// Coordinates are window-local, matching the rest of the inject protocol.
-pub fn inject_click(
- window_id: u64,
- x: f64,
- y: f64,
- count: u32,
- button: u8,
-) -> anyhow::Result<()> {
+pub fn inject_click(window_id: u64, x: f64, y: f64, count: u32, button: u8) -> anyhow::Result<()> {
let app = app_id_for_window(window_id)
.ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?;
let btn = evdev_button(button as u32);
@@ -1306,8 +1361,10 @@ fn resample(path: &[(f64, f64)], steps: usize) -> Vec<(f64, f64)> {
for (i, &l) in seglen.iter().enumerate() {
if acc + l >= target || i == seglen.len() - 1 {
let f = if l > 0.0 { (target - acc) / l } else { 0.0 };
- pt = (path[i].0 + (path[i + 1].0 - path[i].0) * f,
- path[i].1 + (path[i + 1].1 - path[i].1) * f);
+ pt = (
+ path[i].0 + (path[i + 1].0 - path[i].0) * f,
+ path[i].1 + (path[i + 1].1 - path[i].1) * f,
+ );
break;
}
acc += l;
@@ -1324,15 +1381,22 @@ pub fn inject_parallel_drags(drags: &[InjectDrag]) -> anyhow::Result<()> {
if drags.is_empty() {
return Ok(());
}
- let resampled: Vec<Vec<(f64, f64)>> =
- drags.iter().map(|d| resample(&d.path, d.steps.max(1))).collect();
+ let resampled: Vec<Vec<(f64, f64)>> = drags
+ .iter()
+ .map(|d| resample(&d.path, d.steps.max(1)))
+ .collect();
let max_steps = resampled.iter().map(|p| p.len()).max().unwrap_or(0);
let mut lines = Vec::new();
// Press each cursor at its start point.
for (d, pts) in drags.iter().zip(&resampled) {
let (x, y) = pts[0];
lines.push(format!("m {} {} {x:.1} {y:.1}", d.app_id, d.idx));
- lines.push(format!("b {} {} {} 1", d.app_id, d.idx, evdev_button(d.x_button)));
+ lines.push(format!(
+ "b {} {} {} 1",
+ d.app_id,
+ d.idx,
+ evdev_button(d.x_button)
+ ));
}
// Glide all cursors together, one interleaved step at a time.
for s in 1..max_steps {
@@ -1343,7 +1407,12 @@ pub fn inject_parallel_drags(drags: &[InjectDrag]) -> anyhow::Result<()> {
}
// Release each cursor.
for (d, _) in drags.iter().zip(&resampled) {
- lines.push(format!("b {} {} {} 0", d.app_id, d.idx, evdev_button(d.x_button)));
+ lines.push(format!(
+ "b {} {} {} 0",
+ d.app_id,
+ d.idx,
+ evdev_button(d.x_button)
+ ));
}
inject_send(&lines)
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs
index 7ffac0a279..ea87168be4 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs
@@ -24,7 +24,7 @@ use std::thread;
use std::time::Instant;
use crossbeam_channel::{bounded, Receiver, Sender};
-use cursor_overlay::{CursorConfig, OverlayCommand, OverlayMsg, CursorKey, RenderStateCore};
+use cursor_overlay::{CursorConfig, OverlayCommand, OverlayMsg, RenderStateCore};
use wayland_client::{
protocol::{
wl_buffer::WlBuffer,
@@ -39,7 +39,7 @@ use wayland_client::{
Connection, Dispatch, Proxy, QueueHandle,
};
use wayland_protocols_wlr::layer_shell::v1::client::{
- zwlr_layer_shell_v1::{self, Layer, ZwlrLayerShellV1},
+ zwlr_layer_shell_v1::{Layer, ZwlrLayerShellV1},
zwlr_layer_surface_v1::{self, Anchor, KeyboardInteractivity, ZwlrLayerSurfaceV1},
};
@@ -48,8 +48,8 @@ use wayland_protocols_wlr::layer_shell::v1::client::{
/// SetPressed) are forwarded as-is so the layer-shell overlay matches the
/// X11 visual: bloom + animated arrow + click pulse + press ring.
enum WlOverlayCmd {
- Cmd { key: CursorKey, cmd: OverlayCommand },
- Remove { key: CursorKey },
+ Cmd { cmd: OverlayCommand },
+ Remove,
Shutdown,
}
@@ -91,7 +91,8 @@ pub fn forward(msg: &OverlayMsg) -> bool {
let Some(tx) = tx() else { return false };
match msg {
OverlayMsg::Remove(k) => {
- let _ = tx.try_send(WlOverlayCmd::Remove { key: k.clone() });
+ let _ = k;
+ let _ = tx.try_send(WlOverlayCmd::Remove);
true
}
OverlayMsg::Cmd(kc) => {
@@ -99,7 +100,6 @@ pub fn forward(msg: &OverlayMsg) -> bool {
return false;
}
let _ = tx.try_send(WlOverlayCmd::Cmd {
- key: kc.key.clone(),
cmd: kc.cmd.clone(),
});
true
@@ -242,7 +242,10 @@ fn owner_thread(rx: Receiver<WlOverlayCmd>) -> anyhow::Result<()> {
if !state.configured {
anyhow::bail!("layer surface never received configure event");
}
- dbg(&format!("configured: w={} h={}", state.output_w, state.output_h));
+ dbg(&format!(
+ "configured: w={} h={}",
+ state.output_w, state.output_h
+ ));
// Main loop. Tick the render core at ~60Hz so motion + spring physics
// + click pulse animate smoothly; redraw every tick when the cursor is
@@ -258,8 +261,11 @@ fn owner_thread(rx: Receiver<WlOverlayCmd>) -> anyhow::Result<()> {
let mut shutdown = false;
loop {
match rx.try_recv() {
- Ok(WlOverlayCmd::Shutdown) => { shutdown = true; break; }
- Ok(WlOverlayCmd::Cmd { cmd, .. }) => {
+ Ok(WlOverlayCmd::Shutdown) => {
+ shutdown = true;
+ break;
+ }
+ Ok(WlOverlayCmd::Cmd { cmd }) => {
// Seed: if the cursor is still at the off-screen sentinel
// `(-200, -200)` from `RenderStateCore::new`, snap to a
// point near the MoveTo / SnapTo target so the spring
@@ -286,17 +292,22 @@ fn owner_thread(rx: Receiver<WlOverlayCmd>) -> anyhow::Result<()> {
// _sentinel_only` are both `false` here — same as X11.
let _ = state.core.apply_command_base(cmd, false, false);
}
- Ok(WlOverlayCmd::Remove { .. }) => {
+ Ok(WlOverlayCmd::Remove) => {
// Single-cursor overlay: removing the active cursor
// hides it. Multi-cursor wlroots support can layer on
// top of this in a follow-up if needed.
state.core.visible = false;
}
Err(crossbeam_channel::TryRecvError::Empty) => break,
- Err(crossbeam_channel::TryRecvError::Disconnected) => { shutdown = true; break; }
+ Err(crossbeam_channel::TryRecvError::Disconnected) => {
+ shutdown = true;
+ break;
+ }
}
}
- if shutdown { break; }
+ if shutdown {
+ break;
+ }
// Tick animation forward and repaint if anything changed.
let now = Instant::now();
@@ -343,16 +354,22 @@ fn owner_thread(rx: Receiver<WlOverlayCmd>) -> anyhow::Result<()> {
/// When the cursor is hidden (`core.visible == false`, idle-faded, or
/// off-screen sentinel) the pixmap is all zeros — the surface remains
/// transparent and click-through.
-fn redraw(state: &mut OverlayState, shm: &WlShm, qh: &QueueHandle<OverlayState>) -> anyhow::Result<()> {
- let Some(surface) = state.surface.as_ref() else { return Ok(()) };
+fn redraw(
+ state: &mut OverlayState,
+ shm: &WlShm,
+ qh: &QueueHandle<OverlayState>,
+) -> anyhow::Result<()> {
+ let Some(surface) = state.surface.as_ref() else {
+ return Ok(());
+ };
let w = state.output_w.max(1);
let h = state.output_h.max(1);
let stride = w as i32 * 4;
let size = (stride as usize) * (h as usize);
// Reuses the same anon_shm pattern as the screencopy path in mod.rs.
- let (fd, ptr) = super::anon_shm(size)
- .map_err(|e| anyhow::anyhow!("overlay shm allocation failed: {e}"))?;
+ let (fd, ptr) =
+ super::anon_shm(size).map_err(|e| anyhow::anyhow!("overlay shm allocation failed: {e}"))?;
// SAFETY: ptr came from mmap of `size` bytes, lifetime bounded to this
// function.
@@ -397,7 +414,7 @@ fn redraw(state: &mut OverlayState, shm: &WlShm, qh: &QueueHandle<OverlayState>)
continue;
}
let off = ((py as usize) * (w as usize) + (px as usize)) * 4;
- pm.data_mut()[off] = 0xFF; // R
+ pm.data_mut()[off] = 0xFF; // R
pm.data_mut()[off + 1] = 0x00; // G
pm.data_mut()[off + 2] = 0xFF; // B
pm.data_mut()[off + 3] = 0xFF; // A
@@ -405,15 +422,15 @@ fn redraw(state: &mut OverlayState, shm: &WlShm, qh: &QueueHandle<OverlayState>)
}
}
-// RGBA → BGRA channel swap. tiny_skia stores pixels as RGBA8888
+ // RGBA → BGRA channel swap. tiny_skia stores pixels as RGBA8888
// (premultiplied); wl_shm Argb8888 is little-endian = BGRA in memory.
// Mirrors the inverse swap in ext_screencopy::encode_buffer_to_png.
let src = pm.data();
for i in (0..size).step_by(4) {
// pm.data() is already RGBA premultiplied; just swap R↔B.
- pixels[i] = src[i + 2]; // B ← R
+ pixels[i] = src[i + 2]; // B ← R
pixels[i + 1] = src[i + 1]; // G
- pixels[i + 2] = src[i]; // R ← B
+ pixels[i + 2] = src[i]; // R ← B
pixels[i + 3] = src[i + 3]; // A
}
@@ -436,7 +453,10 @@ fn redraw(state: &mut OverlayState, shm: &WlShm, qh: &QueueHandle<OverlayState>)
let buffer_id = buffer.id().protocol_id();
state.pending_buffers.insert(buffer_id, (ptr, size, fd));
- dbg(&format!("redraw w={w} h={h} stride={stride} buf_id={buffer_id} pos=({:.1},{:.1}) visible={}", state.core.pos.0, state.core.pos.1, state.core.visible));
+ dbg(&format!(
+ "redraw w={w} h={h} stride={stride} buf_id={buffer_id} pos=({:.1},{:.1}) visible={}",
+ state.core.pos.0, state.core.pos.1, state.core.visible
+ ));
surface.attach(Some(&buffer), 0, 0);
surface.damage_buffer(0, 0, w as i32, h as i32);
surface.commit();
@@ -455,23 +475,29 @@ impl Dispatch<wl_registry::WlRegistry, ()> for OverlayState {
_conn: &Connection,
qh: &QueueHandle<Self>,
) {
- if let wl_registry::Event::Global { name, interface, version } = event {
+ if let wl_registry::Event::Global {
+ name,
+ interface,
+ version,
+ } = event
+ {
match interface.as_str() {
"wl_compositor" => {
- state.compositor = Some(registry.bind::<WlCompositor, _, _>(
- name, version.min(6), qh, ()));
+ state.compositor =
+ Some(registry.bind::<WlCompositor, _, _>(name, version.min(6), qh, ()));
}
"wl_shm" => {
state.shm = Some(registry.bind::<WlShm, _, _>(name, version.min(1), qh, ()));
}
"wl_output" => {
if state.output.is_none() {
- state.output = Some(registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ()));
+ state.output =
+ Some(registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ()));
}
}
"zwlr_layer_shell_v1" => {
- state.layer_shell = Some(registry.bind::<ZwlrLayerShellV1, _, _>(
- name, version.min(4), qh, ()));
+ state.layer_shell =
+ Some(registry.bind::<ZwlrLayerShellV1, _, _>(name, version.min(4), qh, ()));
}
_ => {}
}
@@ -487,7 +513,8 @@ impl Dispatch<WlCompositor, ()> for OverlayState {
_: &(),
_: &Connection,
_: &QueueHandle<Self>,
- ) {}
+ ) {
+ }
}
impl Dispatch<WlShm, ()> for OverlayState {
@@ -498,7 +525,8 @@ impl Dispatch<WlShm, ()> for OverlayState {
_: &(),
_: &Connection,
_: &QueueHandle<Self>,
- ) {}
+ ) {
+ }
}
impl Dispatch<WlOutput, ()> for OverlayState {
@@ -528,7 +556,8 @@ impl Dispatch<ZwlrLayerShellV1, ()> for OverlayState {
_: &(),
_: &Connection,
_: &QueueHandle<Self>,
- ) {}
+ ) {
+ }
}
impl Dispatch<ZwlrLayerSurfaceV1, ()> for OverlayState {
@@ -540,7 +569,12 @@ impl Dispatch<ZwlrLayerSurfaceV1, ()> for OverlayState {
_: &Connection,
_: &QueueHandle<Self>,
) {
- if let zwlr_layer_surface_v1::Event::Configure { serial, width, height } = event {
+ if let zwlr_layer_surface_v1::Event::Configure {
+ serial,
+ width,
+ height,
+ } = event
+ {
layer.ack_configure(serial);
if width > 0 {
state.output_w = width;
@@ -561,7 +595,8 @@ impl Dispatch<WlSurface, ()> for OverlayState {
_: &(),
_: &Connection,
_: &QueueHandle<Self>,
- ) {}
+ ) {
+ }
}
impl Dispatch<WlShmPool, ()> for OverlayState {
@@ -572,7 +607,8 @@ impl Dispatch<WlShmPool, ()> for OverlayState {
_: &(),
_: &Connection,
_: &QueueHandle<Self>,
- ) {}
+ ) {
+ }
}
impl Dispatch<WlBuffer, ()> for OverlayState {
@@ -605,5 +641,6 @@ impl Dispatch<WlRegion, ()> for OverlayState {
_: &(),
_: &Connection,
_: &QueueHandle<Self>,
- ) {}
+ ) {
+ }
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs
index 7f3eaa2bcd..4b9df956a9 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs
@@ -32,13 +32,10 @@ use std::sync::OnceLock;
use std::thread;
use crossbeam_channel::{bounded, Receiver, Sender};
-use wayland_client::{
- protocol::wl_pointer::ButtonState,
- Connection, Proxy,
-};
+use wayland_client::{protocol::wl_pointer::ButtonState, Connection};
use wayland_protocols_wlr::virtual_pointer::v1::client::zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1;
-use super::{open_vptr_session, evdev_pointer_button};
+use super::{evdev_pointer_button, open_vptr_session};
/// One in-flight command from the public API to the owner thread.
enum Cmd {
@@ -99,15 +96,31 @@ fn owner_thread(rx: Receiver<Cmd>) {
let mut active: HashMap<String, ActivePointer> = HashMap::new();
while let Ok(cmd) = rx.recv() {
match cmd {
- Cmd::Press { cursor_id, window_id, x, y, button, reply } => {
+ Cmd::Press {
+ cursor_id,
+ window_id,
+ x,
+ y,
+ button,
+ reply,
+ } => {
let r = handle_press(&mut active, &cursor_id, window_id, x, y, button);
let _ = reply.send(r);
}
- Cmd::MoveTo { cursor_id, x, y, reply } => {
+ Cmd::MoveTo {
+ cursor_id,
+ x,
+ y,
+ reply,
+ } => {
let r = handle_move(&mut active, &cursor_id, x, y);
let _ = reply.send(r);
}
- Cmd::Release { cursor_id, button, reply } => {
+ Cmd::Release {
+ cursor_id,
+ button,
+ reply,
+ } => {
let r = handle_release(&mut active, &cursor_id, button);
let _ = reply.send(r);
}
@@ -158,7 +171,15 @@ fn handle_press(
let mut held = HashSet::new();
held.insert(btn);
- active.insert(cursor_id.to_string(), ActivePointer { vptr, held, out_w: w, out_h: h });
+ active.insert(
+ cursor_id.to_string(),
+ ActivePointer {
+ vptr,
+ held,
+ out_w: w,
+ out_h: h,
+ },
+ );
Ok(())
}
@@ -169,11 +190,15 @@ fn handle_move(
y: i32,
) -> anyhow::Result<()> {
let entry = active.get_mut(cursor_id).ok_or_else(|| {
- anyhow::anyhow!("no held mouse button for cursor '{cursor_id}'; call mouse_button_down first")
+ anyhow::anyhow!(
+ "no held mouse button for cursor '{cursor_id}'; call mouse_button_down first"
+ )
})?;
let px = x.clamp(0, entry.out_w as i32 - 1) as u32;
let py = y.clamp(0, entry.out_h as i32 - 1) as u32;
- entry.vptr.motion_absolute(0, px, py, entry.out_w, entry.out_h);
+ entry
+ .vptr
+ .motion_absolute(0, px, py, entry.out_w, entry.out_h);
entry.vptr.frame();
roundtrip_on_persistent(cursor_id)?;
Ok(())
@@ -186,9 +211,9 @@ fn handle_release(
) -> anyhow::Result<()> {
let btn = evdev_pointer_button(button);
let drop_entry = {
- let entry = active.get_mut(cursor_id).ok_or_else(|| {
- anyhow::anyhow!("no held mouse button for cursor '{cursor_id}'")
- })?;
+ let entry = active
+ .get_mut(cursor_id)
+ .ok_or_else(|| anyhow::anyhow!("no held mouse button for cursor '{cursor_id}'"))?;
entry.vptr.button(0, btn, ButtonState::Released);
entry.vptr.frame();
roundtrip_on_persistent(cursor_id)?;
@@ -219,20 +244,27 @@ thread_local! {
fn persist_conn(cursor_id: &str, conn: Connection) {
let queue = conn.new_event_queue::<super::State>();
- CONNS.with(|c| { c.borrow_mut().insert(cursor_id.to_string(), (conn, queue)); });
+ CONNS.with(|c| {
+ c.borrow_mut().insert(cursor_id.to_string(), (conn, queue));
+ });
}
fn forget_conn(cursor_id: &str) {
- CONNS.with(|c| { c.borrow_mut().remove(cursor_id); });
+ CONNS.with(|c| {
+ c.borrow_mut().remove(cursor_id);
+ });
}
fn roundtrip_on_persistent(cursor_id: &str) -> anyhow::Result<()> {
CONNS.with(|c| {
let mut b = c.borrow_mut();
- let (_conn, queue) = b.get_mut(cursor_id)
+ let (_conn, queue) = b
+ .get_mut(cursor_id)
.ok_or_else(|| anyhow::anyhow!("no persistent connection for cursor '{cursor_id}'"))?;
let mut tmp = super::State::default();
- queue.roundtrip(&mut tmp).map_err(|e| anyhow::anyhow!("compositor roundtrip failed: {e}"))?;
+ queue
+ .roundtrip(&mut tmp)
+ .map_err(|e| anyhow::anyhow!("compositor roundtrip failed: {e}"))?;
Ok(())
})
}
@@ -248,10 +280,15 @@ pub fn press(cursor_id: &str, window_id: u64, x: i32, y: i32, button: u8) -> any
let (tx_r, rx_r) = bounded(1);
tx().send(Cmd::Press {
cursor_id: cursor_id.to_string(),
- window_id, x, y, button,
+ window_id,
+ x,
+ y,
+ button,
reply: tx_r,
- }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
+ })
+ .map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
+ rx_r.recv()
+ .map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
}
/// Emit motion_absolute on the held cursor's virtual-pointer. Errors if there
@@ -259,9 +296,14 @@ pub fn press(cursor_id: &str, window_id: u64, x: i32, y: i32, button: u8) -> any
pub fn move_to(cursor_id: &str, x: i32, y: i32) -> anyhow::Result<()> {
let (tx_r, rx_r) = bounded(1);
tx().send(Cmd::MoveTo {
- cursor_id: cursor_id.to_string(), x, y, reply: tx_r,
- }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
+ cursor_id: cursor_id.to_string(),
+ x,
+ y,
+ reply: tx_r,
+ })
+ .map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
+ rx_r.recv()
+ .map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
}
/// Release `button` on the held cursor. If no other buttons remain held the
@@ -269,9 +311,13 @@ pub fn move_to(cursor_id: &str, x: i32, y: i32) -> anyhow::Result<()> {
pub fn release(cursor_id: &str, button: u8) -> anyhow::Result<()> {
let (tx_r, rx_r) = bounded(1);
tx().send(Cmd::Release {
- cursor_id: cursor_id.to_string(), button, reply: tx_r,
- }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
+ cursor_id: cursor_id.to_string(),
+ button,
+ reply: tx_r,
+ })
+ .map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
+ rx_r.recv()
+ .map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
}
/// Drop the entry for `cursor_id` without emitting any Wayland events.
@@ -281,7 +327,10 @@ pub fn release(cursor_id: &str, button: u8) -> anyhow::Result<()> {
pub fn forget(cursor_id: &str) -> anyhow::Result<()> {
let (tx_r, rx_r) = bounded(1);
tx().send(Cmd::Forget {
- cursor_id: cursor_id.to_string(), reply: tx_r,
- }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
+ cursor_id: cursor_id.to_string(),
+ reply: tx_r,
+ })
+ .map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
+ rx_r.recv()
+ .map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
}