Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): confirm X11 foreground actions that close a window when focus parks on the WM (#4248)

Antoine Bonnet55c7f27b19e8
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/foreground.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/foreground.rs
index 4968f60f3a..33eb16bbcc 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/foreground.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/foreground.rs
@@ -124,6 +124,13 @@ pub struct ForegroundReport {
/// a menu opened, a dialog mapped or closed. `None` when nothing changed
/// or the observation was unavailable.
pub window_change: Option<String>,
+ /// A window in `window_change` is positively attributed to the target
+ /// process: its pid was known and the window resolved to it. The window
+ /// set also lists override-redirect popups that name no pid (a VCL menu,
+ /// but equally another client's popup) and, when the target pid is
+ /// unknown, every on-screen window; a change among those alone is not
+ /// the target's own.
+ pub window_change_owned: bool,
}
impl ForegroundReport {
@@ -145,12 +152,40 @@ impl ForegroundReport {
pub fn focus_kept(&self) -> bool {
matches!(self.focus_after, FocusAfter::Target | FocusAfter::SamePid)
}
+
+ /// The window change confirms the action wherever the focus ended up
+ /// when the changed window is the target process's own (a dialog closed
+ /// by its own OK button takes the focus with it); a change among
+ /// unattributed windows only counts while the focus stayed in the
+ /// target process.
+ pub fn window_change_confirms(&self) -> bool {
+ self.window_change.is_some() && (self.window_change_owned || self.focus_kept())
+ }
+}
+
+/// One on-screen toplevel of a window set read for a before/after diff.
+#[derive(Clone, Debug)]
+struct WindowEntry {
+ window: u64,
+ description: String,
+ /// Positively attributed to the target pid (see
+ /// [`ForegroundReport::window_change_owned`]).
+ owned: bool,
+}
+
+/// What changed between two window sets.
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct WindowChange {
+ summary: String,
+ owned: bool,
}
/// The on-screen toplevels of `pid` (managed windows plus override-redirect
-/// popups), as `(window, description)` pairs, for a before/after diff.
-fn pid_window_set(pid: Option<u32>) -> Vec<(u64, String)> {
- let mut set: Vec<(u64, String)> = crate::x11::list_windows(pid)
+/// popups) for a before/after diff. Managed windows are listed by resolved
+/// owner, so they are `owned` whenever `pid` is known; a popup is `owned`
+/// only when it carries `pid` itself.
+fn pid_window_set(pid: Option<u32>) -> Vec<WindowEntry> {
+ let mut set: Vec<WindowEntry> = crate::x11::list_windows(pid)
.into_iter()
.filter(|w| w.is_on_screen)
.map(|w| {
@@ -159,44 +194,53 @@ fn pid_window_set(pid: Option<u32>) -> Vec<(u64, String)> {
} else {
format!(" \"{}\"", w.title)
};
- (w.xid, format!("window {}{title}", w.xid))
+ WindowEntry {
+ window: w.xid,
+ description: format!("window {}{title}", w.xid),
+ owned: pid.is_some(),
+ }
})
.collect();
set.extend(
super::mapped_popup_windows()
.into_iter()
.filter(|p| pid.is_none() || p.pid.is_none() || p.pid == pid)
- .map(|p| (p.window, p.describe())),
+ .map(|p| WindowEntry {
+ window: p.window,
+ description: p.describe(),
+ owned: pid.is_some() && p.pid == pid,
+ }),
);
set
}
/// Describe what changed between two window sets, or `None` when nothing did.
-pub(crate) fn describe_window_change(
- before: &[(u64, String)],
- after: &[(u64, String)],
-) -> Option<String> {
- let appeared: Vec<&str> = after
- .iter()
- .filter(|(id, _)| !before.iter().any(|(b, _)| b == id))
- .map(|(_, d)| d.as_str())
- .collect();
- let vanished: Vec<&str> = before
- .iter()
- .filter(|(id, _)| !after.iter().any(|(a, _)| a == id))
- .map(|(_, d)| d.as_str())
- .collect();
+fn describe_window_change(before: &[WindowEntry], after: &[WindowEntry]) -> Option<WindowChange> {
+ let absent_from =
+ |set: &[WindowEntry], entry: &WindowEntry| !set.iter().any(|e| e.window == entry.window);
+ let appeared: Vec<&WindowEntry> = after.iter().filter(|e| absent_from(before, e)).collect();
+ let vanished: Vec<&WindowEntry> = before.iter().filter(|e| absent_from(after, e)).collect();
if appeared.is_empty() && vanished.is_empty() {
return None;
}
+ let join = |entries: &[&WindowEntry]| {
+ entries
+ .iter()
+ .map(|e| e.description.as_str())
+ .collect::<Vec<_>>()
+ .join(", ")
+ };
let mut parts = Vec::new();
if !appeared.is_empty() {
- parts.push(format!("appeared: {}", appeared.join(", ")));
+ parts.push(format!("appeared: {}", join(&appeared)));
}
if !vanished.is_empty() {
- parts.push(format!("closed: {}", vanished.join(", ")));
+ parts.push(format!("closed: {}", join(&vanished)));
}
- Some(parts.join("; "))
+ Some(WindowChange {
+ summary: parts.join("; "),
+ owned: appeared.iter().chain(&vanished).any(|e| e.owned),
+ })
}
/// How often the target process's window set is re-read after the body.
@@ -212,10 +256,14 @@ const WINDOW_CHANGE_DEADLINE: Duration = Duration::from_millis(800);
/// (`_NET_WM_NAME` arrives a moment after the map) so the summary names it.
const WINDOW_CHANGE_TITLE_GRACE: Duration = Duration::from_millis(60);
-/// How long the post-check retries an empty core focus (`None` /
-/// `PointerRoot`): a popup destroyed by the click leaves the focus unset for
-/// a beat before the toolkit re-focuses its toplevel.
-const FOCUS_RETRY: Duration = Duration::from_millis(250);
+/// How long the post-check retries a core focus that says nothing about
+/// where input went: `None` / `PointerRoot`, or a window of the window
+/// manager's own X client. A popup destroyed by the click leaves the focus
+/// unset for a beat before the toolkit re-focuses its toplevel, and mutter
+/// parks the focus on its `no_focus_window` while it picks the next window
+/// after a dialog closes (measured on GNOME 42: from ~500 ms after the close
+/// for ~200 ms).
+const FOCUS_RETRY: Duration = Duration::from_millis(500);
/// Post-action observation bounds: `WINDOW_CHANGE_DEADLINE` /
/// `WINDOW_CHANGE_POLL` unless the embedding host set
@@ -235,9 +283,9 @@ fn window_change_bounds() -> WindowObservationBounds {
/// `confirmed` window-change evidence it never looked for.
fn wait_for_window_change(
pid: Option<u32>,
- before: &[(u64, String)],
+ before: &[WindowEntry],
bounds: WindowObservationBounds,
-) -> (Vec<(u64, String)>, Option<String>) {
+) -> (Vec<WindowEntry>, Option<WindowChange>) {
if bounds.skips_observation() {
return (pid_window_set(pid), None);
}
@@ -307,6 +355,17 @@ struct X11 {
root: Window,
net_active_window: u32,
net_wm_pid: u32,
+ resource_id_mask: u32,
+ /// The window manager's `_NET_SUPPORTING_WM_CHECK` window, when one is
+ /// advertised: any window sharing its X client base belongs to the WM.
+ wm_check_window: Option<Window>,
+}
+
+/// Whether two X resource ids were allocated by the same client: the server
+/// hands each client a base and lets it fill in the bits under
+/// `resource_id_mask`.
+fn same_x11_client(a: Window, b: Window, resource_id_mask: u32) -> bool {
+ a & !resource_id_mask == b & !resource_id_mask
}
impl X11 {
@@ -315,19 +374,40 @@ impl X11 {
anyhow!("{CODE_UNAVAILABLE}: cannot open DISPLAY to verify X11 input focus: {e}")
})?;
let root = conn.setup().roots[screen].root;
+ let resource_id_mask = conn.setup().resource_id_mask;
let net_active_window = conn
.intern_atom(false, b"_NET_ACTIVE_WINDOW")?
.reply()?
.atom;
let net_wm_pid = conn.intern_atom(false, b"_NET_WM_PID")?.reply()?.atom;
+ let wm_check = conn
+ .intern_atom(false, b"_NET_SUPPORTING_WM_CHECK")?
+ .reply()?
+ .atom;
+ let wm_check_window = conn
+ .get_property(false, root, wm_check, AtomEnum::WINDOW, 0, 1)
+ .ok()
+ .and_then(|cookie| cookie.reply().ok())
+ .and_then(|reply| reply.value32()?.next())
+ .filter(|w| *w != 0);
Ok(Self {
conn,
root,
net_active_window,
net_wm_pid,
+ resource_id_mask,
+ wm_check_window,
})
}
+ /// The focus sits on a window the window manager created for itself
+ /// (mutter's `no_focus_window` between two focused clients): a parking
+ /// spot, not a destination.
+ fn held_by_window_manager(&self, focused: Window) -> bool {
+ self.wm_check_window
+ .is_some_and(|check| same_x11_client(focused, check, self.resource_id_mask))
+ }
+
fn active_window(&self) -> Option<Window> {
let reply = self
.conn
@@ -594,35 +674,49 @@ fn confirm_phase(target: Window, settle: Duration) -> Result<ConfirmOutcome> {
}
}
+/// Poll `read_focus` until it names a window that is not a parking spot
+/// (`read_focus` already drops `None` / `PointerRoot`; `parked` marks the
+/// window manager's own windows) or `deadline` passes.
+fn await_focus_destination(
+ deadline: Instant,
+ mut read_focus: impl FnMut() -> Option<Window>,
+ parked: impl Fn(Window) -> bool,
+) -> Option<Window> {
+ loop {
+ if let Some(focused) = read_focus().filter(|f| !parked(*f)) {
+ return Some(focused);
+ }
+ if Instant::now() >= deadline {
+ return None;
+ }
+ std::thread::sleep(Duration::from_millis(25));
+ }
+}
+
/// Where the focus sits after the body. `pid_windows` is the target
/// process's window set read after the body (managed windows and popups):
/// focus inside any of them is `SamePid` even when neither the target nor
/// the focused window carries `_NET_WM_PID` (a VCL popup menu closed by the
-/// click, the popup that the click opened).
-fn post_check(
- target: Window,
- target_pid: Option<u32>,
- pid_windows: &[(u64, String)],
-) -> FocusAfter {
+/// click, the popup that the click opened). A focus that is unset or parked
+/// on the window manager's own window for the whole `FOCUS_RETRY` is
+/// `Unknown`: it names no client the input could have reached.
+fn post_check(target: Window, target_pid: Option<u32>, pid_windows: &[WindowEntry]) -> FocusAfter {
let Ok(x) = X11::open() else {
return FocusAfter::Unknown;
};
- let retry_until = Instant::now() + FOCUS_RETRY;
- let focused = loop {
- if let Some(focused) = x.focused() {
- break focused;
- }
- if Instant::now() >= retry_until {
- return FocusAfter::Elsewhere;
- }
- std::thread::sleep(Duration::from_millis(25));
+ let Some(focused) = await_focus_destination(
+ Instant::now() + FOCUS_RETRY,
+ || x.focused(),
+ |focused| x.held_by_window_manager(focused),
+ ) else {
+ return FocusAfter::Unknown;
};
if x.is_within(focused, target) {
return FocusAfter::Target;
}
if pid_windows
.iter()
- .any(|(window, _)| x.is_within(focused, *window as Window))
+ .any(|entry| x.is_within(focused, entry.window as Window))
{
return FocusAfter::SamePid;
}
@@ -700,7 +794,8 @@ pub fn with_x11_foreground_opts<T>(
retried_activation: outcome.retried,
confirm_ms: outcome.elapsed.as_millis() as u64,
focus_after,
- window_change,
+ window_change_owned: window_change.as_ref().is_some_and(|c| c.owned),
+ window_change: window_change.map(|c| c.summary),
},
))
}
@@ -808,13 +903,68 @@ mod tests {
WINDOW_CHANGE_DEADLINE,
WINDOW_CHANGE_POLL,
);
- let before = vec![(u64::MAX, "window that never existed".to_string())];
+ let before = vec![entry(u64::MAX, "window that never existed", true)];
let started = Instant::now();
let (_after, change) = wait_for_window_change(Some(u32::MAX), &before, bounds);
assert_eq!(change, None);
assert!(started.elapsed() < WINDOW_CHANGE_DEADLINE);
}
+ /// mutter's `no_focus_window` (0x1600058) and its `_NET_SUPPORTING_WM_CHECK`
+ /// window (0x1600006) share the client base above the 21-bit resource mask;
+ /// a LibreOffice toplevel (0x3e00096) does not.
+ #[test]
+ fn wm_windows_share_the_wm_check_client_base() {
+ let mask = 0x1f_ffff;
+ assert!(same_x11_client(0x160_0058, 0x160_0006, mask));
+ assert!(!same_x11_client(0x3e0_0096, 0x160_0006, mask));
+ }
+
+ /// The focus sequence mutter produces after a focused dialog closes:
+ /// unset, then parked on its own window, then the next toplevel. The
+ /// post-check waits the parking out and reports the toplevel.
+ #[test]
+ fn focus_parked_on_the_window_manager_is_waited_out() {
+ let mask = 0x1f_ffff;
+ let wm_check = 0x160_0006;
+ let mut readings = [None, Some(0x160_0058), Some(0x160_0058), Some(0x3e0_0096)].into_iter();
+ let focused = await_focus_destination(
+ Instant::now() + Duration::from_secs(2),
+ || readings.next().flatten(),
+ |w| same_x11_client(w, wm_check, mask),
+ );
+ assert_eq!(focused, Some(0x3e0_0096));
+ }
+
+ /// A focus that never leaves the window manager names no destination:
+ /// `None` at the deadline, never the parking window itself.
+ #[test]
+ fn focus_left_on_the_window_manager_is_unknown_at_the_deadline() {
+ let mask = 0x1f_ffff;
+ let deadline = Instant::now() + Duration::from_millis(80);
+ let focused = await_focus_destination(
+ deadline,
+ || Some(0x160_0058),
+ |w| same_x11_client(w, 0x160_0006, mask),
+ );
+ assert_eq!(focused, None);
+ assert!(Instant::now() >= deadline);
+ }
+
+ /// A focus already on a client window returns on the first read: the
+ /// happy path pays no retry.
+ #[test]
+ fn settled_focus_returns_without_waiting() {
+ let started = Instant::now();
+ let focused = await_focus_destination(
+ started + Duration::from_secs(2),
+ || Some(0x3e0_0096),
+ |_| false,
+ );
+ assert_eq!(focused, Some(0x3e0_0096));
+ assert!(started.elapsed() < Duration::from_millis(25));
+ }
+
#[test]
fn deadline_returns_value_when_work_finishes() {
assert_eq!(run_with_deadline(Duration::from_secs(2), || 7), Some(7));
@@ -848,6 +998,7 @@ mod tests {
confirm_ms: 3,
focus_after: FocusAfter::SamePid,
window_change: Some("appeared: popup window 7 (200x300 at 1,2)".into()),
+ window_change_owned: false,
};
let json = report.to_json();
assert_eq!(json["activated"], false);
@@ -860,23 +1011,47 @@ mod tests {
assert!(report.focus_kept());
}
+ fn entry(window: u64, description: &str, owned: bool) -> WindowEntry {
+ WindowEntry {
+ window,
+ description: description.to_string(),
+ owned,
+ }
+ }
+
#[test]
fn window_change_diff_names_appeared_and_closed() {
let before = vec![
- (1u64, "window 1 \"GIMP\"".to_string()),
- (2, "window 2".to_string()),
+ entry(1, "window 1 \"GIMP\"", true),
+ entry(2, "window 2", true),
];
let after = vec![
- (1u64, "window 1 \"GIMP\"".to_string()),
- (9, "popup window 9".to_string()),
+ entry(1, "window 1 \"GIMP\"", true),
+ entry(9, "popup window 9", false),
];
assert_eq!(
- describe_window_change(&before, &after).as_deref(),
- Some("appeared: popup window 9; closed: window 2")
+ describe_window_change(&before, &after),
+ Some(WindowChange {
+ summary: "appeared: popup window 9; closed: window 2".into(),
+ owned: true,
+ })
);
assert_eq!(describe_window_change(&before, &before), None);
}
+ /// A popup that names no pid (a VCL menu, or another client's) is listed
+ /// in the target's set, but its change alone is not the target's own.
+ #[test]
+ fn unattributed_popup_change_is_not_owned() {
+ let before = vec![entry(1, "window 1 \"Calc\"", true)];
+ let after = vec![
+ entry(1, "window 1 \"Calc\"", true),
+ entry(9, "popup window 9", false),
+ ];
+ let change = describe_window_change(&before, &after).expect("a popup appeared");
+ assert!(!change.owned);
+ }
+
#[test]
fn unavailable_display_is_structured() {
let display = crate::test_env::unreachable_x11_display();
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 29e1939b77..fc199ce043 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -2506,24 +2506,24 @@ fn foreground_structured(
/// Fold a foreground transaction's post-check into a result: the report
/// itself, `focus_after` at the top level, and an `evidence` item so the
-/// observation survives the public-record reduction. Focus that stayed in
-/// the target process AND a window that appeared / closed in it is
-/// `window_change` evidence and an `effect: confirmed`; focus that left the
-/// process is a suspected no-op; anything else stays unverifiable with a
-/// `native_api_result` item.
+/// observation survives the public-record reduction. A window of the target
+/// process that appeared or closed is `window_change` evidence and an
+/// `effect: confirmed` wherever the focus sits afterwards (the OK button of a
+/// dialog destroys the very window that held the focus, and closing the last
+/// window of a process leaves the focus to the window manager); a change
+/// among unattributed popups only counts while the focus stayed in the
+/// process (`ForegroundReport::window_change_confirms`). Otherwise, focus that
+/// left the process is a suspected no-op and anything else stays
+/// unverifiable with a `native_api_result` item.
fn apply_foreground_report(v: &mut Value, report: &crate::input::ForegroundReport) {
v["foreground"] = report.to_json();
v["focus_after"] = json!(report.focus_after.as_str());
- if report.focus_after == crate::input::FocusAfter::Elsewhere {
- v["effect"] = json!("suspected_noop");
- v["warning"] = json!(
- "input focus left the target process before the post-check; the input \
- may have reached another window. Verify with a screenshot."
- );
- return;
- }
- let evidence = match (&report.window_change, report.focus_kept()) {
- (Some(change), true) => {
+ let confirming_change = report
+ .window_change
+ .as_ref()
+ .filter(|_| report.window_change_confirms());
+ let evidence = match confirming_change {
+ Some(change) => {
v["effect"] = json!("confirmed");
v["verified"] = json!(true);
json!({
@@ -2531,12 +2531,23 @@ fn apply_foreground_report(v: &mut Value, report: &crate::input::ForegroundRepor
"detail": format!("focus_after={}; {change}", report.focus_after.as_str()),
})
}
- _ => json!({
+ None if report.focus_after == crate::input::FocusAfter::Elsewhere => {
+ v["effect"] = json!("suspected_noop");
+ v["warning"] = json!(
+ "input focus left the target process before the post-check; the input \
+ may have reached another window. Verify with a screenshot."
+ );
+ return;
+ }
+ None => json!({
"kind": "native_api_result",
"detail": format!(
- "real input delivered to the activated window (focus_after={}); no window \
- change observed",
- report.focus_after.as_str()
+ "real input delivered to the activated window (focus_after={}); {}",
+ report.focus_after.as_str(),
+ match &report.window_change {
+ Some(change) => format!("{change}, not attributed to the target process"),
+ None => "no window change observed".to_owned(),
+ }
),
}),
};
@@ -5793,7 +5804,7 @@ impl ClickTool {
.unwrap_or((sx.round() as i32, sy.round() as i32));
match result {
Ok(Ok(((), report))) => {
- let confirmed = report.focus_kept() && report.window_change.is_some();
+ let confirmed = report.window_change_confirms();
let result = ToolResult::text(format!(
"Clicked element [{idx}] (pid {pid}) with a real pointer click \
(delivery_mode=foreground, focus_after={}){}{}",
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/pointer_argument_tests.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/pointer_argument_tests.rs
index d4f42dddc1..e9d95c76a5 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/pointer_argument_tests.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/pointer_argument_tests.rs
@@ -30,6 +30,7 @@ fn foreground_report_becomes_evidence() {
confirm_ms: 12,
focus_after: crate::input::FocusAfter::SamePid,
window_change: Some("appeared: window 9 \"Brightness-Contrast\"".into()),
+ window_change_owned: true,
};
let v = foreground_structured("x11_xtest_fg", report, serde_json::Map::new());
assert_eq!(v["effect"], "confirmed");
@@ -41,16 +42,59 @@ fn foreground_report_becomes_evidence() {
confirm_ms: 0,
focus_after: crate::input::FocusAfter::Target,
window_change: None,
+ window_change_owned: false,
};
let v = foreground_structured("x11_xtest_fg", quiet, serde_json::Map::new());
assert_eq!(v["effect"], "unverifiable");
assert_eq!(v["evidence"][0]["kind"], "native_api_result");
+ let closed = crate::input::ForegroundReport {
+ already_active: true,
+ retried_activation: false,
+ confirm_ms: 0,
+ focus_after: crate::input::FocusAfter::Unknown,
+ window_change: Some("closed: window 3 \"Format Cells\"".into()),
+ window_change_owned: true,
+ };
+ let v = foreground_structured("x11_xtest_fg", closed, serde_json::Map::new());
+ assert_eq!(v["effect"], "confirmed");
+ assert_eq!(v["evidence"][0]["kind"], "window_change");
+ // A popup that names no pid is listed in the target's window set (a VCL
+ // menu carries none), but it may be another client's: once the focus
+ // left the target process its change is no confirmation.
+ let unattributed = |focus_after| crate::input::ForegroundReport {
+ already_active: true,
+ retried_activation: false,
+ confirm_ms: 0,
+ focus_after,
+ window_change: Some("appeared: popup window 7 (200x300 at 1,2)".into()),
+ window_change_owned: false,
+ };
+ let v = foreground_structured(
+ "x11_xtest_fg",
+ unattributed(crate::input::FocusAfter::Elsewhere),
+ serde_json::Map::new(),
+ );
+ assert_eq!(v["effect"], "suspected_noop");
+ let v = foreground_structured(
+ "x11_xtest_fg",
+ unattributed(crate::input::FocusAfter::Unknown),
+ serde_json::Map::new(),
+ );
+ assert_eq!(v["effect"], "unverifiable");
+ assert_eq!(v["evidence"][0]["kind"], "native_api_result");
+ let v = foreground_structured(
+ "x11_xtest_fg",
+ unattributed(crate::input::FocusAfter::SamePid),
+ serde_json::Map::new(),
+ );
+ assert_eq!(v["effect"], "confirmed");
let lost = crate::input::ForegroundReport {
already_active: true,
retried_activation: false,
confirm_ms: 0,
focus_after: crate::input::FocusAfter::Elsewhere,
- window_change: Some("closed: window 3".into()),
+ window_change: None,
+ window_change_owned: false,
};
let v = foreground_structured("x11_xtest_fg", lost, serde_json::Map::new());
assert_eq!(v["effect"], "suspected_noop");