Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

11.4 KB
Raw Blame
// SPDX-License-Identifier: MIT
// Copyright (c) 2026 Cua AI, Inc.

//! Detect target processes whose UI toolkit derives pointer-event locations
//! from the hardware pointer rather than from the delivered event.
//!
//! Tk's macOS backend reads the global pointer position when it translates an
//! incoming mouse event into Tk coordinates. A PID-routed background CGEvent
//! never moves the hardware pointer, so such an app receives the click wherever
//! the real pointer happens to be (for example the visual-only canvas fixture's
//! `ignored click outside cards at canvas (-86, -114)`), while the driver can
//! only report the post as unverifiable. Background delivery therefore cannot
//! satisfy these targets; the explicit foreground rung moves the pointer first.
//!
//! Detection is intentionally cheap and conservative: walk the target's
//! file-backed memory regions with `proc_pidinfo` (same-user, no task port) and
//! look for a mapped Tk library or Python's `_tkinter` extension. A toolkit
//! loaded only from the dyld shared cache is not visible this way; such targets
//! fall back to the honest "not driver-verified" background result.

/// A UI toolkit that background (PID-routed) mouse events cannot reach at the
/// requested point.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PointerReadingToolkit {
    /// Tk derives click locations from the hardware pointer.
    Tk,
    /// LibreOffice's macOS VCL plug-in (`libvclplug_osxlo.dylib`) drops
    /// PID-routed mouse events: a background click or right-click on a Calc
    /// cell leaves the selection where it was (VM check, 8 Oct 2026), and
    /// the AX press its cells accept does not select them either.
    Vcl,
}

impl PointerReadingToolkit {
    pub fn name(self) -> &'static str {
        match self {
            Self::Tk => "tk",
            Self::Vcl => "libreoffice-vcl",
        }
    }

    /// Why background delivery cannot reach this toolkit, as a clause that
    /// follows "its <name> toolkit".
    pub fn why(self) -> &'static str {
        match self {
            Self::Tk => {
                "derives click locations from the hardware pointer, and background delivery \
                 never moves the pointer, so the click would land wherever the pointer \
                 currently is"
            }
            Self::Vcl => {
                "ignores background (PID-routed) mouse events, so the click would not reach \
                 the target and the selection would stay where it is"
            }
        }
    }
}

/// Classify one mapped image path. Pure so the matcher is unit-testable.
///
/// Matches `Tk.framework` bundles, Tk shared libraries (`libtk8.6.dylib`,
/// Tk 9's `libtcl9tk9.0.dylib`), and CPython's `_tkinter` extension module,
/// which either links Tk dynamically or embeds it statically.
pub fn classify_image_path(path: &str) -> Option<PointerReadingToolkit> {
    if path.contains("/Tk.framework/") {
        return Some(PointerReadingToolkit::Tk);
    }
    let file = path.rsplit('/').next().unwrap_or(path);
    let lower = file.to_ascii_lowercase();
    // LibreOffice's macOS VCL plug-in: `libvclplug_osxlo.dylib`.
    if lower.starts_with("libvclplug_osx") && lower.ends_with(".dylib") {
        return Some(PointerReadingToolkit::Vcl);
    }
    if lower.starts_with("_tkinter") && lower.ends_with(".so") {
        return Some(PointerReadingToolkit::Tk);
    }
    if !lower.ends_with(".dylib") {
        return None;
    }
    let rest = lower.strip_prefix("lib")?;
    // Tk 9 prefixes the Tcl major version: `libtcl9tk9.0.dylib`.
    let rest = match rest.strip_prefix("tcl") {
        Some(after_tcl) => {
            let digits = after_tcl.bytes().take_while(u8::is_ascii_digit).count();
            if digits == 0 {
                return None;
            }
            &after_tcl[digits..]
        }
        None => rest,
    };
    let after_tk = rest.strip_prefix("tk")?;
    after_tk
        .bytes()
        .next()
        .filter(u8::is_ascii_digit)
        .map(|_| PointerReadingToolkit::Tk)
}

/// Return the pointer-reading toolkit mapped into `pid`, if any.
#[cfg(target_os = "macos")]
pub fn detect(pid: i32) -> Option<PointerReadingToolkit> {
    // Prefer the vnode-only flavor, which skips anonymous regions; fall back
    // to the full walk on kernels that reject it.
    match walk_regions(pid, PROC_PIDREGIONPATHINFO2) {
        Some(found) => found,
        None => walk_regions(pid, PROC_PIDREGIONPATHINFO).flatten(),
    }
}

#[cfg(not(target_os = "macos"))]
pub fn detect(_pid: i32) -> Option<PointerReadingToolkit> {
    None
}

#[cfg(target_os = "macos")]
const PROC_PIDREGIONPATHINFO: libc::c_int = 8;
#[cfg(target_os = "macos")]
const PROC_PIDREGIONPATHINFO2: libc::c_int = 22;
/// Bound the walk so a pathological address space cannot stall a click.
#[cfg(target_os = "macos")]
const MAX_REGIONS: usize = 65_536;

/// `struct proc_regioninfo` from `<sys/proc_info.h>`.
#[cfg(target_os = "macos")]
#[repr(C)]
#[derive(Clone, Copy)]
struct ProcRegionInfo {
    pri_protection: u32,
    pri_max_protection: u32,
    pri_inheritance: u32,
    pri_flags: u32,
    pri_offset: u64,
    pri_behavior: u32,
    pri_user_wired_count: u32,
    pri_user_tag: u32,
    pri_pages_resident: u32,
    pri_pages_shared_now_private: u32,
    pri_pages_swapped_out: u32,
    pri_pages_dirtied: u32,
    pri_ref_count: u32,
    pri_shadow_depth: u32,
    pri_share_mode: u32,
    pri_private_pages_resident: u32,
    pri_shared_pages_resident: u32,
    pri_obj_id: u32,
    pri_depth: u32,
    pri_address: u64,
    pri_size: u64,
}

/// `struct proc_regionwithpathinfo`: region info followed by
/// `struct vnode_info_path` (152-byte `vnode_info` + `MAXPATHLEN` path).
#[cfg(target_os = "macos")]
#[repr(C)]
struct ProcRegionWithPathInfo {
    prp_prinfo: ProcRegionInfo,
    prp_vip_vi: [u8; 152],
    prp_vip_path: [libc::c_char; 1024],
}

/// Walk `pid`'s regions with `flavor`. `None` means the flavor was rejected
/// before any region was read; `Some(result)` is a completed walk.
#[cfg(target_os = "macos")]
fn walk_regions(pid: i32, flavor: libc::c_int) -> Option<Option<PointerReadingToolkit>> {
    let size = std::mem::size_of::<ProcRegionWithPathInfo>();
    let mut address: u64 = 0;
    let mut read_any = false;
    for _ in 0..MAX_REGIONS {
        // SAFETY: an all-zero bit pattern is valid for this plain C struct.
        let mut info: ProcRegionWithPathInfo = unsafe { std::mem::zeroed() };
        // SAFETY: the buffer is exactly the kernel structure size and lives for
        // the duration of the call.
        let written = unsafe {
            libc::proc_pidinfo(
                pid,
                flavor,
                address,
                (&mut info as *mut ProcRegionWithPathInfo).cast(),
                size as libc::c_int,
            )
        };
        if written <= 0 || (written as usize) < size {
            break;
        }
        read_any = true;
        // SAFETY: the kernel NUL-terminates `vip_path` within MAXPATHLEN.
        let path = unsafe { std::ffi::CStr::from_ptr(info.prp_vip_path.as_ptr()) };
        if let Some(toolkit) = classify_image_path(&path.to_string_lossy()) {
            return Some(Some(toolkit));
        }
        let region = info.prp_prinfo;
        let next = region.pri_address.saturating_add(region.pri_size);
        if next <= address {
            break;
        }
        address = next;
    }
    if read_any {
        Some(None)
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn classifies_tk_images_and_rejects_neighbours() {
        for path in [
            "/Library/Frameworks/Tk.framework/Versions/8.6/Tk",
            "/opt/homebrew/Cellar/tcl-tk/9.0.2/lib/libtcl9tk9.0.dylib",
            "/opt/homebrew/opt/tcl-tk@8/lib/libtk8.6.dylib",
            "/usr/local/lib/python3.12/lib-dynload/_tkinter.cpython-312-darwin.so",
            "/Users/me/.local/share/uv/python/cpython-3.13/lib/python3.13/lib-dynload/_tkinter.cpython-313-darwin.so",
        ] {
            assert_eq!(
                classify_image_path(path),
                Some(PointerReadingToolkit::Tk),
                "{path}"
            );
        }
        for path in [
            "",
            "/usr/lib/libSystem.B.dylib",
            "/opt/homebrew/lib/libtcl9.0.dylib",
            "/opt/homebrew/lib/libtcl8.6.dylib",
            "/opt/homebrew/lib/libtkrzw.dylib",
            "/System/Library/Frameworks/AppKit.framework/Versions/C/AppKit",
            "/usr/lib/python3/_tkinter_helper.py",
            "/Applications/Foo.app/Contents/MacOS/tk8.6",
        ] {
            assert_eq!(classify_image_path(path), None, "{path}");
        }
    }

    #[test]
    fn classifies_the_libreoffice_vcl_plugin() {
        assert_eq!(
            classify_image_path(
                "/Applications/LibreOffice.app/Contents/Frameworks/libvclplug_osxlo.dylib"
            ),
            Some(PointerReadingToolkit::Vcl)
        );
        for path in [
            "/Applications/LibreOffice.app/Contents/Frameworks/libvcllo.dylib",
            "/Applications/LibreOffice.app/Contents/Frameworks/libvclplug_osxlo.dylib.bak",
        ] {
            assert_eq!(classify_image_path(path), None, "{path}");
        }
    }

    #[cfg(target_os = "macos")]
    #[test]
    fn kernel_region_structure_matches_proc_info_layout() {
        assert_eq!(std::mem::size_of::<ProcRegionInfo>(), 96);
        assert_eq!(std::mem::size_of::<ProcRegionWithPathInfo>(), 1272);
    }

    #[cfg(target_os = "macos")]
    #[test]
    fn detect_is_negative_for_a_non_tk_process() {
        // The test binary maps no Tk image. A completed negative walk proves the
        // region flavor is readable for a same-user process.
        assert_eq!(detect(std::process::id() as i32), None);
        assert!(walk_regions(std::process::id() as i32, PROC_PIDREGIONPATHINFO).is_some());
    }

    /// Loads `_tkinter` without creating any window. Skips when the host
    /// Python lacks Tk support. Ignored under plain `cargo test`: on a Mac
    /// without the Command Line Tools, `/usr/bin/python3` is a stub that opens
    /// the developer-tools install dialog. The macOS canonical runner's native
    /// lane, which provisions Python for the Tk rows, selects it.
    #[cfg(target_os = "macos")]
    #[test]
    #[ignore = "host desktop: spawns the host python3, which can open the Command Line Tools installer; run by scripts/ci/macos/run-rust-e2e.sh"]
    fn detect_finds_tk_in_a_python_process_that_imported_tkinter() {
        use std::io::BufRead;
        let spawned = std::process::Command::new("python3")
            .args([
                "-c",
                "import _tkinter, sys, time; print('ready', flush=True); time.sleep(30)",
            ])
            .stdout(std::process::Stdio::piped())
            .stderr(std::process::Stdio::null())
            .spawn();
        let Ok(mut child) = spawned else {
            eprintln!("skipping: python3 is unavailable");
            return;
        };
        let mut line = String::new();
        let ready = child
            .stdout
            .take()
            .map(|stdout| std::io::BufReader::new(stdout).read_line(&mut line))
            .is_some_and(|read| read.is_ok())
            && line.trim() == "ready";
        let detected = ready.then(|| detect(child.id() as i32));
        let _ = child.kill();
        let _ = child.wait();
        match detected {
            None => eprintln!("skipping: python3 has no importable _tkinter"),
            Some(found) => assert_eq!(found, Some(PointerReadingToolkit::Tk)),
        }
    }
}