Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
//! Multi-cursor background computer-use demo orchestrator.
//!
//! Launches the "National Records System" terminal in five UI frameworks: a 2x2
//! grid of background windows (each = half the work area) around one foreground
//! "master" in the middle that overlaps all four. When the human submits a
//! record in the master, the master emits the action on stdout; this
//! orchestrator replays it onto all four background corners concurrently, each
//! via its OWN cua-driver session (= its own uniquely-coloured agent cursor),
//! in the background — no window raised, the user's cursor never moved.
//!
//! Driving is element-based where an accessibility tree exists (set_value for
//! the SUBJECT-NAME field + UIA-Invoke for SUBMIT — no foreground steal, no
//! SendInput) and pixel-based for the GDI corner that has no a11y tree. Each
//! a11y corner is VERIFIED by reading the records grid back: if the typed
//! record didn't land, it's logged as FAIL.
use std::io::{BufRead, BufReader};
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::sync::mpsc::{channel, Sender};
use std::thread;
use std::time::{Duration, Instant};
use windows::core::PWSTR;
use windows::Win32::Foundation::{BOOL, HANDLE, HWND, LPARAM, POINT, RECT, TRUE};
use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};
use windows::Win32::Graphics::Gdi::ClientToScreen;
use windows::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
};
use windows::Win32::UI::WindowsAndMessaging::{
EnumWindows, GetClientRect, GetWindowTextW, GetWindowThreadProcessId, IsWindowVisible,
SetForegroundWindow, SetWindowPos, HWND_TOP, SWP_NOACTIVATE, SWP_NOZORDER, SWP_SHOWWINDOW,
};
const FIELD_FRAC: (f64, f64) = (0.28, 0.145); // Account-Name field (GDI pixel fallback)
// ── maze (line tool) ───────────────────────────────────────────────────────────
// A spiral maze as straight segments in [0,1] of the drawing REGION. cua-driver
// draws each as one press-drag-release (no curves = "line tool only"). The same
// segments rasterized are the reference the screenshot is diffed against.
const MAZE: [(f64, f64, f64, f64); 7] = [
(0.10, 0.12, 0.90, 0.12),
(0.90, 0.12, 0.90, 0.88),
(0.90, 0.88, 0.26, 0.88),
(0.26, 0.88, 0.26, 0.40),
(0.26, 0.40, 0.66, 0.40),
(0.66, 0.40, 0.66, 0.64),
(0.66, 0.64, 0.46, 0.64),
];
// Client-fraction rect of the window that sits INSIDE every framework's drawing
// pad (all four put the doodle top-right). cua-driver draws here and the
// verifier crops the same rect — so the rasterized reference aligns regardless
// of where each framework actually lays its canvas out. Tuned via screenshots.
// Kept inside the SHORTEST drawing pad: WPF/Electron host theirs in a fixed
// ~150px row, so the usable band ends well above the spreadsheet below it.
// Overshooting here drops the lower maze onto the grid (and selects rows).
const REGION: [f64; 4] = [0.470, 0.180, 0.900, 0.295]; // l, t, r, b (client fractions)
// A drawn maze must score at least this (F1 of ink overlap vs the reference,
// dilated) to count as COMMITTED; below it = FAIL.
const MAZE_PASS: f64 = 0.45;
// ── kill-on-exit job ──────────────────────────────────────────────────────────
static JOB: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
fn job() -> HANDLE {
let raw = *JOB.get_or_init(|| unsafe {
let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW");
let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
let _ = SetInformationJobObject(
h,
JobObjectExtendedLimitInformation,
&info as *const _ as *const core::ffi::c_void,
std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
);
h.0 as usize
});
HANDLE(raw as *mut core::ffi::c_void)
}
fn assign_to_job(child: &Child) {
use std::os::windows::io::AsRawHandle;
unsafe {
let _ = AssignProcessToJobObject(
job(),
HANDLE(child.as_raw_handle() as *mut core::ffi::c_void),
);
}
}
struct Corner {
title: &'static str, // unique substring to find the window
session: &'static str, // palette name -> cursor color
hwnd: HWND,
pid: u32,
field_idx: Option<u64>, // UIA element for the SUBJECT-NAME field (a11y corners)
submit_idx: Option<u64>, // UIA element for the SUBMIT button
}
fn main() {
let demo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.to_path_buf();
let repo_root = demo_root.parent().unwrap().parent().unwrap().to_path_buf();
let cua = std::env::var("CUA_DRIVER_EXE")
.map(PathBuf::from)
.unwrap_or_else(|_| repo_root.join("libs/cua-driver/rust/target/debug/cua-driver.exe"));
let legacy_app = std::env::var("LEGACY_APP_EXE")
.map(PathBuf::from)
.unwrap_or_else(|_| demo_root.join("target/debug/legacy-app.exe"));
if !cua.exists() {
eprintln!("cua-driver.exe not found at {cua:?}");
std::process::exit(1);
}
if !legacy_app.exists() {
eprintln!("legacy-app.exe not found at {legacy_app:?}");
std::process::exit(1);
}
eprintln!("[orch] starting cua-driver daemon…");
let mut daemon = Command::new(&cua)
.arg("serve")
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("spawn cua-driver serve");
assign_to_job(&daemon);
thread::sleep(Duration::from_millis(1500));
// Launch corners + master.
let mut kids: Vec<Child> = Vec::new();
let mut launch = |kids: &mut Vec<Child>, cmd: &mut Command| {
if let Ok(c) = cmd.stdout(Stdio::null()).stderr(Stdio::null()).spawn() {
assign_to_job(&c);
kids.push(c);
}
};
launch(
&mut kids,
Command::new(&legacy_app).args(["gdi", "Win32 GDI (no a11y)"]),
);
let winforms = demo_root.join("dotnet/winforms/bin/Debug/net10.0-windows/winforms-legacy.exe");
if winforms.exists() {
launch(&mut kids, &mut Command::new(&winforms));
} else {
eprintln!("[orch] (skip) winforms missing");
}
let wpf = demo_root.join("dotnet/wpf/bin/Debug/net10.0-windows/wpf-legacy.exe");
if wpf.exists() {
launch(&mut kids, &mut Command::new(&wpf));
} else {
eprintln!("[orch] (skip) wpf missing");
}
let electron_bin = demo_root.join("electron/node_modules/.bin/electron.cmd");
if electron_bin.exists() {
let mut c = Command::new(&electron_bin);
c.arg(".").current_dir(demo_root.join("electron"));
launch(&mut kids, &mut c);
} else {
eprintln!("[orch] (skip) electron not installed");
}
let mut master = Command::new(&legacy_app)
.args(["master", "Master (Win32 controls)"])
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.expect("spawn master");
assign_to_job(&master);
let master_out = master.stdout.take().unwrap();
thread::sleep(Duration::from_secs(3)); // window + electron warmup
let mut corners = vec![
Corner {
title: "Win32 GDI",
session: "crimson",
hwnd: HWND::default(),
pid: 0,
field_idx: None,
submit_idx: None,
},
Corner {
title: "WinForms",
session: "amber",
hwnd: HWND::default(),
pid: 0,
field_idx: None,
submit_idx: None,
},
Corner {
title: "WPF",
session: "aqua",
hwnd: HWND::default(),
pid: 0,
field_idx: None,
submit_idx: None,
},
Corner {
title: "Electron",
session: "mint_lime",
hwnd: HWND::default(),
pid: 0,
field_idx: None,
submit_idx: None,
},
];
for c in corners.iter_mut() {
if let Some((h, pid)) = find_window_by_title(c.title) {
c.hwnd = h;
c.pid = pid;
} else {
eprintln!("[orch] (skip) no window for '{}'", c.title);
}
}
corners.retain(|c| !c.hwnd.0.is_null());
let master_hwnd = find_window_by_title("Master (Win32").map(|(h, _)| h);
// Layout: each window half the work area; corners tile quadrants, master centered on top.
let wa = work_area();
let ww = (wa.right - wa.left) / 2;
let wh = (wa.bottom - wa.top) / 2;
let positions = [
(wa.left, wa.top),
(wa.left + ww, wa.top),
(wa.left, wa.top + wh),
(wa.left + ww, wa.top + wh),
];
for (i, c) in corners.iter().enumerate() {
let (x, y) = positions[i % 4];
place(c.hwnd, x, y, ww, wh, false);
}
if let Some(mh) = master_hwnd {
// Master is smaller than a quadrant so it overlaps the central seam of
// all four corners without hiding their forms (which sit at the
// quadrant centers, outside this box). Centered on the work area.
let (mw, mh2) = (ww / 2, wh / 2);
let (mx, my) = (wa.left + ww - mw / 2, wa.top + wh - mh2 / 2);
place(mh, mx, my, mw, mh2, true);
unsafe {
let _ = SetForegroundWindow(mh);
}
}
thread::sleep(Duration::from_millis(800)); // let resized layouts settle
// Discover UIA elements per corner (field + SUBMIT). GDI has none -> pixel.
// Reads pass tree_format "both": rows are found in the markdown and their
// element_token comes from `elements` (cua-driver 0.35 returns only the
// markdown by default).
for c in corners.iter_mut() {
let tree = run_call_out(
&cua,
"get_window_state",
&format!(
r#"{{"pid":{},"window_id":{},"capture_mode":"ax","tree_format":"both","session":"{}"}}"#,
c.pid, c.hwnd.0 as isize, c.session
),
);
c.field_idx = find_idx(&tree, |line| line.contains("] Edit"));
c.submit_idx = find_idx(&tree, |line| {
line.contains("] Button") && line.to_uppercase().contains("ADD RECORD")
});
eprintln!(
"[orch] {:<9} session={:<9} field={:?} submit={:?} {}",
c.title,
c.session,
c.field_idx,
c.submit_idx,
if c.field_idx.is_some() {
"(a11y: set_value+invoke)"
} else {
"(no a11y: pixel)"
}
);
// Pre-arm a coloured cursor for this session.
let _ = run_call(
&cua,
"set_agent_cursor_enabled",
&format!(r#"{{"enabled":true,"session":"{}"}}"#, c.session),
);
}
// One driver thread per corner.
let mut senders: Vec<Sender<Action>> = Vec::new();
let mut handles = Vec::new();
for c in &corners {
let (tx, rx) = channel::<Action>();
senders.push(tx);
let cua = cua.clone();
let pid = c.pid;
let hwnd_addr = c.hwnd.0 as isize;
let session = c.session.to_string();
let title = c.title.to_string();
handles.push(thread::spawn(move || {
let hwnd = HWND(hwnd_addr as *mut _);
let ax = format!(r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"ax","tree_format":"both","session":"{session}"}}"#);
let mut seen_records: i64 = -1;
for act in rx {
// Re-read the window before each action: element tokens are
// bound to the snapshot that read minted, and a11y vs pixel is
// decided per action (Chromium's tree appears late and
// re-numbers as rows are added; GDI never has a tree).
let tree = run_call_out(&cua, "get_window_state", &ax);
let field = find_idx(&tree, |l| l.contains("] Edit"))
.and_then(|idx| element_token(&tree, idx));
let submit = find_idx(&tree, |l| l.contains("] Button") && l.to_uppercase().contains("ADD RECORD"))
.and_then(|idx| element_token(&tree, idx));
match act {
Action::Type { text } => {
let ok = match field {
// a11y: plain type_text with element_token — cua-driver
// auto-routes to UIA ValuePattern.SetValue on its own.
Some(token) => run_call(&cua, "type_text", &format!(
r#"{{"pid":{pid},"window_id":{hwnd_addr},"element_token":"{}","text":"{}","session":"{session}"}}"#, json_escape(&token), json_escape(&text))),
// no a11y (GDI): focus the field by pixel, then WM_CHAR.
None => {
if let Some((x, y)) = client_rel_to_local_px(hwnd, FIELD_FRAC.0, FIELD_FRAC.1) {
let _ = run_call(&cua, "click", &format!(
r#"{{"pid":{pid},"window_id":{hwnd_addr},"x":{x},"y":{y},"session":"{session}"}}"#));
}
run_call(&cua, "type_text", &format!(
r#"{{"pid":{pid},"window_id":{hwnd_addr},"text":"{}","session":"{session}"}}"#, json_escape(&text)))
}
};
eprintln!("[drive {session}] type {text:?} -> {}", if ok { "ok" } else { "FAIL" });
}
Action::Click { rx, ry } => {
let ok = match submit {
Some(token) => run_call(&cua, "click", &format!(
r#"{{"pid":{pid},"window_id":{hwnd_addr},"element_token":"{}","session":"{session}"}}"#, json_escape(&token))),
None => {
if let Some((x, y)) = client_rel_to_local_px(hwnd, rx, ry) {
run_call(&cua, "click", &format!(
r#"{{"pid":{pid},"window_id":{hwnd_addr},"x":{x},"y":{y},"session":"{session}"}}"#))
} else { false }
}
};
eprintln!("[drive {session}] click Add Record -> {}", if ok { "ok" } else { "FAIL" });
// Verify a record was actually COMMITTED: the "Records: N"
// counter must increment. (Checking for the typed text
// alone is fooled by it sitting in the field.) Only the
// a11y corners expose the counter; GDI's is drawn pixels,
// so it's verified visually (its grid grows on screen).
thread::sleep(Duration::from_millis(300));
let t2 = run_call_out(&cua, "get_window_state", &ax);
if let Some(n) = records_count(&t2) {
let landed = n as i64 > seen_records;
seen_records = n as i64;
eprintln!("[verify {session}] {title} Records={n} -> {}",
if landed { "COMMITTED ✓" } else { "FAIL (no new record)" });
}
}
Action::DrawMaze => {
// Pure coordinate drawing — NO element targeting, NO app
// cooperation. Each maze segment is one straight
// press-drag-release into the common REGION; cua-driver
// PostMessages the drag where it can and pen-injects it
// where the canvas (Chromium/WPF) drops posted mouse.
let map = |mx: f64, my: f64| {
let cfx = REGION[0] + mx * (REGION[2] - REGION[0]);
let cfy = REGION[1] + my * (REGION[3] - REGION[1]);
client_rel_to_local_px(hwnd, cfx, cfy)
};
let mut drawn = 0;
for (x0, y0, x1, y1) in MAZE.iter() {
if let (Some((fx, fy)), Some((tx, ty))) = (map(*x0, *y0), map(*x1, *y1)) {
if run_call(&cua, "drag", &format!(
r#"{{"pid":{pid},"window_id":{hwnd_addr},"from_x":{fx},"from_y":{fy},"to_x":{tx},"to_y":{ty},"dispatch":"background","session":"{session}"}}"#)) {
drawn += 1;
}
}
thread::sleep(Duration::from_millis(120));
}
eprintln!("[draw {session}] maze: {drawn}/{} segments", MAZE.len());
// Read back by SCREENSHOT (WGC captures the occluded
// window) and diff the rendered ink against the
// rasterized reference maze — normalized into the same
// REGION grid, pass only under threshold.
thread::sleep(Duration::from_millis(400));
let shot = std::env::temp_dir().join(format!("maze_{session}.png"));
let _ = std::fs::remove_file(&shot);
run_call_shot(&cua, &format!(
r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"vision","max_image_dimension":4096,"session":"{session}"}}"#), &shot);
match score_maze(hwnd, &shot) {
Some(score) => eprintln!("[verify {session}] {title} maze score={score:.2} -> {}",
if score >= MAZE_PASS { "DREW MAZE ✓" } else { "FAIL (lines don't match)" }),
None => eprintln!("[verify {session}] {title} maze -> FAIL (no screenshot/region)"),
}
}
}
}
}));
}
eprintln!(
"[orch] ready — submit a record in the center MASTER; watch {} coloured cursors \
drive the corner terminals in the background.",
corners.len()
);
if std::env::args().any(|a| a == "--auto") {
let s2 = senders.clone();
thread::spawn(move || {
for i in 1..=3 {
thread::sleep(Duration::from_secs(3));
eprintln!("[orch] AUTO {i}: TYPE then CLICK");
for tx in &s2 {
let _ = tx.send(Action::Type {
text: format!("SUBJECT-{i:03}"),
});
}
thread::sleep(Duration::from_millis(1800));
// SAVE/"Add Record" button center (matches the GDI/master layout)
// for the pixel-driven GDI corner; a11y corners ignore rx/ry.
for tx in &s2 {
let _ = tx.send(Action::Click {
rx: 0.22,
ry: 0.495,
});
}
}
// Finally: draw the same maze into every corner's drawing pad by
// pure coordinate drags, then screenshot-diff each against the
// reference. Proves canvas actuation with no a11y, no app help.
thread::sleep(Duration::from_secs(2));
eprintln!("[orch] AUTO 4: DRAW MAZE (line tool) + screenshot diff");
for tx in &s2 {
let _ = tx.send(Action::DrawMaze);
}
});
}
let reader = BufReader::new(master_out);
for line in reader.lines().map_while(Result::ok) {
let parts: Vec<&str> = line.trim().split('\t').collect();
let action = match parts.as_slice() {
["CLICK", rx, ry] => rx
.parse::<f64>()
.ok()
.zip(ry.parse::<f64>().ok())
.map(|(rx, ry)| Action::Click { rx, ry }),
["TYPE", text] => Some(Action::Type {
text: (*text).to_string(),
}),
_ => None,
};
if let Some(a) = action {
eprintln!(
"[orch] user action: {a:?} -> driving {} corners",
senders.len()
);
for tx in &senders {
let _ = tx.send(a.clone());
}
}
}
drop(senders);
for h in handles {
let _ = h.join();
}
let _ = master.kill();
for mut k in kids {
let _ = k.kill();
}
let _ = daemon.kill();
}
#[derive(Clone, Debug)]
enum Action {
Click { rx: f64, ry: f64 },
Type { text: String },
DrawMaze,
}
fn json_escape(s: &str) -> String {
let mut o = String::with_capacity(s.len());
for ch in s.chars() {
match ch {
'"' => o.push_str("\\\""),
'\\' => o.push_str("\\\\"),
'\n' => o.push_str("\\n"),
'\r' => {}
'\t' => o.push_str("\\t"),
c => o.push(c),
}
}
o
}
/// `cua-driver call <tool> <json>` (proxies to the daemon). True on success.
fn run_call(cua: &PathBuf, tool: &str, json: &str) -> bool {
Command::new(cua)
.arg("call")
.arg(tool)
.arg(json)
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
}
/// Same, capturing stdout (for get_window_state readback).
fn run_call_out(cua: &PathBuf, tool: &str, json: &str) -> String {
Command::new(cua)
.arg("call")
.arg(tool)
.arg(json)
.stderr(Stdio::null())
.output()
.map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
.unwrap_or_default()
}
/// get_window_state writing the screenshot PNG to `out` (WGC; captures even a
/// fully occluded window). Used to read back what cua-driver drew.
fn run_call_shot(cua: &PathBuf, json: &str, out: &std::path::Path) -> bool {
Command::new(cua)
.arg("call")
.arg("get_window_state")
.arg(json)
.arg("--screenshot-out-file")
.arg(out)
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
}
/// DWM extended-frame size of `hwnd` — the pixel size of the WGC bitmap before
/// get_window_state downscales it to fit `max_image_dimension`.
fn window_frame_size(hwnd: HWND) -> Option<(i32, i32)> {
unsafe {
let mut d = RECT::default();
if DwmGetWindowAttribute(
hwnd,
DWMWA_EXTENDED_FRAME_BOUNDS,
&mut d as *mut _ as *mut core::ffi::c_void,
std::mem::size_of::<RECT>() as u32,
)
.is_err()
{
return None;
}
Some((d.right - d.left, d.bottom - d.top))
}
}
/// Shape-only F1 (0..1) of the maze cua-driver drew vs the reference maze.
///
/// Reads the ink from the window screenshot (cropped to the drawing REGION),
/// then normalizes BOTH the drawn ink and the reference into a unit grid by
/// their own bounding boxes — so the score reflects the *shape* of the lines,
/// not where on the canvas they landed or at what scale (the user only wants
/// the shape penalized). The screenshot is downscaled to fit
/// `max_image_dimension`, so REGION (full-res bitmap px) is scaled by the
/// PNG/frame ratio before cropping.
fn score_maze(hwnd: HWND, png: &std::path::Path) -> Option<f64> {
let img = image::open(png).ok()?.to_luma8();
let (iw, ih) = (img.width() as i32, img.height() as i32);
let (fw, fh) = window_frame_size(hwnd)?;
if fw <= 0 || fh <= 0 {
return None;
}
let (fx, fy) = (iw as f64 / fw as f64, ih as f64 / fh as f64);
let (ax, ay) = client_rel_to_local_px(hwnd, REGION[0], REGION[1])?;
let (bx, by) = client_rel_to_local_px(hwnd, REGION[2], REGION[3])?;
let cx0 = (((ax.min(bx)) as f64 * fx).floor() as i32).max(0);
let cy0 = (((ay.min(by)) as f64 * fy).floor() as i32).max(0);
let cx1 = (((ax.max(bx)) as f64 * fx).ceil() as i32).min(iw - 1);
let cy1 = (((ay.max(by)) as f64 * fy).ceil() as i32).min(ih - 1);
if cx1 - cx0 < 12 || cy1 - cy0 < 12 {
return None;
}
// Drawn ink + its bounding box.
let mut ink: Vec<(i32, i32)> = Vec::new();
let (mut nx, mut ny, mut xx, mut xy) = (i32::MAX, i32::MAX, i32::MIN, i32::MIN);
for y in cy0..=cy1 {
for x in cx0..=cx1 {
if img.get_pixel(x as u32, y as u32).0[0] < 150 {
ink.push((x, y));
if x < nx {
nx = x
}
if x > xx {
xx = x
}
if y < ny {
ny = y
}
if y > xy {
xy = y
}
}
}
}
const N: usize = 96;
if ink.len() < 20 || xx <= nx || xy <= ny {
return None;
}
let (bw, bh) = ((xx - nx) as f64, (xy - ny) as f64);
let mut a = vec![false; N * N];
for (x, y) in ink {
let gx = (((x - nx) as f64 / bw) * (N as f64 - 1.0)).round() as usize;
let gy = (((y - ny) as f64 / bh) * (N as f64 - 1.0)).round() as usize;
a[gy.min(N - 1) * N + gx.min(N - 1)] = true;
}
// Reference maze, normalized to ITS bounding box the same way.
let (mut rnx, mut rny, mut rxx, mut rxy) = (f64::MAX, f64::MAX, f64::MIN, f64::MIN);
for (x0, y0, x1, y1) in MAZE.iter() {
for (px, py) in [(*x0, *y0), (*x1, *y1)] {
if px < rnx {
rnx = px
}
if px > rxx {
rxx = px
}
if py < rny {
rny = py
}
if py > rxy {
rxy = py
}
}
}
let (rbw, rbh) = ((rxx - rnx).max(1e-6), (rxy - rny).max(1e-6));
let mut b = vec![false; N * N];
for (x0, y0, x1, y1) in MAZE.iter() {
raster_line(
&mut b,
N,
(x0 - rnx) / rbw,
(y0 - rny) / rbh,
(x1 - rnx) / rbw,
(y1 - rny) / rbh,
);
}
Some(mask_f1(&a, &b, N, 2))
}
/// Bresenham rasterize a normalized [0,1] segment into the N×N reference mask.
fn raster_line(mask: &mut [bool], n: usize, x0: f64, y0: f64, x1: f64, y1: f64) {
let s = n as f64 - 1.0;
let (px1, py1) = ((x1 * s) as i32, (y1 * s) as i32);
let (mut x, mut y) = ((x0 * s) as i32, (y0 * s) as i32);
let (dx, dy) = ((px1 - x).abs(), -(py1 - y).abs());
let (sx, sy) = (if x < px1 { 1 } else { -1 }, if y < py1 { 1 } else { -1 });
let mut err = dx + dy;
loop {
if x >= 0 && y >= 0 && (x as usize) < n && (y as usize) < n {
mask[y as usize * n + x as usize] = true;
}
if x == px1 && y == py1 {
break;
}
let e2 = 2 * err;
if e2 >= dy {
err += dy;
x += sx;
}
if e2 <= dx {
err += dx;
y += sy;
}
}
}
/// Chebyshev dilation by radius `r`.
fn dilate(m: &[bool], n: usize, r: i32) -> Vec<bool> {
let mut o = vec![false; n * n];
for y in 0..n as i32 {
for x in 0..n as i32 {
if !m[y as usize * n + x as usize] {
continue;
}
for dy in -r..=r {
for dx in -r..=r {
let (nx, ny) = (x + dx, y + dy);
if nx >= 0 && ny >= 0 && (nx as usize) < n && (ny as usize) < n {
o[ny as usize * n + nx as usize] = true;
}
}
}
}
}
o
}
/// F1 of two ink masks, each dilated by `r` so near-misses count as matches.
fn mask_f1(a: &[bool], b: &[bool], n: usize, r: i32) -> f64 {
let (da, db) = (dilate(a, n, r), dilate(b, n, r));
let (mut ca, mut cb, mut ma, mut mb) = (0usize, 0usize, 0usize, 0usize);
for i in 0..n * n {
if a[i] {
ca += 1;
if db[i] {
ma += 1;
}
}
if b[i] {
cb += 1;
if da[i] {
mb += 1;
}
}
}
if ca == 0 || cb == 0 {
return 0.0;
}
let precision = ma as f64 / ca as f64; // drawn ink that is near the reference
let recall = mb as f64 / cb as f64; // reference covered by drawn ink
if precision + recall == 0.0 {
0.0
} else {
2.0 * precision * recall / (precision + recall)
}
}
/// First `[N]` element index on a tree line matching `pred`.
fn find_idx(tree: &str, pred: impl Fn(&str) -> bool) -> Option<u64> {
// The tree is one JSON line with literal "\n" separators between rows.
for line in tree.split("\\n") {
if !pred(line) {
continue;
}
let st = line.find('[')? + 1;
let en = line[st..].find(']')? + st;
if let Ok(n) = line[st..en].trim().parse() {
return Some(n);
}
}
None
}
/// The `element_token` that get_window_state minted for row `[idx]`. Actions
/// address elements only by this token, which is bound to that read.
fn element_token(state: &str, idx: u64) -> Option<String> {
let state: serde_json::Value = serde_json::from_str(state).ok()?;
state["elements"]
.as_array()?
.iter()
.find(|element| element["element_index"].as_u64() == Some(idx))?["element_token"]
.as_str()
.map(str::to_owned)
}
/// Parse the "Records: N" counter from a window-state tree (status bar).
fn records_count(tree: &str) -> Option<u32> {
let pos = tree.find("Records:")?;
let rest = &tree[pos + "Records:".len()..];
let digits: String = rest
.trim_start()
.chars()
.take_while(|c| c.is_ascii_digit())
.collect();
digits.parse().ok()
}
fn work_area() -> RECT {
use windows::Win32::UI::WindowsAndMessaging::{
SystemParametersInfoW, SPI_GETWORKAREA, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS,
};
let mut wa = RECT::default();
unsafe {
let _ = SystemParametersInfoW(
SPI_GETWORKAREA,
0,
Some(&mut wa as *mut _ as *mut core::ffi::c_void),
SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0),
);
}
if wa.right <= wa.left || wa.bottom <= wa.top {
use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};
unsafe {
wa = RECT {
left: 0,
top: 0,
right: GetSystemMetrics(SM_CXSCREEN),
bottom: GetSystemMetrics(SM_CYSCREEN),
};
}
}
wa
}
fn place(hwnd: HWND, x: i32, y: i32, w: i32, h: i32, activate: bool) {
if hwnd.0.is_null() {
return;
}
let flags = if activate {
SWP_SHOWWINDOW
} else {
SWP_NOACTIVATE | SWP_SHOWWINDOW | SWP_NOZORDER
};
unsafe {
let _ = SetWindowPos(hwnd, HWND_TOP, x, y, w, h, flags);
}
}
/// Client-relative (0..1) -> the click tool's window-local screenshot-pixel
/// space (ClientToScreen minus the DWM extended-frame top-left + 1px inset).
fn client_rel_to_local_px(hwnd: HWND, rx: f64, ry: f64) -> Option<(i32, i32)> {
unsafe {
let mut cr = RECT::default();
GetClientRect(hwnd, &mut cr).ok()?;
let mut pt = POINT {
x: (rx * (cr.right - cr.left) as f64) as i32,
y: (ry * (cr.bottom - cr.top) as f64) as i32,
};
let _ = ClientToScreen(hwnd, &mut pt);
let mut dwm = RECT::default();
if DwmGetWindowAttribute(
hwnd,
DWMWA_EXTENDED_FRAME_BOUNDS,
&mut dwm as *mut _ as *mut core::ffi::c_void,
std::mem::size_of::<RECT>() as u32,
)
.is_err()
{
return Some((pt.x, pt.y));
}
Some((pt.x - dwm.left - 1, pt.y - dwm.top - 1))
}
}
// ── window discovery by title substring (case-insensitive) ────────────────────
struct Finder {
needle: String,
hwnd: HWND,
pid: u32,
}
unsafe extern "system" fn enum_cb(hwnd: HWND, lparam: LPARAM) -> BOOL {
let f = &mut *(lparam.0 as *mut Finder);
if !IsWindowVisible(hwnd).as_bool() {
return TRUE;
}
let mut buf = [0u16; 256];
let n = GetWindowTextW(hwnd, &mut buf);
if n > 0 {
let title = String::from_utf16_lossy(&buf[..n as usize]);
if title.to_lowercase().contains(&f.needle.to_lowercase()) {
let mut pid = 0u32;
GetWindowThreadProcessId(hwnd, Some(&mut pid));
f.hwnd = hwnd;
f.pid = pid;
return BOOL(0);
}
}
let _ = PWSTR::null();
TRUE
}
fn find_window_by_title(needle: &str) -> Option<(HWND, u32)> {
let deadline = Instant::now() + Duration::from_secs(8);
loop {
let mut f = Finder {
needle: needle.to_string(),
hwnd: HWND::default(),
pid: 0,
};
unsafe {
let _ = EnumWindows(Some(enum_cb), LPARAM(&mut f as *mut _ as isize));
}
if !f.hwnd.0.is_null() {
return Some((f.hwnd, f.pid));
}
if Instant::now() > deadline {
return None;
}
thread::sleep(Duration::from_millis(300));
}
}