Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(desktop): 修复截图内容区吸附与区域内移动闪烁 (#7661)

DavidShenXDdb628cc700af
diff --git a/apps/desktop/native/region-screenshot/macos-snapshot.swift b/apps/desktop/native/region-screenshot/macos-snapshot.swift
index 5e890c5730..a12394c628 100644
--- a/apps/desktop/native/region-screenshot/macos-snapshot.swift
+++ b/apps/desktop/native/region-screenshot/macos-snapshot.swift
@@ -7,10 +7,11 @@
//
// - {"id":1,"op":"windows"} on-screen windows, front to back (with owner pid).
// - {"id":2,"op":"elements","pid":P,"x":X,"y":Y} accessibility elements of
-// that app under the point, innermost first. The tree is walked from the
-// app's windows instead of hit-testing the screen, because Cindy's overlay
-// covers the screen. Needs the Accessibility permission; without it the
-// answer is empty and the overlay snaps to windows only.
+// that app under the point, innermost first. Hit-test the application and
+// walk its parents, with a bounded window-tree fallback. Needs Accessibility;
+// without it the answer is empty and the overlay snaps to windows only.
+// - {"id":6,"op":"window-at","x":X,"y":Y,"exclude":[N]} mouse hit window,
+// excluding only this screenshot's overlay window numbers.
// - {"id":3,"op":"frontmost"} / {"id":4,"op":"activate","pid":P} remember and
// give focus back to the app the user was in before the screenshot.
// - {"id":5,"op":"screens"} per display the top inset taken by the camera
@@ -132,6 +133,26 @@ func children(of element: AXUIElement) -> [AXUIElement] {
(attribute(element, kAXChildrenAttribute) as? [AXUIElement]) ?? []
}
+/// AppKit skips transparent pixels and windows that ignore mouse events.
+/// Only Cindy's screenshot overlays are excluded explicitly; an unknown front
+/// window must still block the captured windows behind it.
+func windowAt(point: CGPoint, excluding: Set<Int>) -> Int {
+ let cocoaPoint = CGPoint(x: point.x, y: (NSScreen.screens.first?.frame.maxY ?? 0) - point.y)
+ var below = 0
+ for _ in 0..<(excluding.count + 1) {
+ let hit = NSWindow.windowNumber(at: cocoaPoint, belowWindowWithWindowNumber: below)
+ if hit == 0 || hit == below { return 0 }
+ if !excluding.contains(hit) { return hit }
+ below = hit
+ }
+ return 0
+}
+
+func matches(_ a: CGRect, _ b: CGRect) -> Bool {
+ abs(a.minX - b.minX) < 2 && abs(a.minY - b.minY) < 2
+ && abs(a.width - b.width) < 2 && abs(a.height - b.height) < 2
+}
+
/// Smallest element among `candidates` whose frame contains the point.
func smallest(_ candidates: [AXUIElement], containing point: CGPoint) -> (AXUIElement, CGRect)? {
var best: (AXUIElement, CGRect)?
@@ -145,22 +166,85 @@ func smallest(_ candidates: [AXUIElement], containing point: CGPoint) -> (AXUIEl
return best
}
-func elementChain(pid: pid_t, point: CGPoint) -> [[Double]] {
- guard AXIsProcessTrusted() else { return [] }
+func elementChain(pid: pid_t, point: CGPoint, bounds: CGRect?) -> [String: Any] {
+ guard AXIsProcessTrusted() else { return ["rects": []] }
let started = Date()
elementDeadline = started.addingTimeInterval(elementBudget)
defer { elementDeadline = nil }
let app = AXUIElementCreateApplication(pid)
+ AXUIElementSetMessagingTimeout(app, 0.05)
+ var hit: AXUIElement?
+ if AXUIElementCopyElementAtPosition(app, Float(point.x), Float(point.y), &hit) == .success,
+ let hit = hit
+ {
+ var current = hit
+ var visited: [AXUIElement] = []
+ var chain: [CGRect] = []
+ var document: CGRect?
+ var scroll: CGRect?
+ for _ in 0..<maxDepth {
+ if elementDeadline!.timeIntervalSinceNow <= 0 || visited.contains(where: { CFEqual($0, current) }) { break }
+ visited.append(current)
+ let rect = frame(of: current)
+ if let rect = rect, rect.contains(point), chain.last != rect {
+ chain.append(rect)
+ }
+ let role = attribute(current, kAXRoleAttribute) as? String
+ if let rect = rect, rect.contains(point) {
+ if role == "AXWebArea" { document = rect }
+ if role == kAXScrollAreaRole { scroll = rect }
+ }
+ if role == kAXWindowRole {
+ // The app may own both the target and our screenshot overlay. Never use
+ // a hit from another window of that app (or an overlapping sibling).
+ if let rect = rect, bounds.map({ matches(rect, $0) }) ?? true {
+ var result: [String: Any] = ["rects": chain.map(rectArray)]
+ result["preferredRect"] = rectArray(document ?? scroll ?? rect)
+ return result
+ }
+ break
+ }
+ guard let parent = attribute(current, kAXParentAttribute),
+ CFGetTypeID(parent) == AXUIElementGetTypeID()
+ else { break }
+ current = parent as! AXUIElement
+ }
+ }
let windows = (attribute(app, kAXWindowsAttribute) as? [AXUIElement]) ?? []
- guard var current = smallest(windows, containing: point) else { return [] }
- var chain: [CGRect] = [current.1]
- for _ in 0..<maxDepth {
- if Date().timeIntervalSince(started) > elementBudget { break }
- guard let next = smallest(children(of: current.0), containing: point) else { break }
- if next.1 != chain.last { chain.append(next.1) }
- current = next
+ let matching = windows.filter { window in
+ guard let rect = frame(of: window) else { return false }
+ return bounds.map { matches(rect, $0) } ?? rect.contains(point)
+ }
+ guard let root = smallest(matching, containing: point) else { return ["rects": []] }
+ var pending: [(AXUIElement, [CGRect], CGRect?, CGRect?, Int)] = [(root.0, [root.1], nil, nil, 0)]
+ var best: [CGRect] = []
+ var preferred: CGRect?
+ var bestIsDocument = false
+ var visited = 0
+ while let (current, chain, document, scroll, depth) = pending.popLast() {
+ if Date().timeIntervalSince(started) > elementBudget || visited >= 256 { break }
+ visited += 1
+ if (document != nil && !bestIsDocument) || ((document != nil) == bestIsDocument && chain.count > best.count) {
+ best = chain; preferred = document ?? scroll; bestIsDocument = document != nil
+ }
+ if depth >= maxDepth { continue }
+ var candidates: [(AXUIElement, CGRect)] = []
+ for child in children(of: current).prefix(512) {
+ if Date().timeIntervalSince(started) > elementBudget { break }
+ if let rect = frame(of: child), rect.contains(point) { candidates.append((child, rect)) }
+ }
+ candidates.sort { $0.1.width * $0.1.height < $1.1.width * $1.1.height }
+ for (child, rect) in candidates.reversed() {
+ let role = attribute(child, kAXRoleAttribute) as? String
+ var next = chain
+ if next.last != rect { next.append(rect) }
+ pending.append((child, next, document ?? (role == "AXWebArea" ? rect : nil),
+ scroll ?? (role == kAXScrollAreaRole ? rect : nil), depth + 1))
+ }
}
- return chain.reversed().map(rectArray)
+ var result: [String: Any] = ["rects": best.reversed().map(rectArray)]
+ result["preferredRect"] = rectArray(preferred ?? root.1)
+ return result
}
func screenInsets() -> [[String: Any]] {
@@ -233,14 +317,30 @@ while let line = readLine() {
emit(["id": id, "keys": Array(labels.filter { CGEventSource.keyState(.combinedSessionState, key: $0.0) }.prefix(8).map { $0.1 })])
case "windows":
emit(["id": id, "windows": onScreenWindows()])
+ case "window-at":
+ if let x = (request["x"] as? NSNumber)?.doubleValue,
+ let y = (request["y"] as? NSNumber)?.doubleValue, x.isFinite, y.isFinite
+ {
+ let excluded = Set(request["exclude"] as? [Int] ?? [])
+ emit(["id": id, "hwnd": windowAt(point: CGPoint(x: x, y: y), excluding: excluded)])
+ } else {
+ emit(["id": id, "error": "invalid-request"])
+ }
case "screens":
emit(["id": id, "screens": screenInsets()])
case "elements":
if let pid = (request["pid"] as? NSNumber)?.int32Value,
let x = (request["x"] as? NSNumber)?.doubleValue,
- let y = (request["y"] as? NSNumber)?.doubleValue
+ let y = (request["y"] as? NSNumber)?.doubleValue, x.isFinite, y.isFinite
{
- emit(["id": id, "rects": elementChain(pid: pid, point: CGPoint(x: x, y: y))])
+ let rect = request["rect"] as? [Double]
+ let bounds = rect.flatMap { r -> CGRect? in
+ guard r.count == 4, r.allSatisfy({ $0.isFinite }), r[2] > r[0], r[3] > r[1] else { return nil }
+ return CGRect(x: r[0], y: r[1], width: r[2] - r[0], height: r[3] - r[1])
+ }
+ var result = elementChain(pid: pid, point: CGPoint(x: x, y: y), bounds: bounds)
+ result["id"] = id
+ emit(result)
} else {
emit(["id": id, "error": "invalid-request"])
}
diff --git a/apps/desktop/native/region-screenshot/windows-snapshot/src/main.rs b/apps/desktop/native/region-screenshot/windows-snapshot/src/main.rs
index 51c6dc53e7..7944674d9e 100644
--- a/apps/desktop/native/region-screenshot/windows-snapshot/src/main.rs
+++ b/apps/desktop/native/region-screenshot/windows-snapshot/src/main.rs
@@ -6,7 +6,8 @@
//! Electron converts them to DIP.
//!
//! - `{"id":1,"op":"windows"}` → visible top-level windows, top of the z-order
-//! first, using the DWM frame (without the invisible resize border).
+//! first, using the DWM frame (without the invisible resize border), plus
+//! bounded native child rectangles for synchronous default-region hit testing.
//! - `{"id":2,"op":"elements","hwnd":N,"x":X,"y":Y}` → UI Automation elements of
//! that window under the point, innermost first. The window is located from
//! the snapshot instead of hit-testing the screen, because Cindy's overlay
@@ -33,6 +34,8 @@ use windows::{
WindowsAndMessaging::{
EnumWindows, GetWindowLongW, GetWindowRect, IsIconic, IsWindowVisible, GWL_EXSTYLE,
WS_EX_TRANSPARENT, WindowFromPoint, GetAncestor, GA_ROOT, PostMessageW, WM_MOUSEWHEEL,
+ ChildWindowFromPointEx, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT,
+ GetWindow, GW_CHILD, GW_HWNDNEXT,
},
},
},
@@ -68,12 +71,14 @@ fn main() {
let hwnd = request.get("hwnd").and_then(Value::as_i64);
let x = request.get("x").and_then(Value::as_i64);
let y = request.get("y").and_then(Value::as_i64);
- match (&automation, hwnd, x, y) {
- (Some(automation), Some(hwnd), Some(x), Some(y)) => json!({
- "id": id,
- "rects": element_chain(automation, hwnd as isize, x as i32, y as i32),
- }),
- (None, ..) => json!({ "id": id, "rects": [] }),
+ match (hwnd, x.and_then(|v| i32::try_from(v).ok()), y.and_then(|v| i32::try_from(v).ok())) {
+ (Some(hwnd), Some(x), Some(y)) => {
+ let native = content_rect(HWND(hwnd as isize as *mut _), x, y);
+ let (rects, document) = automation.as_ref()
+ .map(|a| element_chain(a, hwnd as isize, x, y)).unwrap_or_default();
+ json!({ "id": id, "rects": rects,
+ "preferredRect": native.or(document).or_else(|| window_rect(HWND(hwnd as isize as *mut _))).as_ref().map(rect_json) })
+ },
_ => json!({ "id": id, "error": "invalid-request" }),
}
}
@@ -230,7 +235,10 @@ unsafe extern "system" fn collect_window(hwnd: HWND, data: LPARAM) -> BOOL {
return true.into();
}
if let Some(rect) = window_rect(hwnd) {
- windows.push(json!({ "hwnd": hwnd.0 as isize, "rect": rect_json(&rect) }));
+ let mut regions = Vec::new();
+ let mut remaining = 256;
+ snapshot_children(hwnd, &rect, 0, &mut remaining, &mut regions);
+ windows.push(json!({ "hwnd": hwnd.0 as isize, "rect": rect_json(&rect), "snapRects": regions }));
}
true.into()
}
@@ -251,47 +259,97 @@ fn area(rect: &RECT) -> i64 {
(rect.right - rect.left) as i64 * (rect.bottom - rect.top) as i64
}
-/// Walks the control view from the window down, at each level taking the
-/// smallest child that contains the point. Returns innermost first.
-fn element_chain(automation: &IUIAutomation, hwnd: isize, x: i32, y: i32) -> Vec<Value> {
+/// Native content hosts remain available even when Chromium has not enabled UIA.
+/// Start inside the captured window: screen hit testing would hit our overlay.
+fn content_rect(root: HWND, x: i32, y: i32) -> Option<RECT> {
+ let outer = window_rect(root)?;
+ let mut current = root;
+ let mut result = None;
+ for _ in 0..MAX_DEPTH {
+ let mut point = POINT { x, y };
+ if !unsafe { ScreenToClient(current, &mut point) }.as_bool() { break; }
+ let child = unsafe { ChildWindowFromPointEx(current, point, CWP_SKIPINVISIBLE | CWP_SKIPTRANSPARENT) };
+ if child.0.is_null() || child == current { break; }
+ let mut rect = RECT::default();
+ if unsafe { GetWindowRect(child, &mut rect) }.is_ok() && usable(&rect)
+ && contains(&rect, x, y) && area(&rect) < area(&outer) {
+ result = Some(rect);
+ }
+ current = child;
+ }
+ result
+}
+
+/// Capture once, before the overlay appears. Descendants precede parents and
+/// sibling subtrees retain native z-order, so the first containing region wins.
+/// Clipping to ancestors prevents a child from snapping outside its host.
+unsafe fn snapshot_children(parent: HWND, bounds: &RECT, depth: usize, remaining: &mut usize, regions: &mut Vec<Value>) {
+ if depth >= MAX_DEPTH || *remaining == 0 { return; }
+ let mut child = GetWindow(parent, GW_CHILD).unwrap_or_default();
+ while !child.0.is_null() && *remaining > 0 {
+ *remaining -= 1;
+ if IsWindowVisible(child).as_bool()
+ && GetWindowLongW(child, GWL_EXSTYLE) as u32 & WS_EX_TRANSPARENT.0 == 0 {
+ let mut rect = RECT::default();
+ if GetWindowRect(child, &mut rect).is_ok() {
+ rect.left = rect.left.max(bounds.left);
+ rect.top = rect.top.max(bounds.top);
+ rect.right = rect.right.min(bounds.right);
+ rect.bottom = rect.bottom.min(bounds.bottom);
+ if usable(&rect) {
+ snapshot_children(child, &rect, depth + 1, remaining, regions);
+ regions.push(rect_json(&rect));
+ }
+ }
+ }
+ child = GetWindow(child, GW_HWNDNEXT).unwrap_or_default();
+ }
+}
+
+/// Explore overlapping siblings within one budget. Chrome can expose an empty
+/// container before the equally sized branch that actually owns the document.
+fn element_chain(automation: &IUIAutomation, hwnd: isize, x: i32, y: i32) -> (Vec<Value>, Option<RECT>) {
let started = Instant::now();
- let mut chain: Vec<RECT> = Vec::new();
+ let mut best: Vec<RECT> = Vec::new();
+ let mut preferred = None;
let mut walk = || -> windows::core::Result<()> {
unsafe {
let cache = automation.CreateCacheRequest()?;
cache.AddProperty(UIA_BoundingRectanglePropertyId)?;
+ cache.AddProperty(UIA_ControlTypePropertyId)?;
let condition = automation.ControlViewCondition()?;
cache.SetTreeFilter(&condition)?;
- let mut current = automation.ElementFromHandle(HWND(hwnd as *mut _))?;
- for _ in 0..MAX_DEPTH {
- if started.elapsed() > ELEMENT_BUDGET {
- break;
+ let root = automation.ElementFromHandle(HWND(hwnd as *mut _))?;
+ let mut pending = vec![(root, Vec::<RECT>::new(), None::<RECT>, 0usize)];
+ let mut visited = 0;
+ while let Some((current, chain, document, depth)) = pending.pop() {
+ if started.elapsed() > ELEMENT_BUDGET || visited >= 256 { break; }
+ visited += 1;
+ if (document.is_some() && preferred.is_none())
+ || (document.is_some() == preferred.is_some() && chain.len() > best.len()) {
+ best = chain.clone(); preferred = document;
}
- let children = current.FindAllBuildCache(TreeScope_Children, &condition, &cache)?;
- let mut best: Option<(i64, IUIAutomationElement, RECT)> = None;
- for index in 0..children.Length()? {
- if started.elapsed() > ELEMENT_BUDGET {
- break;
- }
+ if depth >= MAX_DEPTH { continue; }
+ let Ok(children) = current.FindAllBuildCache(TreeScope_Children, &condition, &cache) else { continue };
+ let mut candidates = Vec::new();
+ for index in 0..children.Length()?.min(512) {
+ if started.elapsed() > ELEMENT_BUDGET { break; }
let Ok(child) = children.GetElement(index) else { continue };
let Ok(rect) = child.CachedBoundingRectangle() else { continue };
- if !usable(&rect) || !contains(&rect, x, y) {
- continue;
- }
- let size = area(&rect);
- if best.as_ref().map_or(true, |(best_size, ..)| size < *best_size) {
- best = Some((size, child, rect));
- }
+ if !usable(&rect) || !contains(&rect, x, y) { continue; }
+ let document = document.or_else(|| (child.CachedControlType().ok() == Some(UIA_DocumentControlTypeId)).then_some(rect));
+ candidates.push((child, rect, document));
}
- let Some((_, child, rect)) = best else { break };
- if chain.last() != Some(&rect) {
- chain.push(rect);
+ candidates.sort_by_key(|(_, rect, _)| area(rect));
+ for (child, rect, document) in candidates.into_iter().rev() {
+ let mut next = chain.clone();
+ if next.last() != Some(&rect) { next.push(rect); }
+ pending.push((child, next, document, depth + 1));
}
- current = child;
}
Ok(())
}
};
let _ = walk();
- chain.iter().rev().map(rect_json).collect()
+ (best.iter().rev().map(rect_json).collect(), preferred)
}
diff --git a/apps/desktop/scripts/check-region-snapping.mjs b/apps/desktop/scripts/check-region-snapping.mjs
new file mode 100644
index 0000000000..1f2197c33e
--- /dev/null
+++ b/apps/desktop/scripts/check-region-snapping.mjs
@@ -0,0 +1,259 @@
+#!/usr/bin/env node
+// E2E, explicitly invoked: node apps/desktop/scripts/check-region-snapping.mjs
+// Bug: Chrome's first of two equal UIA containers was empty, so hovering its
+// document selected the entire browser. Unit/fixture checks fake native replies.
+// This checks a headed Chrome, the shipping native helper and production overlay
+// using real replies. IPC sender validation remains in windows.test.ts.
+// Optional --position=2700,100 exercises a secondary display; --helper=<binary>
+// permits running the exact same regression against a pre-fix helper.
+import assert from 'node:assert/strict';
+import { spawn, execFileSync } from 'node:child_process';
+import { createInterface } from 'node:readline';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { chromium } from 'playwright-core';
+import { buildPage, desktop } from './ui-page/build.mjs';
+import { adhocPage } from './ui-page/registry.mjs';
+import { pageResponse, origin } from './ui-page/run.mjs';
+
+assert.equal(process.platform, 'win32', 'This native regression requires an interactive Windows desktop');
+const option = (name) => process.argv.find(arg => arg.startsWith(`--${name}=`))?.slice(name.length + 3);
+const position = (option('position') ?? '80,80').split(',').map(Number);
+assert.ok(position.length === 2 && position.every(Number.isFinite));
+const temp = await mkdtemp(path.join(os.tmpdir(), 'cindy-region-blackbox-'));
+const browserPath = option('browser') ?? path.join(process.env.PROGRAMFILES, 'Google/Chrome/Application/chrome.exe');
+assert.ok(existsSync(browserPath), 'Chrome is required (or pass --browser=<path>)');
+let browser, renderer, helper, nativeWindow;
+const pending = new Map();
+let sequence = 0;
+const ask = (payload) => new Promise((resolve, reject) => {
+ const id = ++sequence;
+ const timeout = setTimeout(() => { pending.delete(id); reject(new Error(`Native timeout: ${payload.op}`)); }, 5000);
+ pending.set(id, { resolve: (reply) => { clearTimeout(timeout); resolve(reply); }, reject, timeout });
+ helper.stdin.write(`${JSON.stringify({ id, ...payload })}\n`);
+});
+try {
+ let binary = option('helper');
+ if (!binary) {
+ execFileSync('cargo', ['build', '--release', '--locked', '--manifest-path', path.join(desktop, 'native/region-screenshot/windows-snapshot/Cargo.toml')], { stdio: 'inherit', windowsHide: true });
+ binary = path.join(desktop, 'native/region-screenshot/windows-snapshot/target/release/cindy-windows-screenshot-snapshot.exe');
+ }
+ helper = spawn(binary, [], { windowsHide: true, stdio: 'pipe' });
+ createInterface({ input: helper.stdout }).on('line', line => {
+ const reply = JSON.parse(line), task = pending.get(reply.id);
+ if (task) { pending.delete(reply.id); task.resolve(reply); }
+ });
+ helper.on('error', error => { for (const task of pending.values()) { clearTimeout(task.timeout); task.reject(error); } pending.clear(); });
+ // Real HWND hierarchy proves the snapshot path independently of Chromium
+ // versions that render the whole page into one native window.
+ nativeWindow = spawn('powershell.exe', ['-NoProfile', '-STA', '-Command', `
+ Add-Type -AssemblyName System.Windows.Forms
+ $form = New-Object System.Windows.Forms.Form
+ $form.StartPosition = 'Manual'
+ $form.Location = New-Object System.Drawing.Point(100,100)
+ $form.ClientSize = New-Object System.Drawing.Size(500,400)
+ $panel = New-Object System.Windows.Forms.Panel
+ $panel.Bounds = New-Object System.Drawing.Rectangle(30,60,350,250)
+ $inner = New-Object System.Windows.Forms.Panel
+ $inner.Bounds = New-Object System.Drawing.Rectangle(20,30,180,100)
+ $panel.Controls.Add($inner)
+ $form.Controls.Add($panel)
+ $form.Show()
+ $form.Hide()
+ $form.Show()
+ $r = $inner.RectangleToScreen($inner.ClientRectangle)
+ $json = @{ hwnd=$form.Handle.ToInt64(); rect=@($r.Left,$r.Top,$r.Right,$r.Bottom) } | ConvertTo-Json -Compress
+ [Console]::Out.WriteLine($json)
+ [Console]::Out.Flush()
+ [System.Windows.Forms.Application]::Run($form)
+ `], { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
+ const nativeGeometry = await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => reject(new Error('Native fixture did not open')), 10000);
+ createInterface({ input: nativeWindow.stdout }).once('line', line => { clearTimeout(timer); resolve(JSON.parse(line)); });
+ nativeWindow.once('error', error => { clearTimeout(timer); reject(error); });
+ });
+ let nativeSnapshot;
+ const nativeDeadline = Date.now() + 3000;
+ do {
+ nativeSnapshot = (await ask({ op: 'windows' })).windows.find(w => w.hwnd === nativeGeometry.hwnd);
+ if (nativeSnapshot) break;
+ await delay(20);
+ } while (Date.now() < nativeDeadline);
+ assert.ok(nativeSnapshot, `native test window is captured: ${JSON.stringify(nativeGeometry)}`);
+ const center = { x: (nativeGeometry.rect[0] + nativeGeometry.rect[2]) / 2, y: (nativeGeometry.rect[1] + nativeGeometry.rect[3]) / 2 };
+ assert.deepEqual(nativeSnapshot.snapRects.find(r => r[0] <= center.x && center.x < r[2] && r[1] <= center.y && center.y < r[3]),
+ nativeGeometry.rect, 'native snapshot hits the inner child before its parent');
+ nativeWindow.kill();
+ nativeWindow = undefined;
+ browser = await chromium.launchPersistentContext(path.join(temp, 'chrome'), {
+ executablePath: browserPath, headless: false, viewport: null,
+ args: ['--force-device-scale-factor=1', `--window-position=${position.join(',')}`, '--window-size=1000,800', '--no-first-run'],
+ });
+ const source = browser.pages()[0];
+ await source.setContent('<!doctype html><title>Region snapping regression</title><style>body{margin:0;font:18px sans-serif}header{padding:30px}main{padding:40px}button{padding:18px}section{height:260px;border:1px solid}</style><header><button>Code</button></header><main><section>File list</section><p>Document body</p></main>');
+ await source.bringToFront();
+ const geometry = await source.evaluate(() => ({ x: screenX, y: screenY, outerWidth, outerHeight, width: innerWidth, height: innerHeight }));
+ // Independently measure the webpage viewport from browser geometry, never from
+ // the helper reply being tested. Chrome's invisible resize border is symmetric.
+ const border = (geometry.outerWidth - geometry.width) / 2;
+ const content = { x: geometry.x + border, y: geometry.y + geometry.outerHeight - geometry.height - border, width: geometry.width, height: geometry.height };
+ const windows = (await ask({ op: 'windows' })).windows;
+ const target = windows.find(({ rect: r }) => Math.abs(r[0] - geometry.x) <= 10
+ && Math.abs(r[1] - geometry.y) <= 10 && Math.abs(r[2] - r[0] - geometry.outerWidth) <= 20);
+ assert.ok(target, 'Headed Chrome is visible');
+ const [left, top, right, bottom] = target.rect;
+ const viewport = { width: right - left, height: bottom - top };
+ const local = (r) => ({ x: r[0] - left, y: r[1] - top, width: r[2] - r[0], height: r[3] - r[1] });
+ const closeRect = (actual, expected, label) => {
+ assert.ok(actual, `${label}: missing region`);
+ for (const key of ['x', 'y', 'width', 'height']) assert.ok(Math.abs(actual[key] - expected[key]) <= 2, `${label} ${key}: ${actual[key]} != ${expected[key]}`);
+ };
+ assert.ok(Array.isArray(target.snapRects), 'native window snapshot includes default regions');
+ const contentLocal = { ...content, x: content.x - left, y: content.y - top };
+ const samples = [];
+ for (const selector of ['button', 'section', 'p']) {
+ const box = await source.locator(selector).boundingBox();
+ const point = { x: Math.round(content.x + box.x + box.width / 2), y: Math.round(content.y + box.y + box.height / 2) };
+ // A new Chrome starts accessibility lazily after its first UIA query. Poll
+ // the observable result with a deadline; an old helper still fails here.
+ let hit;
+ const deadline = Date.now() + 4000;
+ do {
+ hit = await ask({ op: 'elements', hwnd: target.hwnd, ...point });
+ if (hit.preferredRect && Math.abs(hit.preferredRect[1] - content.y) <= 2) break;
+ await delay(100);
+ } while (Date.now() < deadline);
+ const expected = { ...content, x: content.x - left, y: content.y - top };
+ closeRect(local(hit.preferredRect ?? hit.rects[0] ?? target.rect), expected, selector);
+ if (selector === 'button') assert.ok(hit.rects.some(r => r[2] - r[0] < content.width && r[3] - r[1] < content.height), 'native button branch must remain reachable');
+ samples.push({ selector, point, expected, native: hit });
+ }
+ const chromeHit = await ask({ op: 'elements', hwnd: target.hwnd, x: Math.round(left + viewport.width / 2), y: top + 25 });
+ closeRect(local(chromeHit.preferredRect ?? chromeHit.rects[0] ?? target.rect), { x: 0, y: 0, ...viewport }, 'browser-chrome');
+ // Some Chromium versions expose content only through UIA, without child HWNDs.
+ // Keep the actual native snapshot; never fabricate regions from expected DOM geometry.
+ const captured = (await ask({ op: 'windows' })).windows.find(w => w.hwnd === target.hwnd);
+ assert.ok(captured, 'browser remains available for capture');
+ // Playwright's bringToFront activates the tab but Windows can refuse focus
+ // stealing. Keep only our disposable Chrome above other apps during capture.
+ execFileSync('powershell.exe', ['-NoProfile', '-Command', `Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class RegionTestWindow { [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr h, IntPtr after, int x, int y, int w, int height, uint flags); }'; if (![RegionTestWindow]::SetWindowPos([IntPtr]${target.hwnd}, [IntPtr](-1), 0, 0, 0, 0, 0x13)) { exit 1 }`], { windowsHide: true });
+ const visible = (await ask({ op: 'windows' })).windows.find(({ rect: r }) => r[0] <= content.x + 50 && content.x + 50 < r[2] && r[1] <= content.y + 50 && content.y + 50 < r[3]);
+ assert.equal(visible?.hwnd, target.hwnd, 'Native capture must show the tested Chrome, not an occluding app');
+ // Wait for Windows to compose the newly exposed native window (E2E only).
+ await delay(100);
+ const frame = await ask({ op: 'frame', x: left, y: top, ...viewport, edge: 1600 });
+ assert.ok(frame.pixels, 'Native screenshot is available');
+ const bundle = await buildPage(adhocPage(path.join(desktop, 'scripts/fixtures/region-capture/native-overlay.tsx')));
+ renderer = await chromium.launch({ executablePath: browserPath, headless: true });
+ const report = [];
+ for (const theme of ['light', 'dark']) {
+ const context = await renderer.newContext({ viewport });
+ const page = await context.newPage();
+ const errors = []; page.on('pageerror', error => errors.push(error.message));
+ const queried = [];
+ // Hold actual native replies to expose intermediate fallback frames. The
+ // expected geometry is used only by assertions, never supplied to the UI.
+ let releaseElements;
+ let elementsReady = Promise.resolve();
+ await page.exposeFunction('nativeElements', async (x, y) => {
+ await elementsReady;
+ const hit = await ask({ op: 'elements', hwnd: target.hwnd, x: Math.round(x + left), y: Math.round(y + top) });
+ queried.push({ x, y, hit });
+ return { window: { id: 0, rect: { x: 0, y: 0, ...viewport } }, rects: hit.rects.map(local), ...(hit.preferredRect ? { preferredRect: local(hit.preferredRect) } : {}) };
+ });
+ await page.addInitScript(begin => { window.nativeBegin = begin; }, {
+ token: 'native-blackbox', locale: 'en', viewport, safeArea: { x: 0, y: 0, ...viewport }, scaleFactor: 1,
+ frame: { width: frame.width, height: frame.height, data: Array.from(Buffer.from(frame.pixels, 'base64')) },
+ windows: [{ id: 0, rect: { x: 0, y: 0, ...viewport }, snapRects: captured.snapRects.map(local) }], cursor: null, lastRegion: null,
+ });
+ await page.route('**/*', route => {
+ const response = pageResponse(route.request().url(), bundle, { page: 'native-region', case: 'default', theme, locale: 'en', platform: 'win32', viewport, data: {} });
+ return response ? route.fulfill(response) : route.abort();
+ });
+ await page.goto(origin);
+ await page.waitForFunction(() => window.__uiPage?.ready);
+ await page.mouse.move(contentLocal.x + 25, contentLocal.y + 35);
+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor();
+ elementsReady = new Promise(resolve => { releaseElements = resolve; });
+ // Observe DOM commits and painted frames across continuous movement, not
+ // just the final result after an async wait (the gap in the original test).
+ await page.evaluate(() => {
+ window.regionTrace = [];
+ const sample = () => {
+ const outline = document.querySelector('div[style*="100000px"]');
+ if (outline) {
+ const { x, y, width, height } = outline.getBoundingClientRect();
+ window.regionTrace.push({ x, y, width, height });
+ }
+ };
+ const observer = new MutationObserver(sample);
+ observer.observe(document.body, { attributes: true, childList: true, subtree: true });
+ let frame;
+ const tick = () => { sample(); frame = requestAnimationFrame(tick); };
+ frame = requestAnimationFrame(tick);
+ window.stopRegionTrace = () => { observer.disconnect(); cancelAnimationFrame(frame); sample(); return window.regionTrace; };
+ });
+ for (let step = 0; step < 40; step++) {
+ await page.mouse.move(contentLocal.x + 30 + step * 10, contentLocal.y + 40 + step * 4);
+ closeRect(await page.locator('div[style*="100000px"]').boundingBox(), contentLocal, `${theme}/pending-move-${step}`);
+ }
+ const trace = await page.evaluate(() => window.stopRegionTrace());
+ assert.ok(trace.length >= 40, 'observe each move while native queries are pending');
+ trace.forEach((rect, index) => closeRect(rect, contentLocal, `${theme}/frame-${index}`));
+ await writeFile(path.join(temp, `${theme}-movement.json`), JSON.stringify(trace));
+ // Crossing the content boundary is synchronous even with the query held.
+ await page.mouse.move(viewport.width / 2, 25);
+ closeRect(await page.locator('div[style*="100000px"]').boundingBox(), { x: 0, y: 0, ...viewport }, `${theme}/pending-boundary`);
+ releaseElements();
+ for (const sample of samples) {
+ await page.mouse.move(sample.point.x - left, sample.point.y - top);
+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor();
+ // Observe the actual rendered outline, not a test-only selection accessor.
+ const outline = page.locator('div[style*="100000px"]');
+ closeRect(await outline.boundingBox(), sample.expected, `${theme}/${sample.selector}`);
+ await page.screenshot({ path: path.join(temp, `${theme}-${sample.selector}.png`) });
+ }
+ const before = queried.length;
+ await page.mouse.move(viewport.width / 2, 25);
+ await page.getByText(`${viewport.width} × ${viewport.height}`, { exact: true }).waitFor();
+ // Return to the button and exercise both finer and coarser candidates.
+ const first = samples[0];
+ const buttonQueries = queried.length;
+ await page.mouse.move(first.point.x - left, first.point.y - top);
+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor();
+ // The default is now immediate; wheel checks must wait for fine elements,
+ // rather than treating the content outline as proof that UIA has completed.
+ const fineDeadline = Date.now() + 5000;
+ const fineReady = () => queried.slice(buttonQueries).some(({ x, y, hit }) =>
+ Math.abs(x - (first.point.x - left)) < 1 && Math.abs(y - (first.point.y - top)) < 1
+ && hit.rects.some(r => r[2] - r[0] < content.width && r[3] - r[1] < content.height));
+ while (!fineReady() && Date.now() < fineDeadline) await delay(20);
+ assert.ok(fineReady(), 'fresh native fine elements are available at the button');
+ await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));
+ await page.mouse.wheel(0, 120);
+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor({ state: 'hidden' });
+ await page.mouse.wheel(0, -120);
+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor();
+ await page.mouse.click(first.point.x - left, first.point.y - top);
+ await page.mouse.move(viewport.width - 20, 25);
+ assert.equal(await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).count(), 1, 'click locks content selection');
+ assert.ok(queried.length > before, 'moving from chrome into content performs a fresh native query');
+ assert.deepEqual(errors, []);
+ await page.screenshot({ path: path.join(temp, `${theme}-selected.png`) });
+ report.push({ theme, content, window: target.rect, nativeRegions: captured.snapRects.length, movementSamples: trace.length, checks: ['pending-movement', 'pending-boundary', 'button', 'file-list', 'body', 'browser-chrome', 'wheel', 'click'] });
+ await context.close();
+ }
+ await writeFile(path.join(temp, 'report.json'), JSON.stringify(report, null, 2));
+ console.log(`REGION_BLACKBOX=passed evidence=${temp}`);
+} finally {
+ await renderer?.close();
+ await browser?.close();
+ helper?.kill();
+ nativeWindow?.kill();
+ for (const task of pending.values()) clearTimeout(task.timeout);
+ // Keep only reviewable screenshots/report; remove the isolated browser profile.
+ await rm(path.join(temp, 'chrome'), { recursive: true, force: true });
+}
diff --git a/apps/desktop/scripts/fixtures/region-capture/capture-fixture.tsx b/apps/desktop/scripts/fixtures/region-capture/capture-fixture.tsx
index ab1bb57b44..4912d80dd3 100644
--- a/apps/desktop/scripts/fixtures/region-capture/capture-fixture.tsx
+++ b/apps/desktop/scripts/fixtures/region-capture/capture-fixture.tsx
@@ -49,6 +49,7 @@ function ScrollFixture({ bridge, locale }: { bridge: RegionScreenshotOverlayBrid
mountPage({
render: (ctx) => {
+ const snapping = ctx.case === 'snapping';
const mode = (['default', 'idle', 'inactive', 'drag', 'preview-motion'].includes(ctx.case) ? 'scroll' : ctx.case) as RegionCaptureMode;
qa.idle = ctx.case === 'idle';
qa.manualScroll = ctx.case === 'preview-motion' ? 300 : null;
@@ -56,10 +57,11 @@ mountPage({
let selected = { x: 100, y: 100, width: 480, height: 240 };
const bridge: RegionScreenshotOverlayBridge = {
autoScroll: async (_token, enabled) => { qa.automatic = enabled; return true; },
- shellReady() { queueMicrotask(() => begin?.({ token: 'fixture', locale: ctx.locale, viewport: { width: innerWidth, height: innerHeight }, scaleFactor: 1, frame: frame(), windows: [], lastRegion: null, cursor: ctx.case === 'inactive' ? null : { x: 0, y: 0 }, safeArea: { x: 0, y: 0, width: innerWidth, height: innerHeight } })); },
+ shellReady() { queueMicrotask(() => begin?.({ token: 'fixture', locale: ctx.locale, viewport: { width: innerWidth, height: innerHeight }, scaleFactor: 1, frame: frame(), windows: snapping ? [{ id: 0, rect: { x: 0, y: 0, width: 700, height: 500 } }] : [], lastRegion: null, cursor: ctx.case === 'inactive' ? null : snapping ? { x: 150, y: 180 } : { x: 0, y: 0 }, safeArea: { x: 0, y: 0, width: innerWidth, height: innerHeight } })); },
onBegin: (callback) => { begin = callback; return () => { begin = undefined; }; }, onEnd: () => () => {}, onClear: () => () => {},
painted() {}, focused() {}, selectionStarted() {}, copyText() {}, cancel() {}, abandon() {},
- elementsAt: async () => [], startLive: async (_token, captureMode, selection) => { selected = selection; return { mode: captureMode, selection, first: frame(qa.manualScroll ?? 0), scrollLayout: scrollCaptureLayout(selection, { width: innerWidth, height: innerHeight })!, recordingLayout: { viewport: { width: innerWidth, height: innerHeight }, controls: recordingControlsBounds(selection, { width: innerWidth, height: innerHeight }, false)! } }; },
+ elementsAt: async (_token, _window, x) => snapping ? { window: { id: 1, rect: { x: 100, y: 100, width: 480, height: 360 } }, rects: [x < 320 ? { x: 120, y: 140, width: 180, height: 140 } : { x: 340, y: 140, width: 200, height: 140 }] } : null,
+ startLive: async (_token, captureMode, selection) => { selected = selection; return { mode: captureMode, selection, first: frame(qa.manualScroll ?? 0), scrollLayout: scrollCaptureLayout(selection, { width: innerWidth, height: innerHeight })!, recordingLayout: { viewport: { width: innerWidth, height: innerHeight }, controls: recordingControlsBounds(selection, { width: innerWidth, height: innerHeight }, false)! } }; },
recordingControls: async (_token, expanded, point) => recordingControlsBounds(selected, { width: innerWidth, height: innerHeight }, expanded, point),
previewRecording: async () => false,
restoreEditor: async () => { qa.restored = true; return true; },
diff --git a/apps/desktop/scripts/fixtures/region-capture/native-overlay.tsx b/apps/desktop/scripts/fixtures/region-capture/native-overlay.tsx
new file mode 100644
index 0000000000..385ddcabeb
--- /dev/null
+++ b/apps/desktop/scripts/fixtures/region-capture/native-overlay.tsx
@@ -0,0 +1,25 @@
+// Native black-box runner supplies captured pixels and live helper replies.
+// Only the transport is adapted; selection, pointer/wheel handling and painting
+// all run through the production overlay. No expected rectangle enters the UI.
+import { mountPage } from '../../ui-page/harness';
+import { RegionScreenshotOverlay } from '../../../src/renderer/features/region-screenshot/RegionScreenshotOverlay';
+import type { RegionScreenshotBegin, RegionScreenshotHit } from '../../../src/shared/regionScreenshot';
+import type { RegionScreenshotOverlayBridge } from '../../../src/renderer/features/region-screenshot/bridge';
+
+const native = window as unknown as {
+ nativeBegin: RegionScreenshotBegin;
+ nativeElements: (x: number, y: number) => Promise<RegionScreenshotHit>;
+};
+mountPage({ render: () => {
+ let begin: Parameters<RegionScreenshotOverlayBridge['onBegin']>[0] | undefined;
+ const bridge = {
+ onBegin: (callback: typeof begin) => { begin = callback; return () => { begin = undefined; }; },
+ onEnd: () => () => {}, onClear: () => () => {},
+ shellReady: () => queueMicrotask(() => begin?.({ ...native.nativeBegin,
+ frame: { ...native.nativeBegin.frame, data: new Uint8Array(native.nativeBegin.frame.data) },
+ })),
+ elementsAt: (_token: string, _window: number, x: number, y: number) => native.nativeElements(x, y),
+ painted() {}, focused() {}, selectionStarted() {}, abandon() {}, cancel() {}, copyText() {},
+ } as RegionScreenshotOverlayBridge;
+ return <RegionScreenshotOverlay bridge={bridge} />;
+} });
diff --git a/apps/desktop/scripts/fixtures/region-capture/ui-page.mjs b/apps/desktop/scripts/fixtures/region-capture/ui-page.mjs
index b09da3dabc..5cc66a3a9a 100644
--- a/apps/desktop/scripts/fixtures/region-capture/ui-page.mjs
+++ b/apps/desktop/scripts/fixtures/region-capture/ui-page.mjs
@@ -4,6 +4,7 @@ export default {
real: 'RegionScreenshotOverlay, ScrollCaptureOverlay, LiveCapturePanel, ScrollCapture, CaptureRecorder, real Chromium canvas/MediaRecorder and GIF encoding',
notCovered: 'Native screen acquisition, global keyboard sampling, macOS permissions and native save dialogs; frame source is synthetic.',
cases: {
+ snapping: { about: 'Correct a transparent window guess, hover adjacent elements, cycle hierarchy and click to select' },
default: { about: 'Scroll overlapping text, finish, export the actual stitched pixels' },
idle: { about: 'Repainting before the first scroll keeps the gesture hint, then scrolling still exports a long image' },
'preview-motion': { about: 'Prepending reveals new rows without jumping existing content; growth and scrolling share one animation' },
@@ -14,6 +15,21 @@ export default {
},
async ready(page) { await page.locator('canvas').first().waitFor(); },
async check(page, ui) {
+ if (ui.case === 'snapping') {
+ await page.getByText('180 × 140', { exact: true }).waitFor();
+ await ui.capture('first-element');
+ await page.mouse.move(400, 180);
+ await page.getByText('200 × 140', { exact: true }).waitFor();
+ await ui.capture('adjacent-element');
+ await page.mouse.wheel(0, -120);
+ await page.getByText('480 × 360', { exact: true }).waitFor();
+ await ui.capture('corrected-window');
+ await page.mouse.click(400, 180);
+ await page.mouse.move(650, 550);
+ assert.equal(await page.getByText('480 × 360', { exact: true }).count(), 1, 'click keeps the resolved window selected');
+ await ui.capture('selected');
+ return;
+ }
if (ui.case === 'preview-motion') await page.emulateMedia({ reducedMotion: 'no-preference' });
const isScroll = ['default', 'idle', 'preview-motion'].includes(ui.case);
const button = async (key) => page.getByRole('button', { name: await ui.t(key), exact: true });
diff --git a/apps/desktop/src/main/region-screenshot/__tests__/snapshotHost.test.ts b/apps/desktop/src/main/region-screenshot/__tests__/snapshotHost.test.ts
index cb522174de..98b7c0e308 100644
--- a/apps/desktop/src/main/region-screenshot/__tests__/snapshotHost.test.ts
+++ b/apps/desktop/src/main/region-screenshot/__tests__/snapshotHost.test.ts
@@ -64,6 +64,31 @@ describe('snapshot helper failure isolation', () => {
await expect(helper.windows()).resolves.toEqual([]);
});
+ // Region identification regression: rectangle z-order cannot identify a
+ // click-through overlay. Exercise the native protocol through the production
+ // host; an opaque unknown window must not expose windows behind it. Unit,
+ // in-memory child streams; existing tests cover failures, not target identity.
+ it('resolves the native hit only against captured windows, without falling through an occluder', async () => {
+ vi.stubGlobal('process', Object.create(process, { platform: { value: 'darwin' } }));
+ const captured = [
+ { handle: 5, pid: 9, rect: [0, 0, 200, 300] as [number, number, number, number] },
+ { handle: 6, pid: 10, rect: [20, 20, 100, 100] as [number, number, number, number] },
+ ];
+ const request = helper.windowAt(captured, 30, 40, [99]);
+ expect(JSON.parse(child.stdin.read().toString())).toMatchObject({ op: 'window-at', x: 30, y: 40, exclude: [99] });
+ child.stdout.write(JSON.stringify({ id: 1, hwnd: 6 }) + '\n');
+ await expect(request).resolves.toBe(captured[1]);
+ const occluded = helper.windowAt(captured, 30, 40, [99]);
+ child.stdout.write(JSON.stringify({ id: 2, hwnd: 77 }) + '\n');
+ await expect(occluded).resolves.toBeNull();
+ const stale = helper.windowAt(captured, 150, 40, [99]);
+ child.stdout.write(JSON.stringify({ id: 3, hwnd: 6 }) + '\n');
+ await expect(stale).resolves.toBeNull();
+ const unavailable = helper.windowAt(captured, 30, 40, [99]);
+ child.stdout.write(JSON.stringify({ id: 4, error: 'unknown-op' }) + '\n');
+ await expect(unavailable).resolves.toBe(captured[0]);
+ });
+
it('bounds a stuck helper instead of accumulating timed-out queries', async () => {
const result = helper.windows();
await vi.advanceTimersByTimeAsync(1500);
@@ -80,12 +105,31 @@ describe('snapshot helper failure isolation', () => {
JSON.stringify({
id: 1,
windows: [
- { hwnd: 5, pid: 9, rect: [0, 0, 20, 30] },
+ // Captured child geometry is optional and must stay within its window.
+ { hwnd: 5, pid: 9, rect: [0, 0, 20, 30], snapRects: [[0, 5, 20, 30], [-1, 0, 20, 30], [0, 0, -1, 2], null] },
+ { hwnd: 7, rect: [0, 0, 20, 30] },
{ hwnd: 6, rect: [0, 0, -1, 2] },
],
}) + '\n',
);
- await expect(result).resolves.toEqual([{ handle: 5, pid: 9, rect: [0, 0, 20, 30] }]);
+ await expect(result).resolves.toEqual([
+ { handle: 5, pid: 9, rect: [0, 0, 20, 30], snapRects: [[0, 5, 20, 30]] },
+ { handle: 7, pid: null, rect: [0, 0, 20, 30] },
+ ]);
+ });
+
+ // Browser content regression: the native preferred rectangle must survive
+ // the production transport, while malformed/old replies remain usable.
+ // Existing tests only cover window geometry; unit, one in-memory response.
+ it.each([
+ { preferredRect: [10, 20, 190, 280], expected: { preferredRect: [10, 20, 190, 280] } },
+ { preferredRect: [100, 100, 190, 280], expected: {} },
+ { preferredRect: [0, 0, -1, 30], expected: {} },
+ { preferredRect: undefined, expected: {} },
+ ])('preserves only a valid content region containing the query point: $preferredRect', async ({ preferredRect, expected }) => {
+ const result = helper.elements({ handle: 5, pid: null, rect: [0, 0, 200, 300] }, 30, 40);
+ child.stdout.write(JSON.stringify({ id: 1, rects: [[20, 30, 60, 70], [100, 100, 110, 110], null], preferredRect }) + '\n');
+ await expect(result).resolves.toEqual({ rects: [[20, 30, 60, 70]], ...expected });
});
// Unit: resolving through a transparent overlay must not admit a new window
diff --git a/apps/desktop/src/main/region-screenshot/__tests__/windows.test.ts b/apps/desktop/src/main/region-screenshot/__tests__/windows.test.ts
index 6d7bb87fb0..e999a32aea 100644
--- a/apps/desktop/src/main/region-screenshot/__tests__/windows.test.ts
+++ b/apps/desktop/src/main/region-screenshot/__tests__/windows.test.ts
@@ -101,6 +101,7 @@ function fakeWindow() {
show: vi.fn(() => { visible = true; window.emit('show'); }),
isDestroyed: () => destroyed,
isVisible: () => visible,
+ getMediaSourceId: () => 'window:999:0',
setVisibleOnAllWorkspaces: vi.fn(),
setAlwaysOnTop: vi.fn(),
setBounds: vi.fn(),
@@ -162,6 +163,44 @@ describe('screenshot window lifecycle', () => {
vi.unstubAllGlobals();
});
+ // Region detection: native hit-testing can replace the transparent front
+ // rectangle. Existing lifecycle tests never queried elements. Minimal input:
+ // two overlapping windows on an offset display; unit, no OS UI or real waits.
+ it('returns the corrected window in display coordinates and drops a query after the session closes', async () => {
+ vi.stubGlobal('process', Object.create(process, { platform: { value: 'darwin' } }));
+ const monitor = { ...display, bounds: { x: -100, y: 100, width: 100, height: 100 } } as Display;
+ const overlay = pool.ensure(monitor)!;
+ const owner = overlay.window.webContents;
+ pool.markShellReady(owner, owner.mainFrame);
+ const captured = [
+ { handle: 1, pid: 1, rect: [-100, 100, 0, 200] as [number, number, number, number], snapRects: [[-100, 120, 0, 200] as [number, number, number, number]] },
+ { handle: 2, pid: 2, rect: [-90, 110, -10, 190] as [number, number, number, number] },
+ ];
+ const hit = vi.fn(async () => captured[1]);
+ const elements = vi.fn(async () => ({ rects: [[-80, 120, -50, 150]], preferredRect: [-80, 120, -50, 150] }));
+ const session = new RegionScreenshotSession({ pool, captures: [{ display: monitor, image }], helper: { windowAt: hit, elements } as unknown as SnapshotHelper, windows: captured, lastRegions: new Map(), onClosed: vi.fn(), onResult: vi.fn() });
+ await session.open();
+ // Snapshot defaults must reach the first paint on an offset display, before
+ // any ELEMENTS_AT reply; previous coverage only inspected query conversion.
+ const begin = vi.mocked(owner.send).mock.calls.find(([name]) => name === 'region-screenshot:begin')![1];
+ expect(begin.windows[0].snapRects).toEqual([{ x: 0, y: 20, width: 100, height: 80 }]);
+ await expect(session.elementsAt(owner, owner.mainFrame, session.token, 0, 25, 25)).resolves.toEqual({
+ window: { id: 1, rect: { x: 10, y: 10, width: 80, height: 80 } },
+ rects: [{ x: 20, y: 20, width: 30, height: 30 }],
+ preferredRect: { x: 20, y: 20, width: 30, height: 30 },
+ });
+ expect(hit).toHaveBeenCalledWith(captured, -75, 125, [999]);
+ expect(elements).toHaveBeenCalledWith(captured[1], -75, 125);
+ hit.mockClear();
+ await expect(session.elementsAt(owner, owner.mainFrame, 'old-token', 0, 25, 25)).resolves.toBeNull();
+ await expect(session.elementsAt(owner, owner.mainFrame, session.token, 0, 125, 25)).resolves.toBeNull();
+ expect(hit).not.toHaveBeenCalled();
+ const pending = session.elementsAt(owner, owner.mainFrame, session.token, 0, 25, 25);
+ session.cancel(owner, owner.mainFrame, session.token);
+ await expect(pending).resolves.toBeNull();
+ expect(elements).toHaveBeenCalledTimes(1);
+ });
+
// 用户反馈:打开截图时整屏缩放。实机验证 Aura 动画;unit 保护临时开关
// 不泄漏到其它窗口、不清除用户启动参数,且异常也恢复。tier: unit。
it.each([
diff --git a/apps/desktop/src/main/region-screenshot/index.ts b/apps/desktop/src/main/region-screenshot/index.ts
index dd6c90156f..ac4a27bd8b 100644
--- a/apps/desktop/src/main/region-screenshot/index.ts
+++ b/apps/desktop/src/main/region-screenshot/index.ts
@@ -210,7 +210,7 @@ function registerIpc(): void {
ipcMain.handle(
C.ELEMENTS_AT,
(event, token: unknown, windowId: unknown, x: unknown, y: unknown) =>
- active?.elementsAt(event.sender, event.senderFrame, token, windowId, x, y) ?? [],
+ active?.elementsAt(event.sender, event.senderFrame, token, windowId, x, y) ?? null,
);
ipcMain.handle(
C.EXPORT,
diff --git a/apps/desktop/src/main/region-screenshot/session.ts b/apps/desktop/src/main/region-screenshot/session.ts
index e362f0d3d0..c714ec03ed 100644
--- a/apps/desktop/src/main/region-screenshot/session.ts
+++ b/apps/desktop/src/main/region-screenshot/session.ts
@@ -29,6 +29,7 @@ import {
type RegionScreenshotBegin,
type RegionScreenshotBitmap,
type RegionScreenshotWindow,
+ type RegionScreenshotHit,
} from '../../shared/regionScreenshot';
import { hideScreenshotOverlay, showScreenshotOverlay, type OverlayPool, type OverlayWindow } from './overlays';
import type { SnapshotHelper, SnapshotWindow } from './snapshotHost';
@@ -204,7 +205,11 @@ export class RegionScreenshotSession {
const windows: RegionScreenshotWindow[] = [];
this.deps.windows.forEach((window, id) => {
const rect = intersectRegion(toLocal(toGlobalDip(window.rect), display), viewport);
- if (rect) windows.push({ id, rect });
+ if (rect) windows.push({ id, rect,
+ ...(window.snapRects ? { snapRects: window.snapRects
+ .map((region) => intersectRegion(toLocal(toGlobalDip(region), display), viewport))
+ .filter((region): region is RegionRect => region !== null) } : {}),
+ });
});
const bitmap = image.toBitmap();
return {
@@ -284,18 +289,30 @@ export class RegionScreenshotSession {
windowId: unknown,
x: unknown,
y: unknown,
- ): Promise<RegionRect[]> {
+ ): Promise<RegionScreenshotHit | null> {
const member = this.member(sender, frame, token);
- if (!member || !Number.isInteger(windowId)) return [];
- if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x + y)) return [];
- const window = this.deps.windows[windowId as number];
- if (!window) return [];
+ if (!member || !Number.isInteger(windowId)) return null;
+ if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x + y)) return null;
+ if (!this.deps.windows[windowId as number]) return null;
const { display } = member.capture;
+ if (x < 0 || y < 0 || x >= display.bounds.width || y >= display.bounds.height) return null;
const global = { x: x + display.bounds.x, y: y + display.bounds.y };
const point = process.platform === 'win32' ? screen.dipToScreenPoint(global) : global;
- const rects = await this.deps.helper.elements(window, point.x, point.y);
- if (this.closed) return [];
- return rects.map((rect) => toLocal(toGlobalDip(rect), display));
+ const excluded = process.platform === 'darwin'
+ ? [...this.members.values()].filter(({ overlay }) => !overlay.window.isDestroyed())
+ .map(({ overlay }) => Number(overlay.window.getMediaSourceId().split(':')[1]))
+ .filter((id) => Number.isSafeInteger(id) && id > 0)
+ : [];
+ const window = await this.deps.helper.windowAt(this.deps.windows, point.x, point.y, excluded);
+ if (this.closed) return null;
+ if (!window) return { window: null, rects: [] };
+ const hit = await this.deps.helper.elements(window, point.x, point.y);
+ if (this.closed) return null;
+ return {
+ window: { id: this.deps.windows.indexOf(window), rect: toLocal(toGlobalDip(window.rect), display) },
+ rects: hit.rects.map((rect) => toLocal(toGlobalDip(rect), display)),
+ ...(hit.preferredRect ? { preferredRect: toLocal(toGlobalDip(hit.preferredRect), display) } : {}),
+ };
}
copyText(
diff --git a/apps/desktop/src/main/region-screenshot/snapshotHost.ts b/apps/desktop/src/main/region-screenshot/snapshotHost.ts
index 0cc76155c1..7cba8ed2ae 100644
--- a/apps/desktop/src/main/region-screenshot/snapshotHost.ts
+++ b/apps/desktop/src/main/region-screenshot/snapshotHost.ts
@@ -32,6 +32,7 @@ export interface SnapshotWindow {
pid: number | null;
/** Windows 为物理像素, macOS 为全局 DIP; [left, top, right, bottom]。 */
rect: [number, number, number, number];
+ snapRects?: [number, number, number, number][];
}
const binaryName =
@@ -220,24 +221,52 @@ export class SnapshotHelper {
const windows: SnapshotWindow[] = [];
for (const raw of list) {
if (!raw || typeof raw !== 'object') continue;
- const { hwnd, pid, rect } = raw as Record<string, unknown>;
+ const { hwnd, pid, rect, snapRects } = raw as Record<string, unknown>;
if (typeof hwnd !== 'number' || !isRectTuple(rect)) continue;
- windows.push({ handle: hwnd, pid: typeof pid === 'number' ? pid : null, rect });
+ windows.push({ handle: hwnd, pid: typeof pid === 'number' ? pid : null, rect,
+ ...(Array.isArray(snapRects) ? { snapRects: snapRects.slice(0, 256).filter(isRectTuple)
+ .filter((r) => r[0] >= rect[0] && r[1] >= rect[1] && r[2] <= rect[2] && r[3] <= rect[3]) } : {}),
+ });
}
return windows;
}
+ /** Resolve only captured targets; an unknown native hit is an occluder, not a reason to look below it. */
+ async windowAt(
+ windows: readonly SnapshotWindow[], x: number, y: number, exclude: readonly number[],
+ ): Promise<SnapshotWindow | null> {
+ const contains = ({ rect }: SnapshotWindow) => x >= rect[0] && x < rect[2] && y >= rect[1] && y < rect[3];
+ if (process.platform === 'darwin') {
+ const response = await this.request({ op: 'window-at', x, y, exclude });
+ if (typeof response?.hwnd === 'number') {
+ return windows.find((window) => window.handle === response.hwnd && contains(window)) ?? null;
+ }
+ }
+ // Windows already excludes click-through windows during enumeration. Old
+ // native helpers and unavailable helpers retain rectangle-only snapping.
+ return windows.find(contains) ?? null;
+ }
+
/** 指针下的界面元素 (由内到外), 坐标系同 {@link SnapshotWindow.rect}。 */
- async elements(window: SnapshotWindow, x: number, y: number): Promise<SnapshotWindow['rect'][]> {
+ async elements(window: SnapshotWindow, x: number, y: number): Promise<{
+ rects: SnapshotWindow['rect'][]; preferredRect?: SnapshotWindow['rect'];
+ }> {
const response = await this.request({
op: 'elements',
hwnd: window.handle,
+ rect: window.rect,
...(window.pid !== null ? { pid: window.pid } : {}),
x: Math.round(x),
y: Math.round(y),
});
const rects = Array.isArray(response?.rects) ? response.rects : [];
- return rects.filter(isRectTuple);
+ const preferred = response?.preferredRect;
+ const valid = (rect: unknown): rect is SnapshotWindow['rect'] => isRectTuple(rect)
+ && rect[0] <= x && x < rect[2] && rect[1] <= y && y < rect[3];
+ return {
+ rects: rects.filter(valid),
+ ...(valid(preferred) ? { preferredRect: preferred } : {}),
+ };
}
/** macOS: 截图前的前台应用进程号, 结束后把焦点还给它。 */
diff --git a/apps/desktop/src/main/remote-agent/__tests__/guestHost.test.ts b/apps/desktop/src/main/remote-agent/__tests__/guestHost.test.ts
index 684a38894c..83cafb46bb 100644
--- a/apps/desktop/src/main/remote-agent/__tests__/guestHost.test.ts
+++ b/apps/desktop/src/main/remote-agent/__tests__/guestHost.test.ts
@@ -600,9 +600,12 @@ describe('guest provider access and usage', () => {
const started: Started[] = [];
const host = makeHost(started, { bindGuestProviderRoute: bind });
await host.handle(GUEST, { op: 'open', runId: RUN_1, agentKind: 'claude-code', payload: { json: openPayload('task-1') } });
- await vi.waitFor(() => expect(currents).toHaveLength(1));
+ await vi.waitFor(() => expect(currents).toHaveLength(1), { timeout: 10_000 });
await host.handle(GUEST, { op: 'open', runId: RUN_2, agentKind: 'claude-code', payload: { json: openPayload('task-1') } });
- await vi.waitFor(() => expect(started).toHaveLength(1));
+ // Windows CI run 40667219234 exceeded the default 1s disk-preparation wait.
+ // Observe RUN_2's completed handshake before checking that RUN_1 stays stale.
+ await waitForStarted(host, GUEST, RUN_2);
+ expect(started).toHaveLength(1);
expect(currents[0]!()).toBe(false);
openGate();
await new Promise((resolve) => setTimeout(resolve, 20));
diff --git a/apps/desktop/src/preload/regionScreenshotPreload.ts b/apps/desktop/src/preload/regionScreenshotPreload.ts
index fe2042e9e3..8b99bb1ed4 100644
--- a/apps/desktop/src/preload/regionScreenshotPreload.ts
+++ b/apps/desktop/src/preload/regionScreenshotPreload.ts
@@ -3,6 +3,7 @@ import { contextBridge, ipcRenderer } from 'electron';
import {
REGION_SCREENSHOT_CHANNEL as C,
type RegionRect,
+ type RegionScreenshotHit,
type RegionScreenshotBegin,
type RegionScreenshotExport,
type RegionScreenshotBitmap,
@@ -36,7 +37,7 @@ contextBridge.exposeInMainWorld('regionScreenshot', {
copyText: (token: string, text: string): void => ipcRenderer.send(C.COPY_TEXT, token, text),
cancel: (token: string): void => ipcRenderer.send(C.CANCEL, token),
abandon: (): void => ipcRenderer.send(C.ABANDON),
- elementsAt: (token: string, windowId: number, x: number, y: number): Promise<RegionRect[]> =>
+ elementsAt: (token: string, windowId: number, x: number, y: number): Promise<RegionScreenshotHit | null> =>
ipcRenderer.invoke(C.ELEMENTS_AT, token, windowId, x, y),
exportImage: (token: string, payload: RegionScreenshotExport): Promise<boolean> =>
ipcRenderer.invoke(C.EXPORT, token, payload),
diff --git a/apps/desktop/src/renderer/features/region-screenshot/RegionScreenshotOverlay.tsx b/apps/desktop/src/renderer/features/region-screenshot/RegionScreenshotOverlay.tsx
index 1ce07a4208..62d435353a 100644
--- a/apps/desktop/src/renderer/features/region-screenshot/RegionScreenshotOverlay.tsx
+++ b/apps/desktop/src/renderer/features/region-screenshot/RegionScreenshotOverlay.tsx
@@ -36,6 +36,8 @@ import {
type RegionScreenshotBegin,
type RegionScreenshotBitmap,
type RegionScreenshotWindow,
+ type RegionScreenshotHit,
+ intersectRegion,
type RegionSize,
} from '../../../shared/regionScreenshot';
import {
@@ -180,7 +182,7 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridg
const longResultRef = useRef(longResult);
longResultRef.current = longResult;
const [selection, setSelection] = useState<RegionRect | null>(null);
- const [hover, setHover] = useState<{ candidates: RegionRect[]; level: number } | null>(null);
+ const [hover, setHover] = useState<{ candidates: RegionRect[]; level: number; defaultRect: RegionRect } | null>(null);
const [pointer, setPointer] = useState<RegionPoint | null>(null);
const [drag, setDrag] = useState<Drag | null>(null);
const [tool, setTool] = useState<AnnotationTool | null>(null);
@@ -239,8 +241,9 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridg
const elementsRef = useRef<{
inFlight: boolean;
pending: RegionPoint | null;
- cache: { windowId: number; rects: RegionRect[] } | null;
- }>({ inFlight: false, pending: null, cache: null });
+ cache: { point: RegionPoint; hit: RegionScreenshotHit } | null;
+ defaults: Map<number, RegionRect[]>;
+ }>({ inFlight: false, pending: null, cache: null, defaults: new Map() });
const viewport = session?.viewport ?? { width: window.innerWidth, height: window.innerHeight };
const safe = session?.safeArea ?? { x: 0, y: 0, width: viewport.width, height: viewport.height };
@@ -272,7 +275,7 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridg
setOverToolbar(false);
busyRef.current = false;
wheelRef.current = 0;
- elementsRef.current = { inFlight: false, pending: null, cache: null };
+ elementsRef.current = { inFlight: false, pending: null, cache: null, defaults: new Map() };
}, []);
useEffect(() => {
@@ -470,13 +473,44 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridg
const refreshHover = useCallback((point: RegionPoint) => {
const current = state.current.session;
if (!current) return;
- const window = topWindowAt(current.windows, point);
const cache = elementsRef.current.cache;
- const elements = cache && window && cache.windowId === window.id ? cache.rects : [];
+ // A parent rectangle containing a new point does not prove that its old
+ // element chain (or the window under a transparent pixel) is still valid.
+ const hit = cache && cache.point.x === point.x && cache.point.y === point.y ? cache.hit : null;
+ const rectangularWindow = topWindowAt(current.windows, point);
+ // Keep a corrected target (including an explicit null for an unknown
+ // occluder) while this point is pending under the same captured front window.
+ // This is only a display fallback: requestElements still queries the new
+ // point, and its reply is authoritative. Leaving a known corrected target's
+ // bounds or entering a different front window ends retention immediately.
+ const pendingHit = !hit && cache && rectangularWindow
+ && cache.hit.window?.id !== rectangularWindow.id
+ && topWindowAt(current.windows, cache.point)?.id === rectangularWindow.id
+ && (!cache.hit.window || rectContains(cache.hit.window.rect, point)) ? cache.hit : null;
+ const window = hit ? hit.window : pendingHit ? pendingHit.window : rectangularWindow;
+ const snapshot = current.windows.find((candidate) => candidate.id === window?.id);
+ // Default regions belong to the frozen capture, not a point query. UIA may
+ // finish later or fail without making the outline bounce back to the window.
+ // Older helpers/macOS can retain a known content region while a new query is
+ // pending, but never reuse its fine element chain at a different point.
+ const remembered = window && elementsRef.current.defaults.get(window.id)?.find((rect) => rectContains(rect, point));
+ const native = snapshot?.snapRects?.find((rect) => rectContains(rect, point));
+ const preferred = native && window && !sameRegion(native, window.rect) ? native
+ : remembered ?? hit?.preferredRect
+ ?? (snapshot?.snapRects ? window?.rect : undefined);
+ const elements = [...(hit?.rects ?? []), ...(snapshot?.snapRects ?? []), ...(preferred ? [preferred] : [])]
+ .sort((a, b) => a.width * a.height - b.width * b.height);
const candidates = snapCandidates(point, window, elements, current.viewport);
+ const clipped = preferred && intersectRegion(preferred, current.viewport);
+ const defaultLevel = clipped ? Math.max(0, candidates.findIndex((rect) => sameRegion(rect, clipped))) : 0;
+ const defaultRect = candidates[defaultLevel];
setHover((previous) => {
- const keep = previous && previous.candidates[0] && sameRegion(previous.candidates[0], candidates[0]);
- return { candidates, level: keep ? Math.min(previous.level, candidates.length - 1) : 0 };
+ const selected = previous && sameRegion(previous.defaultRect, defaultRect)
+ ? candidates.findIndex((rect) => sameRegion(rect, previous.candidates[previous.level])) : -1;
+ const level = selected >= 0 ? selected : defaultLevel;
+ const keep = previous && previous.level === level && previous.candidates.length === candidates.length
+ && previous.candidates.every((rect, index) => sameRegion(rect, candidates[index]));
+ return keep && sameRegion(previous.defaultRect, defaultRect) ? previous : { candidates, level, defaultRect };
});
}, []);
@@ -494,18 +528,29 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridg
tracker.inFlight = true;
void bridge
.elementsAt(current.token, window.id, point.x, point.y)
- .then((rects) => {
- if (state.current.session?.token !== current.token) return;
- tracker.cache = { windowId: window.id, rects: Array.isArray(rects) ? rects : [] };
+ .then((hit) => {
+ if (state.current.session?.token !== current.token || elementsRef.current !== tracker || !hit) return;
const latest = state.current.pointer;
- if (latest && !state.current.selection && !state.current.drag) refreshHover(latest);
+ if (latest && latest.x === point.x && latest.y === point.y && !state.current.selection && !state.current.drag) {
+ // Some browsers have no content HWND. Learn their document regions
+ // once per capture; an empty/partial UIA reply must not erase them.
+ if (hit.window && hit.preferredRect && !sameRegion(hit.preferredRect, hit.window.rect)) {
+ const regions = tracker.defaults.get(hit.window.id) ?? [];
+ if (regions.length < 256 && !regions.some((rect) => sameRegion(rect, hit.preferredRect!))) {
+ tracker.defaults.set(hit.window.id, [...regions, hit.preferredRect]
+ .sort((a, b) => a.width * a.height - b.width * b.height));
+ }
+ }
+ tracker.cache = { point, hit };
+ refreshHover(latest);
+ }
})
.catch(() => undefined)
.finally(() => {
tracker.inFlight = false;
const pending = tracker.pending;
tracker.pending = null;
- if (pending && state.current.session?.token === current.token && !state.current.selection) {
+ if (pending && elementsRef.current === tracker && state.current.session?.token === current.token && !state.current.selection) {
requestElements(pending);
}
});
diff --git a/apps/desktop/src/renderer/features/region-screenshot/__tests__/screenshotRendering.test.tsx b/apps/desktop/src/renderer/features/region-screenshot/__tests__/screenshotRendering.test.tsx
index 3f422bcee5..7868bd5820 100644
--- a/apps/desktop/src/renderer/features/region-screenshot/__tests__/screenshotRendering.test.tsx
+++ b/apps/desktop/src/renderer/features/region-screenshot/__tests__/screenshotRendering.test.tsx
@@ -3,13 +3,20 @@ import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'
import { afterEach, describe, expect, it, vi } from 'vitest';
import { ScreenshotErrorBoundary } from '../ScreenshotErrorBoundary';
import { Magnifier } from '../Magnifier';
-import { bitmapToImageData, imageDataToBitmap } from '../bridge';
-import type { RegionScreenshotOverlayBridge } from '../bridge';
+import { bitmapToImageData, imageDataToBitmap, type RegionScreenshotOverlayBridge } from '../bridge';
+import { RegionScreenshotOverlay } from '../RegionScreenshotOverlay';
+import type { RegionScreenshotHit } from '../../../../shared/regionScreenshot';
import type { RegionCaptureFrame } from '../../../../shared/regionCapture';
import { ScrollCaptureOverlay } from '../ScrollCaptureOverlay';
vi.mock('@/lib/logger', () => ({ createLogger: () => ({ warn: vi.fn() }) }));
vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) }));
+vi.mock('@/i18n', () => ({ default: { language: 'en' } }));
+vi.mock('@/hooks/useTheme', () => ({ getInitialThemeVariant: () => ({ theme: 'default' }) }));
+vi.mock('@/themes/theme-service', () => ({ themeService: { applyTheme: vi.fn() } }));
+vi.mock('../ScreenshotToolbar', () => ({ TOOL_SHORTCUTS: {}, ScreenshotModeBar: () => null, ScreenshotStyleBar: () => null, ScreenshotToolbar: () => null }));
+vi.mock('../LiveCapturePanel', () => ({ LiveCapturePanel: () => null }));
+vi.mock('../ScreenshotEditorViewport', () => ({ ScreenshotEditorViewport: () => null }));
// 测试价值:生产 React 边界出错会释放浮层,放大镜坐标与屏幕像素一致。
// 纯几何测试没有组件/错误路径;最小输入为 render throw、2x 屏配 3x 定格图。
@@ -21,6 +28,177 @@ describe('screenshot rendering recovery', () => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
});
+ // Production overlay regression: moving within the same window must not
+ // reuse the old parent's chain or apply a delayed hit from another point.
+ // Review regression (2026-10-28): a captured click-through front window must
+ // not flash back while a corrected hit is pending. Previously this test
+ // expected that flash. Overlapping captured windows + deferred replies, tier: unit.
+ // Standalone file: 1.52s on XD-PC (2026-10-28).
+ it('keeps point-specific hits current and selects the corrected target', async () => {
+ vi.stubGlobal('PointerEvent', MouseEvent);
+ vi.stubGlobal('ImageData', class {
+ data: Uint8ClampedArray;
+ constructor(readonly width: number, readonly height: number) { this.data = new Uint8ClampedArray(width * height * 4); }
+ });
+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null);
+ let begin!: Parameters<RegionScreenshotOverlayBridge['onBegin']>[0];
+ const replies: ((hit: RegionScreenshotHit) => void)[] = [];
+ const bridge = {
+ onBegin: (callback: typeof begin) => { begin = callback; return () => {}; },
+ onEnd: () => () => {}, onClear: () => () => {},
+ shellReady() {}, abandon() {}, painted() {}, focused() {}, selectionStarted() {},
+ elementsAt: vi.fn(() => new Promise<RegionScreenshotHit>((resolve) => replies.push(resolve))),
+ } as unknown as RegionScreenshotOverlayBridge;
+ const { container } = render(<RegionScreenshotOverlay bridge={bridge} />);
+ const root = container.firstElementChild!;
+ const outer = { id: 0, rect: { x: 0, y: 0, width: 500, height: 400 } };
+ const corrected = { id: 1, rect: { x: 20, y: 20, width: 300, height: 300 } };
+ const front = { id: 2, rect: { x: 200, y: 100, width: 50, height: 40 } };
+ await act(async () => begin({ token: 'one', locale: 'en', viewport: { width: 600, height: 500 }, safeArea: { x: 0, y: 0, width: 600, height: 500 }, scaleFactor: 1, frame: { width: 1, height: 1, data: new Uint8Array(4) }, windows: [front, outer, corrected], cursor: { x: 10, y: 10 }, lastRegion: null }));
+ fireEvent.pointerMove(root, { clientX: 40, clientY: 40 });
+ await act(async () => replies[0]({ window: outer, rects: [{ x: 0, y: 0, width: 200, height: 200 }] }));
+ expect(screen.getByText('500 × 400')).toBeTruthy();
+ expect(bridge.elementsAt).toHaveBeenLastCalledWith('one', 0, 40, 40);
+ const content = { x: 25, y: 25, width: 250, height: 200 };
+ // 2026-10-28 browser regression: default to the document, not its button.
+ // The actual renderer must still let a wheel gesture reach finer elements.
+ await act(async () => replies[1]({ window: corrected, rects: [{ x: 30, y: 30, width: 80, height: 60 }, content], preferredRect: content }));
+ expect(screen.getByText('250 × 200')).toBeTruthy();
+ fireEvent.wheel(root, { deltaY: 120 });
+ expect(screen.getByText('80 × 60')).toBeTruthy();
+ // Keep the default, not the old fine element, during a new point query.
+ fireEvent.pointerMove(root, { clientX: 150, clientY: 150 });
+ expect(screen.getByText('250 × 200')).toBeTruthy();
+ await act(async () => replies[2]({ window: corrected, rects: [] }));
+ expect(screen.getByText('250 × 200')).toBeTruthy();
+ // An opaque pixel in the same front window must still win once confirmed.
+ fireEvent.pointerMove(root, { clientX: 160, clientY: 150 });
+ expect(screen.getByText('250 × 200')).toBeTruthy();
+ await act(async () => replies[3]({ window: outer, rects: [] }));
+ expect(screen.getByText('500 × 400')).toBeTruthy();
+ fireEvent.pointerMove(root, { clientX: 180, clientY: 150 });
+ await act(async () => replies[4]({ window: corrected, rects: [], preferredRect: content }));
+ expect(screen.getByText('250 × 200')).toBeTruthy();
+ // Leaving the corrected window stops retaining it even before a reply.
+ fireEvent.pointerMove(root, { clientX: 450, clientY: 350 });
+ expect(screen.getByText('500 × 400')).toBeTruthy();
+ fireEvent.pointerMove(root, { clientX: 220, clientY: 120 });
+ expect(screen.getByText('50 × 40')).toBeTruthy(); // a different front window wins immediately
+ fireEvent.pointerMove(root, { clientX: 200, clientY: 150 });
+ expect(screen.getByText('250 × 200')).toBeTruthy();
+ await act(async () => replies[5]({ window: outer, rects: [] }));
+ expect(screen.getByText('250 × 200')).toBeTruthy(); // stale opaque hit
+ await act(async () => replies[6]({ window: corrected, rects: [] }));
+ fireEvent.keyDown(window, { key: 'Enter' });
+ fireEvent.pointerMove(root, { clientX: 450, clientY: 350 });
+ expect(screen.getByText('250 × 200')).toBeTruthy();
+ expect(bridge.elementsAt).toHaveBeenCalledTimes(7);
+ });
+ // PR #7661 discussion_r4398847280: a native null window must not flash the
+ // captured window underneath an unknown occluder at each new pointer position.
+ // Existing corrected-target coverage only used non-null windows. Production
+ // overlay + deferred null/recovery replies is the minimal input; tier: unit.
+ it('retains a confirmed unknown occluder while a new point is pending', async () => {
+ vi.stubGlobal('PointerEvent', MouseEvent);
+ vi.stubGlobal('ImageData', class {
+ data = new Uint8ClampedArray(4);
+ constructor(readonly width: number, readonly height: number) {}
+ });
+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null);
+ let begin!: Parameters<RegionScreenshotOverlayBridge['onBegin']>[0];
+ const replies: ((hit: RegionScreenshotHit) => void)[] = [];
+ const bridge = {
+ onBegin: (callback: typeof begin) => { begin = callback; return () => {}; },
+ onEnd: () => () => {}, onClear: () => () => {},
+ shellReady() {}, abandon() {}, painted() {}, focused() {}, selectionStarted() {},
+ elementsAt: vi.fn(() => new Promise<RegionScreenshotHit>((resolve) => replies.push(resolve))),
+ } as unknown as RegionScreenshotOverlayBridge;
+ const outer = { id: 0, rect: { x: 0, y: 0, width: 500, height: 400 } };
+ const front = { id: 1, rect: { x: 400, y: 100, width: 100, height: 100 } };
+ const { container } = render(<RegionScreenshotOverlay bridge={bridge} />);
+ const root = container.firstElementChild!;
+ await act(async () => begin({ token: 'occluded', locale: 'en', viewport: { width: 600, height: 500 }, safeArea: { x: 0, y: 0, width: 600, height: 500 }, scaleFactor: 1, frame: { width: 1, height: 1, data: new Uint8Array(4) }, windows: [front, outer], cursor: { x: 50, y: 80 }, lastRegion: null }));
+ await act(async () => replies[0]({ window: null, rects: [] }));
+ expect(screen.getByText('600 × 500')).toBeTruthy();
+ for (const x of [100, 200]) {
+ fireEvent.pointerMove(root, { clientX: x, clientY: 80 });
+ expect(screen.getByText('600 × 500')).toBeTruthy();
+ }
+ // An obsolete non-null hit cannot select the occluded window either.
+ await act(async () => replies[1]({ window: outer, rects: [] }));
+ expect(screen.getByText('600 × 500')).toBeTruthy();
+ expect(bridge.elementsAt).toHaveBeenLastCalledWith('occluded', 0, 200, 80);
+ await act(async () => replies[2]({ window: null, rects: [] }));
+ fireEvent.pointerMove(root, { clientX: 250, clientY: 80 });
+ expect(screen.getByText('600 × 500')).toBeTruthy();
+ // A fresh hit restores the known window as soon as the occluder is left.
+ await act(async () => replies[3]({ window: outer, rects: [] }));
+ expect(screen.getByText('500 × 400')).toBeTruthy();
+ fireEvent.pointerMove(root, { clientX: 300, clientY: 80 });
+ await act(async () => replies[4]({ window: null, rects: [] }));
+ fireEvent.pointerMove(root, { clientX: 450, clientY: 150 });
+ expect(screen.getByText('100 × 100')).toBeTruthy(); // a different captured front ends retention
+ await act(async () => replies[5]({ window: front, rects: [] }));
+ expect(bridge.elementsAt).toHaveBeenCalledTimes(6);
+ });
+ // 2026-10-28 flicker: final-result assertions missed the whole-window frame
+ // inserted during every pending query. Production overlay, two moves before a
+ // deferred reply (unit); also cover old helpers without captured child regions.
+ // Standalone file: 1.64s on XD-PC (2026-10-28).
+ it.each([true, false])('keeps the default outline stable during pending queries (snapshot=%s)', async (snapshot) => {
+ vi.stubGlobal('PointerEvent', MouseEvent);
+ vi.stubGlobal('ImageData', class {
+ data = new Uint8ClampedArray(4);
+ constructor(readonly width: number, readonly height: number) {}
+ });
+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null);
+ let begin!: Parameters<RegionScreenshotOverlayBridge['onBegin']>[0];
+ const replies: ((hit: RegionScreenshotHit) => void)[] = [];
+ const bridge = {
+ onBegin: (callback: typeof begin) => { begin = callback; return () => {}; },
+ onEnd: () => () => {}, onClear: () => () => {},
+ shellReady() {}, abandon() {}, painted() {}, focused() {}, selectionStarted() {},
+ elementsAt: vi.fn(() => new Promise<RegionScreenshotHit>((resolve) => replies.push(resolve))),
+ } as unknown as RegionScreenshotOverlayBridge;
+ const content = { x: 0, y: 50, width: 500, height: 350 };
+ const outer = { id: 0, rect: { x: 0, y: 0, width: 500, height: 400 }, ...(snapshot ? { snapRects: [content] } : {}) };
+ const front = { id: 1, rect: { x: 400, y: 100, width: 100, height: 100 }, snapRects: [] };
+ const { container } = render(<RegionScreenshotOverlay bridge={bridge} />);
+ const root = container.firstElementChild!;
+ const payload = { token: 'stable', locale: 'en', viewport: { width: 600, height: 500 }, safeArea: { x: 0, y: 0, width: 600, height: 500 }, scaleFactor: 1, frame: { width: 1, height: 1, data: new Uint8Array(4) }, windows: [front, outer], cursor: { x: 50, y: 80 }, lastRegion: null };
+ await act(async () => begin(payload));
+ if (snapshot) expect(screen.getByText('500 × 350')).toBeTruthy(); // before any native reply
+ await act(async () => replies[0]({ window: outer, rects: [], preferredRect: content }));
+ for (const x of [100, 200]) {
+ fireEvent.pointerMove(root, { clientX: x, clientY: 80 });
+ expect(screen.getByText('500 × 350')).toBeTruthy();
+ }
+ // A late result for x=100 must not change even the remembered default.
+ await act(async () => replies[1]({ window: outer, rects: [], preferredRect: outer.rect }));
+ fireEvent.pointerMove(root, { clientX: 250, clientY: 80 });
+ expect(screen.getByText('500 × 350')).toBeTruthy();
+ await act(async () => replies[2]({ window: outer, rects: [], preferredRect: outer.rect }));
+ await act(async () => replies[3]({ window: outer, rects: [{ x: 230, y: 60, width: 80, height: 60 }], preferredRect: content }));
+ expect(screen.getByText('500 × 350')).toBeTruthy();
+ fireEvent.wheel(root, { deltaY: 120 });
+ expect(screen.getByText('80 × 60')).toBeTruthy();
+ fireEvent.pointerMove(root, { clientX: 250, clientY: 81 });
+ await act(async () => replies[4]({ window: outer, rects: [], preferredRect: outer.rect }));
+ expect(screen.getByText('500 × 350')).toBeTruthy(); // partial UIA does not erase known content
+ fireEvent.pointerMove(root, { clientX: 250, clientY: 20 });
+ expect(screen.getByText('500 × 400')).toBeTruthy(); // leave content immediately
+ fireEvent.pointerMove(root, { clientX: 450, clientY: 150 });
+ expect(screen.getByText('100 × 100')).toBeTruthy(); // top window occludes cached content
+ fireEvent.pointerMove(root, { clientX: 250, clientY: 80 });
+ expect(screen.getByText('500 × 350')).toBeTruthy();
+ fireEvent.keyDown(window, { key: 'Enter' });
+ fireEvent.pointerMove(root, { clientX: 550, clientY: 450 });
+ expect(screen.getByText('500 × 350')).toBeTruthy();
+ // A new capture cannot inherit a region or a late reply from this capture.
+ await act(async () => begin({ ...payload, token: 'next', windows: [{ ...outer, snapRects: [] }] }));
+ await act(async () => replies[5]({ window: outer, rects: [], preferredRect: content }));
+ expect(screen.getByText('500 × 400')).toBeTruthy();
+ });
// 用户反馈点击自动滚动立刻提示已停止。已有渲染测试未覆盖开关与在途帧的竞态。
// 最小输入:开启前发出的帧在开启成功后返回 false;下一帧的真实停止仍须显示。tier: unit。
it('ignores a pre-start frame but reports a fresh automatic scrolling failure', async () => {
diff --git a/apps/desktop/src/renderer/features/region-screenshot/bridge.ts b/apps/desktop/src/renderer/features/region-screenshot/bridge.ts
index 38b30b1d6d..a06dc525a3 100644
--- a/apps/desktop/src/renderer/features/region-screenshot/bridge.ts
+++ b/apps/desktop/src/renderer/features/region-screenshot/bridge.ts
@@ -1,5 +1,6 @@
import type {
RegionRect,
+ RegionScreenshotHit,
RegionScreenshotBegin,
RegionScreenshotBitmap,
RegionScreenshotExport,
@@ -30,7 +31,7 @@ export interface RegionScreenshotOverlayBridge {
copyText(token: string, text: string): void;
cancel(token: string): void;
abandon(): void;
- elementsAt(token: string, windowId: number, x: number, y: number): Promise<RegionRect[]>;
+ elementsAt(token: string, windowId: number, x: number, y: number): Promise<RegionScreenshotHit | null>;
exportImage(token: string, payload: RegionScreenshotExport): Promise<boolean>;
}
diff --git a/apps/desktop/src/renderer/region-screenshot-entry.tsx b/apps/desktop/src/renderer/region-screenshot-entry.tsx
index 8de78965ae..0e056587fb 100644
--- a/apps/desktop/src/renderer/region-screenshot-entry.tsx
+++ b/apps/desktop/src/renderer/region-screenshot-entry.tsx
@@ -38,7 +38,7 @@ const bridge: RegionScreenshotOverlayBridge | undefined = editor ? {
abandon: () => { if (editorToken) editor.cancel(editorToken); },
// Rendering adapters only: none of these unused capture capabilities cross preload.
onClear: () => () => {}, focused: () => {}, selectionStarted: () => {}, copyText: () => {},
- elementsAt: async () => [], autoScroll: async () => false, restoreEditor: async () => false,
+ elementsAt: async () => null, autoScroll: async () => false, restoreEditor: async () => false,
holdCapture: async () => {}, startLive: async () => null, liveFrame: async () => null, saveRecording: async () => false,
recordingControls: async () => null, previewRecording: async () => false,
} : window.regionScreenshot;
diff --git a/apps/desktop/src/shared/regionScreenshot.ts b/apps/desktop/src/shared/regionScreenshot.ts
index 32e04f82a8..90a301ae8f 100644
--- a/apps/desktop/src/shared/regionScreenshot.ts
+++ b/apps/desktop/src/shared/regionScreenshot.ts
@@ -23,7 +23,7 @@ export const REGION_SCREENSHOT_CHANNEL = {
CLEAR: 'region-screenshot:clear',
/** main → 浮层: 会话结束, 清空画面与全部交互态后隐藏。 */
END: 'region-screenshot:end',
- /** invoke(token, windowId, x, y): 指针下的界面元素, 由内到外 (DIP)。 */
+ /** invoke(token, windowId, x, y): 实际命中窗口与由内到外的元素 (DIP)。 */
ELEMENTS_AT: 'region-screenshot:elements-at',
/** send(token, text): 复制取到的颜色值。 */
COPY_TEXT: 'region-screenshot:copy-text',
@@ -138,6 +138,16 @@ export interface RegionScreenshotBitmap {
export interface RegionScreenshotWindow {
id: number;
rect: RegionRect;
+ /** Captured native regions in hit order; absent on older helpers/platforms. */
+ snapRects?: RegionRect[];
+}
+
+/** A point query can correct the rectangular window guess through transparent overlays. */
+export interface RegionScreenshotHit {
+ window: RegionScreenshotWindow | null;
+ rects: RegionRect[];
+ /** Default document/content region; finer elements remain available via wheel. */
+ preferredRect?: RegionRect;
}
export interface RegionScreenshotBegin {
diff --git a/docs/dev-rules/electron-security-and-process-boundaries.md b/docs/dev-rules/electron-security-and-process-boundaries.md
index d116e6ec21..36cdfc428b 100644
--- a/docs/dev-rules/electron-security-and-process-boundaries.md
+++ b/docs/dev-rules/electron-security-and-process-boundaries.md
@@ -138,6 +138,34 @@ macOS 浮层是不激活应用的面板(`type: 'panel'`):拿键盘焦点
应用保持前台,菜单栏不切换,滚动截图时触控板滚动与惯性照常交给它;不得改回
`app.focus({ steal: true })`。
+截图自动选区只查询几何信息。macOS 按原生命中规则跳过透明位置与忽略鼠标的窗口,
+只显式排除本次截图浮层;命中未收入截图快照的窗口时不继续向下穿透。辅助功能查询
+优先命中目标应用并沿父级取区域,失败时在有界预算内回退到目标窗口树。
+本机 `ELEMENTS_AT` 返回纠正后的窗口与元素;主进程保留 sender/顶层 frame/token 和
+显示器内坐标校验,只接受快照中的目标。精细元素缓存绑定查询位置,迟到结果不覆盖新位置或默认区域缓存。
+此接口与原生 helper 同包更新,不进入 device-link allowlist;旧 helper 缺少命中查询时
+沿用窗口矩形吸附。回归见 `snapshotHost.test.ts`、`windows.test.ts` 与 `screenshotRendering.test.tsx`。
+首次悬停优先网页/文档内容区,浏览器工具栏与标签栏回到整窗,内部元素保留供滚轮切换。
+Windows 在截图准备阶段保存原生子窗口区域(每窗最多 256 个、最多 40 层,按原生层级/z 序,
+裁到祖先边界),随窗口列表可选 `snapRects` 下发;main 校验并转换为显示器局部 DIP。
+移动时从快照同步命中默认区域,异步元素查询只补滚轮候选,不在等待时退回整窗;
+相同默认区域保留仍有效的滚轮层级,相同候选和层级不重复更新。旧 helper/macOS 没有此字段,
+或 Chromium 没有原生内容子窗口时,将有效的内容区按窗口补入本次截图的默认区域缓存(每窗最多 256 个)。
+等待查询和空/不完整回复不清除仍包含指针的已知内容区;退出/重新截图即清空。
+不复用旧位置的精细元素链。若原生命中曾纠正透明/鼠标穿透前窗,新位置仍在纠正窗口内、
+且矩形命中的前窗未变,等待查询期间沿用已确认目标的默认区域显示;此显示保留不替代新位置的
+原生命中,当前回复确认不透明前窗后立即切换。离开纠正窗口或进入另一前窗时立即按新边界显示;
+未知遮挡物的显式空命中也须保留,不能在同一快照前窗内移动时因新坐标尚未回复而选中底层窗口;
+新位置的有效回复恢复已知窗口,进入另一快照前窗则结束保留。
+双窗口/未知遮挡中间帧、迟到回复和不透明区域切换回归见 `screenshotRendering.test.tsx`。
+Windows 同时在有界预算内遍历重叠的 UIA 分支,不能因首个同尺寸
+空容器而漏掉正文;macOS 沿 AXWebArea/滚动容器获取内容区。helper 可选返回 `preferredRect`,
+主进程验证并转换到显示器局部 DIP;旧 helper 省略时保持原有元素吸附。
+Windows 真实 Chrome+原生 helper+生产浮层的黑盒回归入口为
+`node apps/desktop/scripts/check-region-snapping.mjs`,副屏可加 `--position=2700,100`;
+记录 Light/Dark 连续移动时的 DOM 提交与绘制帧,并延后真实元素响应验证过程中无整窗闪回及跨边界切换;
+只适配测试传输,不替代生产 IPC 授权用例或 macOS 实机验证。
+
滚动截图与区域录制复用该浮层和会话 token,开始后把控制区移到选区之外。
滚动截图在原透明浮层显示选区轮廓与侧边预览,只在预览和自动滚动按钮处接收鼠标。
长图编辑交给独立无标题栏窗口,复用辅助窗口 controller,原浮层结束;首份图片绘制后才显示。