Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(desktop): 修复截图内容区吸附与区域内移动闪烁 (#7661)
DavidShenXDdb628cc700af
diff --git a/apps/desktop/native/region-screenshot/macos-snapshot.swift b/apps/desktop/native/region-screenshot/macos-snapshot.swiftindex 5e890c5730..a12394c628 100644--- a/apps/desktop/native/region-screenshot/macos-snapshot.swift+++ b/apps/desktop/native/region-screenshot/macos-snapshot.swift@@ -7,10 +7,11 @@//// - {"id":1,"op":"windows"} on-screen windows, front to back (with owner pid).// - {"id":2,"op":"elements","pid":P,"x":X,"y":Y} accessibility elements of-// that app under the point, innermost first. The tree is walked from the-// app's windows instead of hit-testing the screen, because Cindy's overlay-// covers the screen. Needs the Accessibility permission; without it the-// answer is empty and the overlay snaps to windows only.+// that app under the point, innermost first. Hit-test the application and+// walk its parents, with a bounded window-tree fallback. Needs Accessibility;+// without it the answer is empty and the overlay snaps to windows only.+// - {"id":6,"op":"window-at","x":X,"y":Y,"exclude":[N]} mouse hit window,+// excluding only this screenshot's overlay window numbers.// - {"id":3,"op":"frontmost"} / {"id":4,"op":"activate","pid":P} remember and// give focus back to the app the user was in before the screenshot.// - {"id":5,"op":"screens"} per display the top inset taken by the camera@@ -132,6 +133,26 @@ func children(of element: AXUIElement) -> [AXUIElement] {(attribute(element, kAXChildrenAttribute) as? [AXUIElement]) ?? []}+/// AppKit skips transparent pixels and windows that ignore mouse events.+/// Only Cindy's screenshot overlays are excluded explicitly; an unknown front+/// window must still block the captured windows behind it.+func windowAt(point: CGPoint, excluding: Set<Int>) -> Int {+ let cocoaPoint = CGPoint(x: point.x, y: (NSScreen.screens.first?.frame.maxY ?? 0) - point.y)+ var below = 0+ for _ in 0..<(excluding.count + 1) {+ let hit = NSWindow.windowNumber(at: cocoaPoint, belowWindowWithWindowNumber: below)+ if hit == 0 || hit == below { return 0 }+ if !excluding.contains(hit) { return hit }+ below = hit+ }+ return 0+}++func matches(_ a: CGRect, _ b: CGRect) -> Bool {+ abs(a.minX - b.minX) < 2 && abs(a.minY - b.minY) < 2+ && abs(a.width - b.width) < 2 && abs(a.height - b.height) < 2+}+/// Smallest element among `candidates` whose frame contains the point.func smallest(_ candidates: [AXUIElement], containing point: CGPoint) -> (AXUIElement, CGRect)? {var best: (AXUIElement, CGRect)?@@ -145,22 +166,85 @@ func smallest(_ candidates: [AXUIElement], containing point: CGPoint) -> (AXUIElreturn best}-func elementChain(pid: pid_t, point: CGPoint) -> [[Double]] {- guard AXIsProcessTrusted() else { return [] }+func elementChain(pid: pid_t, point: CGPoint, bounds: CGRect?) -> [String: Any] {+ guard AXIsProcessTrusted() else { return ["rects": []] }let started = Date()elementDeadline = started.addingTimeInterval(elementBudget)defer { elementDeadline = nil }let app = AXUIElementCreateApplication(pid)+ AXUIElementSetMessagingTimeout(app, 0.05)+ var hit: AXUIElement?+ if AXUIElementCopyElementAtPosition(app, Float(point.x), Float(point.y), &hit) == .success,+ let hit = hit+ {+ var current = hit+ var visited: [AXUIElement] = []+ var chain: [CGRect] = []+ var document: CGRect?+ var scroll: CGRect?+ for _ in 0..<maxDepth {+ if elementDeadline!.timeIntervalSinceNow <= 0 || visited.contains(where: { CFEqual($0, current) }) { break }+ visited.append(current)+ let rect = frame(of: current)+ if let rect = rect, rect.contains(point), chain.last != rect {+ chain.append(rect)+ }+ let role = attribute(current, kAXRoleAttribute) as? String+ if let rect = rect, rect.contains(point) {+ if role == "AXWebArea" { document = rect }+ if role == kAXScrollAreaRole { scroll = rect }+ }+ if role == kAXWindowRole {+ // The app may own both the target and our screenshot overlay. Never use+ // a hit from another window of that app (or an overlapping sibling).+ if let rect = rect, bounds.map({ matches(rect, $0) }) ?? true {+ var result: [String: Any] = ["rects": chain.map(rectArray)]+ result["preferredRect"] = rectArray(document ?? scroll ?? rect)+ return result+ }+ break+ }+ guard let parent = attribute(current, kAXParentAttribute),+ CFGetTypeID(parent) == AXUIElementGetTypeID()+ else { break }+ current = parent as! AXUIElement+ }+ }let windows = (attribute(app, kAXWindowsAttribute) as? [AXUIElement]) ?? []- guard var current = smallest(windows, containing: point) else { return [] }- var chain: [CGRect] = [current.1]- for _ in 0..<maxDepth {- if Date().timeIntervalSince(started) > elementBudget { break }- guard let next = smallest(children(of: current.0), containing: point) else { break }- if next.1 != chain.last { chain.append(next.1) }- current = next+ let matching = windows.filter { window in+ guard let rect = frame(of: window) else { return false }+ return bounds.map { matches(rect, $0) } ?? rect.contains(point)+ }+ guard let root = smallest(matching, containing: point) else { return ["rects": []] }+ var pending: [(AXUIElement, [CGRect], CGRect?, CGRect?, Int)] = [(root.0, [root.1], nil, nil, 0)]+ var best: [CGRect] = []+ var preferred: CGRect?+ var bestIsDocument = false+ var visited = 0+ while let (current, chain, document, scroll, depth) = pending.popLast() {+ if Date().timeIntervalSince(started) > elementBudget || visited >= 256 { break }+ visited += 1+ if (document != nil && !bestIsDocument) || ((document != nil) == bestIsDocument && chain.count > best.count) {+ best = chain; preferred = document ?? scroll; bestIsDocument = document != nil+ }+ if depth >= maxDepth { continue }+ var candidates: [(AXUIElement, CGRect)] = []+ for child in children(of: current).prefix(512) {+ if Date().timeIntervalSince(started) > elementBudget { break }+ if let rect = frame(of: child), rect.contains(point) { candidates.append((child, rect)) }+ }+ candidates.sort { $0.1.width * $0.1.height < $1.1.width * $1.1.height }+ for (child, rect) in candidates.reversed() {+ let role = attribute(child, kAXRoleAttribute) as? String+ var next = chain+ if next.last != rect { next.append(rect) }+ pending.append((child, next, document ?? (role == "AXWebArea" ? rect : nil),+ scroll ?? (role == kAXScrollAreaRole ? rect : nil), depth + 1))+ }}- return chain.reversed().map(rectArray)+ var result: [String: Any] = ["rects": best.reversed().map(rectArray)]+ result["preferredRect"] = rectArray(preferred ?? root.1)+ return result}func screenInsets() -> [[String: Any]] {@@ -233,14 +317,30 @@ while let line = readLine() {emit(["id": id, "keys": Array(labels.filter { CGEventSource.keyState(.combinedSessionState, key: $0.0) }.prefix(8).map { $0.1 })])case "windows":emit(["id": id, "windows": onScreenWindows()])+ case "window-at":+ if let x = (request["x"] as? NSNumber)?.doubleValue,+ let y = (request["y"] as? NSNumber)?.doubleValue, x.isFinite, y.isFinite+ {+ let excluded = Set(request["exclude"] as? [Int] ?? [])+ emit(["id": id, "hwnd": windowAt(point: CGPoint(x: x, y: y), excluding: excluded)])+ } else {+ emit(["id": id, "error": "invalid-request"])+ }case "screens":emit(["id": id, "screens": screenInsets()])case "elements":if let pid = (request["pid"] as? NSNumber)?.int32Value,let x = (request["x"] as? NSNumber)?.doubleValue,- let y = (request["y"] as? NSNumber)?.doubleValue+ let y = (request["y"] as? NSNumber)?.doubleValue, x.isFinite, y.isFinite{- emit(["id": id, "rects": elementChain(pid: pid, point: CGPoint(x: x, y: y))])+ let rect = request["rect"] as? [Double]+ let bounds = rect.flatMap { r -> CGRect? in+ guard r.count == 4, r.allSatisfy({ $0.isFinite }), r[2] > r[0], r[3] > r[1] else { return nil }+ return CGRect(x: r[0], y: r[1], width: r[2] - r[0], height: r[3] - r[1])+ }+ var result = elementChain(pid: pid, point: CGPoint(x: x, y: y), bounds: bounds)+ result["id"] = id+ emit(result)} else {emit(["id": id, "error": "invalid-request"])}diff --git a/apps/desktop/native/region-screenshot/windows-snapshot/src/main.rs b/apps/desktop/native/region-screenshot/windows-snapshot/src/main.rsindex 51c6dc53e7..7944674d9e 100644--- a/apps/desktop/native/region-screenshot/windows-snapshot/src/main.rs+++ b/apps/desktop/native/region-screenshot/windows-snapshot/src/main.rs@@ -6,7 +6,8 @@//! Electron converts them to DIP.//!//! - `{"id":1,"op":"windows"}` → visible top-level windows, top of the z-order-//! first, using the DWM frame (without the invisible resize border).+//! first, using the DWM frame (without the invisible resize border), plus+//! bounded native child rectangles for synchronous default-region hit testing.//! - `{"id":2,"op":"elements","hwnd":N,"x":X,"y":Y}` → UI Automation elements of//! that window under the point, innermost first. The window is located from//! the snapshot instead of hit-testing the screen, because Cindy's overlay@@ -33,6 +34,8 @@ use windows::{WindowsAndMessaging::{EnumWindows, GetWindowLongW, GetWindowRect, IsIconic, IsWindowVisible, GWL_EXSTYLE,WS_EX_TRANSPARENT, WindowFromPoint, GetAncestor, GA_ROOT, PostMessageW, WM_MOUSEWHEEL,+ ChildWindowFromPointEx, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT,+ GetWindow, GW_CHILD, GW_HWNDNEXT,},},},@@ -68,12 +71,14 @@ fn main() {let hwnd = request.get("hwnd").and_then(Value::as_i64);let x = request.get("x").and_then(Value::as_i64);let y = request.get("y").and_then(Value::as_i64);- match (&automation, hwnd, x, y) {- (Some(automation), Some(hwnd), Some(x), Some(y)) => json!({- "id": id,- "rects": element_chain(automation, hwnd as isize, x as i32, y as i32),- }),- (None, ..) => json!({ "id": id, "rects": [] }),+ match (hwnd, x.and_then(|v| i32::try_from(v).ok()), y.and_then(|v| i32::try_from(v).ok())) {+ (Some(hwnd), Some(x), Some(y)) => {+ let native = content_rect(HWND(hwnd as isize as *mut _), x, y);+ let (rects, document) = automation.as_ref()+ .map(|a| element_chain(a, hwnd as isize, x, y)).unwrap_or_default();+ json!({ "id": id, "rects": rects,+ "preferredRect": native.or(document).or_else(|| window_rect(HWND(hwnd as isize as *mut _))).as_ref().map(rect_json) })+ },_ => json!({ "id": id, "error": "invalid-request" }),}}@@ -230,7 +235,10 @@ unsafe extern "system" fn collect_window(hwnd: HWND, data: LPARAM) -> BOOL {return true.into();}if let Some(rect) = window_rect(hwnd) {- windows.push(json!({ "hwnd": hwnd.0 as isize, "rect": rect_json(&rect) }));+ let mut regions = Vec::new();+ let mut remaining = 256;+ snapshot_children(hwnd, &rect, 0, &mut remaining, &mut regions);+ windows.push(json!({ "hwnd": hwnd.0 as isize, "rect": rect_json(&rect), "snapRects": regions }));}true.into()}@@ -251,47 +259,97 @@ fn area(rect: &RECT) -> i64 {(rect.right - rect.left) as i64 * (rect.bottom - rect.top) as i64}-/// Walks the control view from the window down, at each level taking the-/// smallest child that contains the point. Returns innermost first.-fn element_chain(automation: &IUIAutomation, hwnd: isize, x: i32, y: i32) -> Vec<Value> {+/// Native content hosts remain available even when Chromium has not enabled UIA.+/// Start inside the captured window: screen hit testing would hit our overlay.+fn content_rect(root: HWND, x: i32, y: i32) -> Option<RECT> {+ let outer = window_rect(root)?;+ let mut current = root;+ let mut result = None;+ for _ in 0..MAX_DEPTH {+ let mut point = POINT { x, y };+ if !unsafe { ScreenToClient(current, &mut point) }.as_bool() { break; }+ let child = unsafe { ChildWindowFromPointEx(current, point, CWP_SKIPINVISIBLE | CWP_SKIPTRANSPARENT) };+ if child.0.is_null() || child == current { break; }+ let mut rect = RECT::default();+ if unsafe { GetWindowRect(child, &mut rect) }.is_ok() && usable(&rect)+ && contains(&rect, x, y) && area(&rect) < area(&outer) {+ result = Some(rect);+ }+ current = child;+ }+ result+}++/// Capture once, before the overlay appears. Descendants precede parents and+/// sibling subtrees retain native z-order, so the first containing region wins.+/// Clipping to ancestors prevents a child from snapping outside its host.+unsafe fn snapshot_children(parent: HWND, bounds: &RECT, depth: usize, remaining: &mut usize, regions: &mut Vec<Value>) {+ if depth >= MAX_DEPTH || *remaining == 0 { return; }+ let mut child = GetWindow(parent, GW_CHILD).unwrap_or_default();+ while !child.0.is_null() && *remaining > 0 {+ *remaining -= 1;+ if IsWindowVisible(child).as_bool()+ && GetWindowLongW(child, GWL_EXSTYLE) as u32 & WS_EX_TRANSPARENT.0 == 0 {+ let mut rect = RECT::default();+ if GetWindowRect(child, &mut rect).is_ok() {+ rect.left = rect.left.max(bounds.left);+ rect.top = rect.top.max(bounds.top);+ rect.right = rect.right.min(bounds.right);+ rect.bottom = rect.bottom.min(bounds.bottom);+ if usable(&rect) {+ snapshot_children(child, &rect, depth + 1, remaining, regions);+ regions.push(rect_json(&rect));+ }+ }+ }+ child = GetWindow(child, GW_HWNDNEXT).unwrap_or_default();+ }+}++/// Explore overlapping siblings within one budget. Chrome can expose an empty+/// container before the equally sized branch that actually owns the document.+fn element_chain(automation: &IUIAutomation, hwnd: isize, x: i32, y: i32) -> (Vec<Value>, Option<RECT>) {let started = Instant::now();- let mut chain: Vec<RECT> = Vec::new();+ let mut best: Vec<RECT> = Vec::new();+ let mut preferred = None;let mut walk = || -> windows::core::Result<()> {unsafe {let cache = automation.CreateCacheRequest()?;cache.AddProperty(UIA_BoundingRectanglePropertyId)?;+ cache.AddProperty(UIA_ControlTypePropertyId)?;let condition = automation.ControlViewCondition()?;cache.SetTreeFilter(&condition)?;- let mut current = automation.ElementFromHandle(HWND(hwnd as *mut _))?;- for _ in 0..MAX_DEPTH {- if started.elapsed() > ELEMENT_BUDGET {- break;+ let root = automation.ElementFromHandle(HWND(hwnd as *mut _))?;+ let mut pending = vec![(root, Vec::<RECT>::new(), None::<RECT>, 0usize)];+ let mut visited = 0;+ while let Some((current, chain, document, depth)) = pending.pop() {+ if started.elapsed() > ELEMENT_BUDGET || visited >= 256 { break; }+ visited += 1;+ if (document.is_some() && preferred.is_none())+ || (document.is_some() == preferred.is_some() && chain.len() > best.len()) {+ best = chain.clone(); preferred = document;}- let children = current.FindAllBuildCache(TreeScope_Children, &condition, &cache)?;- let mut best: Option<(i64, IUIAutomationElement, RECT)> = None;- for index in 0..children.Length()? {- if started.elapsed() > ELEMENT_BUDGET {- break;- }+ if depth >= MAX_DEPTH { continue; }+ let Ok(children) = current.FindAllBuildCache(TreeScope_Children, &condition, &cache) else { continue };+ let mut candidates = Vec::new();+ for index in 0..children.Length()?.min(512) {+ if started.elapsed() > ELEMENT_BUDGET { break; }let Ok(child) = children.GetElement(index) else { continue };let Ok(rect) = child.CachedBoundingRectangle() else { continue };- if !usable(&rect) || !contains(&rect, x, y) {- continue;- }- let size = area(&rect);- if best.as_ref().map_or(true, |(best_size, ..)| size < *best_size) {- best = Some((size, child, rect));- }+ if !usable(&rect) || !contains(&rect, x, y) { continue; }+ let document = document.or_else(|| (child.CachedControlType().ok() == Some(UIA_DocumentControlTypeId)).then_some(rect));+ candidates.push((child, rect, document));}- let Some((_, child, rect)) = best else { break };- if chain.last() != Some(&rect) {- chain.push(rect);+ candidates.sort_by_key(|(_, rect, _)| area(rect));+ for (child, rect, document) in candidates.into_iter().rev() {+ let mut next = chain.clone();+ if next.last() != Some(&rect) { next.push(rect); }+ pending.push((child, next, document, depth + 1));}- current = child;}Ok(())}};let _ = walk();- chain.iter().rev().map(rect_json).collect()+ (best.iter().rev().map(rect_json).collect(), preferred)}diff --git a/apps/desktop/scripts/check-region-snapping.mjs b/apps/desktop/scripts/check-region-snapping.mjsnew file mode 100644index 0000000000..1f2197c33e--- /dev/null+++ b/apps/desktop/scripts/check-region-snapping.mjs@@ -0,0 +1,259 @@+#!/usr/bin/env node+// E2E, explicitly invoked: node apps/desktop/scripts/check-region-snapping.mjs+// Bug: Chrome's first of two equal UIA containers was empty, so hovering its+// document selected the entire browser. Unit/fixture checks fake native replies.+// This checks a headed Chrome, the shipping native helper and production overlay+// using real replies. IPC sender validation remains in windows.test.ts.+// Optional --position=2700,100 exercises a secondary display; --helper=<binary>+// permits running the exact same regression against a pre-fix helper.+import assert from 'node:assert/strict';+import { spawn, execFileSync } from 'node:child_process';+import { createInterface } from 'node:readline';+import { mkdtemp, rm, writeFile } from 'node:fs/promises';+import { existsSync } from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import { setTimeout as delay } from 'node:timers/promises';+import { chromium } from 'playwright-core';+import { buildPage, desktop } from './ui-page/build.mjs';+import { adhocPage } from './ui-page/registry.mjs';+import { pageResponse, origin } from './ui-page/run.mjs';++assert.equal(process.platform, 'win32', 'This native regression requires an interactive Windows desktop');+const option = (name) => process.argv.find(arg => arg.startsWith(`--${name}=`))?.slice(name.length + 3);+const position = (option('position') ?? '80,80').split(',').map(Number);+assert.ok(position.length === 2 && position.every(Number.isFinite));+const temp = await mkdtemp(path.join(os.tmpdir(), 'cindy-region-blackbox-'));+const browserPath = option('browser') ?? path.join(process.env.PROGRAMFILES, 'Google/Chrome/Application/chrome.exe');+assert.ok(existsSync(browserPath), 'Chrome is required (or pass --browser=<path>)');+let browser, renderer, helper, nativeWindow;+const pending = new Map();+let sequence = 0;+const ask = (payload) => new Promise((resolve, reject) => {+ const id = ++sequence;+ const timeout = setTimeout(() => { pending.delete(id); reject(new Error(`Native timeout: ${payload.op}`)); }, 5000);+ pending.set(id, { resolve: (reply) => { clearTimeout(timeout); resolve(reply); }, reject, timeout });+ helper.stdin.write(`${JSON.stringify({ id, ...payload })}\n`);+});+try {+ let binary = option('helper');+ if (!binary) {+ execFileSync('cargo', ['build', '--release', '--locked', '--manifest-path', path.join(desktop, 'native/region-screenshot/windows-snapshot/Cargo.toml')], { stdio: 'inherit', windowsHide: true });+ binary = path.join(desktop, 'native/region-screenshot/windows-snapshot/target/release/cindy-windows-screenshot-snapshot.exe');+ }+ helper = spawn(binary, [], { windowsHide: true, stdio: 'pipe' });+ createInterface({ input: helper.stdout }).on('line', line => {+ const reply = JSON.parse(line), task = pending.get(reply.id);+ if (task) { pending.delete(reply.id); task.resolve(reply); }+ });+ helper.on('error', error => { for (const task of pending.values()) { clearTimeout(task.timeout); task.reject(error); } pending.clear(); });+ // Real HWND hierarchy proves the snapshot path independently of Chromium+ // versions that render the whole page into one native window.+ nativeWindow = spawn('powershell.exe', ['-NoProfile', '-STA', '-Command', `+ Add-Type -AssemblyName System.Windows.Forms+ $form = New-Object System.Windows.Forms.Form+ $form.StartPosition = 'Manual'+ $form.Location = New-Object System.Drawing.Point(100,100)+ $form.ClientSize = New-Object System.Drawing.Size(500,400)+ $panel = New-Object System.Windows.Forms.Panel+ $panel.Bounds = New-Object System.Drawing.Rectangle(30,60,350,250)+ $inner = New-Object System.Windows.Forms.Panel+ $inner.Bounds = New-Object System.Drawing.Rectangle(20,30,180,100)+ $panel.Controls.Add($inner)+ $form.Controls.Add($panel)+ $form.Show()+ $form.Hide()+ $form.Show()+ $r = $inner.RectangleToScreen($inner.ClientRectangle)+ $json = @{ hwnd=$form.Handle.ToInt64(); rect=@($r.Left,$r.Top,$r.Right,$r.Bottom) } | ConvertTo-Json -Compress+ [Console]::Out.WriteLine($json)+ [Console]::Out.Flush()+ [System.Windows.Forms.Application]::Run($form)+ `], { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });+ const nativeGeometry = await new Promise((resolve, reject) => {+ const timer = setTimeout(() => reject(new Error('Native fixture did not open')), 10000);+ createInterface({ input: nativeWindow.stdout }).once('line', line => { clearTimeout(timer); resolve(JSON.parse(line)); });+ nativeWindow.once('error', error => { clearTimeout(timer); reject(error); });+ });+ let nativeSnapshot;+ const nativeDeadline = Date.now() + 3000;+ do {+ nativeSnapshot = (await ask({ op: 'windows' })).windows.find(w => w.hwnd === nativeGeometry.hwnd);+ if (nativeSnapshot) break;+ await delay(20);+ } while (Date.now() < nativeDeadline);+ assert.ok(nativeSnapshot, `native test window is captured: ${JSON.stringify(nativeGeometry)}`);+ const center = { x: (nativeGeometry.rect[0] + nativeGeometry.rect[2]) / 2, y: (nativeGeometry.rect[1] + nativeGeometry.rect[3]) / 2 };+ assert.deepEqual(nativeSnapshot.snapRects.find(r => r[0] <= center.x && center.x < r[2] && r[1] <= center.y && center.y < r[3]),+ nativeGeometry.rect, 'native snapshot hits the inner child before its parent');+ nativeWindow.kill();+ nativeWindow = undefined;+ browser = await chromium.launchPersistentContext(path.join(temp, 'chrome'), {+ executablePath: browserPath, headless: false, viewport: null,+ args: ['--force-device-scale-factor=1', `--window-position=${position.join(',')}`, '--window-size=1000,800', '--no-first-run'],+ });+ const source = browser.pages()[0];+ await source.setContent('<!doctype html><title>Region snapping regression</title><style>body{margin:0;font:18px sans-serif}header{padding:30px}main{padding:40px}button{padding:18px}section{height:260px;border:1px solid}</style><header><button>Code</button></header><main><section>File list</section><p>Document body</p></main>');+ await source.bringToFront();+ const geometry = await source.evaluate(() => ({ x: screenX, y: screenY, outerWidth, outerHeight, width: innerWidth, height: innerHeight }));+ // Independently measure the webpage viewport from browser geometry, never from+ // the helper reply being tested. Chrome's invisible resize border is symmetric.+ const border = (geometry.outerWidth - geometry.width) / 2;+ const content = { x: geometry.x + border, y: geometry.y + geometry.outerHeight - geometry.height - border, width: geometry.width, height: geometry.height };+ const windows = (await ask({ op: 'windows' })).windows;+ const target = windows.find(({ rect: r }) => Math.abs(r[0] - geometry.x) <= 10+ && Math.abs(r[1] - geometry.y) <= 10 && Math.abs(r[2] - r[0] - geometry.outerWidth) <= 20);+ assert.ok(target, 'Headed Chrome is visible');+ const [left, top, right, bottom] = target.rect;+ const viewport = { width: right - left, height: bottom - top };+ const local = (r) => ({ x: r[0] - left, y: r[1] - top, width: r[2] - r[0], height: r[3] - r[1] });+ const closeRect = (actual, expected, label) => {+ assert.ok(actual, `${label}: missing region`);+ for (const key of ['x', 'y', 'width', 'height']) assert.ok(Math.abs(actual[key] - expected[key]) <= 2, `${label} ${key}: ${actual[key]} != ${expected[key]}`);+ };+ assert.ok(Array.isArray(target.snapRects), 'native window snapshot includes default regions');+ const contentLocal = { ...content, x: content.x - left, y: content.y - top };+ const samples = [];+ for (const selector of ['button', 'section', 'p']) {+ const box = await source.locator(selector).boundingBox();+ const point = { x: Math.round(content.x + box.x + box.width / 2), y: Math.round(content.y + box.y + box.height / 2) };+ // A new Chrome starts accessibility lazily after its first UIA query. Poll+ // the observable result with a deadline; an old helper still fails here.+ let hit;+ const deadline = Date.now() + 4000;+ do {+ hit = await ask({ op: 'elements', hwnd: target.hwnd, ...point });+ if (hit.preferredRect && Math.abs(hit.preferredRect[1] - content.y) <= 2) break;+ await delay(100);+ } while (Date.now() < deadline);+ const expected = { ...content, x: content.x - left, y: content.y - top };+ closeRect(local(hit.preferredRect ?? hit.rects[0] ?? target.rect), expected, selector);+ if (selector === 'button') assert.ok(hit.rects.some(r => r[2] - r[0] < content.width && r[3] - r[1] < content.height), 'native button branch must remain reachable');+ samples.push({ selector, point, expected, native: hit });+ }+ const chromeHit = await ask({ op: 'elements', hwnd: target.hwnd, x: Math.round(left + viewport.width / 2), y: top + 25 });+ closeRect(local(chromeHit.preferredRect ?? chromeHit.rects[0] ?? target.rect), { x: 0, y: 0, ...viewport }, 'browser-chrome');+ // Some Chromium versions expose content only through UIA, without child HWNDs.+ // Keep the actual native snapshot; never fabricate regions from expected DOM geometry.+ const captured = (await ask({ op: 'windows' })).windows.find(w => w.hwnd === target.hwnd);+ assert.ok(captured, 'browser remains available for capture');+ // Playwright's bringToFront activates the tab but Windows can refuse focus+ // stealing. Keep only our disposable Chrome above other apps during capture.+ execFileSync('powershell.exe', ['-NoProfile', '-Command', `Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class RegionTestWindow { [DllImport("user32.dll")] public static extern bool SetWindowPos(IntPtr h, IntPtr after, int x, int y, int w, int height, uint flags); }'; if (![RegionTestWindow]::SetWindowPos([IntPtr]${target.hwnd}, [IntPtr](-1), 0, 0, 0, 0, 0x13)) { exit 1 }`], { windowsHide: true });+ const visible = (await ask({ op: 'windows' })).windows.find(({ rect: r }) => r[0] <= content.x + 50 && content.x + 50 < r[2] && r[1] <= content.y + 50 && content.y + 50 < r[3]);+ assert.equal(visible?.hwnd, target.hwnd, 'Native capture must show the tested Chrome, not an occluding app');+ // Wait for Windows to compose the newly exposed native window (E2E only).+ await delay(100);+ const frame = await ask({ op: 'frame', x: left, y: top, ...viewport, edge: 1600 });+ assert.ok(frame.pixels, 'Native screenshot is available');+ const bundle = await buildPage(adhocPage(path.join(desktop, 'scripts/fixtures/region-capture/native-overlay.tsx')));+ renderer = await chromium.launch({ executablePath: browserPath, headless: true });+ const report = [];+ for (const theme of ['light', 'dark']) {+ const context = await renderer.newContext({ viewport });+ const page = await context.newPage();+ const errors = []; page.on('pageerror', error => errors.push(error.message));+ const queried = [];+ // Hold actual native replies to expose intermediate fallback frames. The+ // expected geometry is used only by assertions, never supplied to the UI.+ let releaseElements;+ let elementsReady = Promise.resolve();+ await page.exposeFunction('nativeElements', async (x, y) => {+ await elementsReady;+ const hit = await ask({ op: 'elements', hwnd: target.hwnd, x: Math.round(x + left), y: Math.round(y + top) });+ queried.push({ x, y, hit });+ return { window: { id: 0, rect: { x: 0, y: 0, ...viewport } }, rects: hit.rects.map(local), ...(hit.preferredRect ? { preferredRect: local(hit.preferredRect) } : {}) };+ });+ await page.addInitScript(begin => { window.nativeBegin = begin; }, {+ token: 'native-blackbox', locale: 'en', viewport, safeArea: { x: 0, y: 0, ...viewport }, scaleFactor: 1,+ frame: { width: frame.width, height: frame.height, data: Array.from(Buffer.from(frame.pixels, 'base64')) },+ windows: [{ id: 0, rect: { x: 0, y: 0, ...viewport }, snapRects: captured.snapRects.map(local) }], cursor: null, lastRegion: null,+ });+ await page.route('**/*', route => {+ const response = pageResponse(route.request().url(), bundle, { page: 'native-region', case: 'default', theme, locale: 'en', platform: 'win32', viewport, data: {} });+ return response ? route.fulfill(response) : route.abort();+ });+ await page.goto(origin);+ await page.waitForFunction(() => window.__uiPage?.ready);+ await page.mouse.move(contentLocal.x + 25, contentLocal.y + 35);+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor();+ elementsReady = new Promise(resolve => { releaseElements = resolve; });+ // Observe DOM commits and painted frames across continuous movement, not+ // just the final result after an async wait (the gap in the original test).+ await page.evaluate(() => {+ window.regionTrace = [];+ const sample = () => {+ const outline = document.querySelector('div[style*="100000px"]');+ if (outline) {+ const { x, y, width, height } = outline.getBoundingClientRect();+ window.regionTrace.push({ x, y, width, height });+ }+ };+ const observer = new MutationObserver(sample);+ observer.observe(document.body, { attributes: true, childList: true, subtree: true });+ let frame;+ const tick = () => { sample(); frame = requestAnimationFrame(tick); };+ frame = requestAnimationFrame(tick);+ window.stopRegionTrace = () => { observer.disconnect(); cancelAnimationFrame(frame); sample(); return window.regionTrace; };+ });+ for (let step = 0; step < 40; step++) {+ await page.mouse.move(contentLocal.x + 30 + step * 10, contentLocal.y + 40 + step * 4);+ closeRect(await page.locator('div[style*="100000px"]').boundingBox(), contentLocal, `${theme}/pending-move-${step}`);+ }+ const trace = await page.evaluate(() => window.stopRegionTrace());+ assert.ok(trace.length >= 40, 'observe each move while native queries are pending');+ trace.forEach((rect, index) => closeRect(rect, contentLocal, `${theme}/frame-${index}`));+ await writeFile(path.join(temp, `${theme}-movement.json`), JSON.stringify(trace));+ // Crossing the content boundary is synchronous even with the query held.+ await page.mouse.move(viewport.width / 2, 25);+ closeRect(await page.locator('div[style*="100000px"]').boundingBox(), { x: 0, y: 0, ...viewport }, `${theme}/pending-boundary`);+ releaseElements();+ for (const sample of samples) {+ await page.mouse.move(sample.point.x - left, sample.point.y - top);+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor();+ // Observe the actual rendered outline, not a test-only selection accessor.+ const outline = page.locator('div[style*="100000px"]');+ closeRect(await outline.boundingBox(), sample.expected, `${theme}/${sample.selector}`);+ await page.screenshot({ path: path.join(temp, `${theme}-${sample.selector}.png`) });+ }+ const before = queried.length;+ await page.mouse.move(viewport.width / 2, 25);+ await page.getByText(`${viewport.width} × ${viewport.height}`, { exact: true }).waitFor();+ // Return to the button and exercise both finer and coarser candidates.+ const first = samples[0];+ const buttonQueries = queried.length;+ await page.mouse.move(first.point.x - left, first.point.y - top);+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor();+ // The default is now immediate; wheel checks must wait for fine elements,+ // rather than treating the content outline as proof that UIA has completed.+ const fineDeadline = Date.now() + 5000;+ const fineReady = () => queried.slice(buttonQueries).some(({ x, y, hit }) =>+ Math.abs(x - (first.point.x - left)) < 1 && Math.abs(y - (first.point.y - top)) < 1+ && hit.rects.some(r => r[2] - r[0] < content.width && r[3] - r[1] < content.height));+ while (!fineReady() && Date.now() < fineDeadline) await delay(20);+ assert.ok(fineReady(), 'fresh native fine elements are available at the button');+ await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));+ await page.mouse.wheel(0, 120);+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor({ state: 'hidden' });+ await page.mouse.wheel(0, -120);+ await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).waitFor();+ await page.mouse.click(first.point.x - left, first.point.y - top);+ await page.mouse.move(viewport.width - 20, 25);+ assert.equal(await page.getByText(`${Math.round(content.width)} × ${Math.round(content.height)}`, { exact: true }).count(), 1, 'click locks content selection');+ assert.ok(queried.length > before, 'moving from chrome into content performs a fresh native query');+ assert.deepEqual(errors, []);+ await page.screenshot({ path: path.join(temp, `${theme}-selected.png`) });+ report.push({ theme, content, window: target.rect, nativeRegions: captured.snapRects.length, movementSamples: trace.length, checks: ['pending-movement', 'pending-boundary', 'button', 'file-list', 'body', 'browser-chrome', 'wheel', 'click'] });+ await context.close();+ }+ await writeFile(path.join(temp, 'report.json'), JSON.stringify(report, null, 2));+ console.log(`REGION_BLACKBOX=passed evidence=${temp}`);+} finally {+ await renderer?.close();+ await browser?.close();+ helper?.kill();+ nativeWindow?.kill();+ for (const task of pending.values()) clearTimeout(task.timeout);+ // Keep only reviewable screenshots/report; remove the isolated browser profile.+ await rm(path.join(temp, 'chrome'), { recursive: true, force: true });+}diff --git a/apps/desktop/scripts/fixtures/region-capture/capture-fixture.tsx b/apps/desktop/scripts/fixtures/region-capture/capture-fixture.tsxindex ab1bb57b44..4912d80dd3 100644--- a/apps/desktop/scripts/fixtures/region-capture/capture-fixture.tsx+++ b/apps/desktop/scripts/fixtures/region-capture/capture-fixture.tsx@@ -49,6 +49,7 @@ function ScrollFixture({ bridge, locale }: { bridge: RegionScreenshotOverlayBridmountPage({render: (ctx) => {+ const snapping = ctx.case === 'snapping';const mode = (['default', 'idle', 'inactive', 'drag', 'preview-motion'].includes(ctx.case) ? 'scroll' : ctx.case) as RegionCaptureMode;qa.idle = ctx.case === 'idle';qa.manualScroll = ctx.case === 'preview-motion' ? 300 : null;@@ -56,10 +57,11 @@ mountPage({let selected = { x: 100, y: 100, width: 480, height: 240 };const bridge: RegionScreenshotOverlayBridge = {autoScroll: async (_token, enabled) => { qa.automatic = enabled; return true; },- shellReady() { queueMicrotask(() => begin?.({ token: 'fixture', locale: ctx.locale, viewport: { width: innerWidth, height: innerHeight }, scaleFactor: 1, frame: frame(), windows: [], lastRegion: null, cursor: ctx.case === 'inactive' ? null : { x: 0, y: 0 }, safeArea: { x: 0, y: 0, width: innerWidth, height: innerHeight } })); },+ shellReady() { queueMicrotask(() => begin?.({ token: 'fixture', locale: ctx.locale, viewport: { width: innerWidth, height: innerHeight }, scaleFactor: 1, frame: frame(), windows: snapping ? [{ id: 0, rect: { x: 0, y: 0, width: 700, height: 500 } }] : [], lastRegion: null, cursor: ctx.case === 'inactive' ? null : snapping ? { x: 150, y: 180 } : { x: 0, y: 0 }, safeArea: { x: 0, y: 0, width: innerWidth, height: innerHeight } })); },onBegin: (callback) => { begin = callback; return () => { begin = undefined; }; }, onEnd: () => () => {}, onClear: () => () => {},painted() {}, focused() {}, selectionStarted() {}, copyText() {}, cancel() {}, abandon() {},- elementsAt: async () => [], startLive: async (_token, captureMode, selection) => { selected = selection; return { mode: captureMode, selection, first: frame(qa.manualScroll ?? 0), scrollLayout: scrollCaptureLayout(selection, { width: innerWidth, height: innerHeight })!, recordingLayout: { viewport: { width: innerWidth, height: innerHeight }, controls: recordingControlsBounds(selection, { width: innerWidth, height: innerHeight }, false)! } }; },+ elementsAt: async (_token, _window, x) => snapping ? { window: { id: 1, rect: { x: 100, y: 100, width: 480, height: 360 } }, rects: [x < 320 ? { x: 120, y: 140, width: 180, height: 140 } : { x: 340, y: 140, width: 200, height: 140 }] } : null,+ startLive: async (_token, captureMode, selection) => { selected = selection; return { mode: captureMode, selection, first: frame(qa.manualScroll ?? 0), scrollLayout: scrollCaptureLayout(selection, { width: innerWidth, height: innerHeight })!, recordingLayout: { viewport: { width: innerWidth, height: innerHeight }, controls: recordingControlsBounds(selection, { width: innerWidth, height: innerHeight }, false)! } }; },recordingControls: async (_token, expanded, point) => recordingControlsBounds(selected, { width: innerWidth, height: innerHeight }, expanded, point),previewRecording: async () => false,restoreEditor: async () => { qa.restored = true; return true; },diff --git a/apps/desktop/scripts/fixtures/region-capture/native-overlay.tsx b/apps/desktop/scripts/fixtures/region-capture/native-overlay.tsxnew file mode 100644index 0000000000..385ddcabeb--- /dev/null+++ b/apps/desktop/scripts/fixtures/region-capture/native-overlay.tsx@@ -0,0 +1,25 @@+// Native black-box runner supplies captured pixels and live helper replies.+// Only the transport is adapted; selection, pointer/wheel handling and painting+// all run through the production overlay. No expected rectangle enters the UI.+import { mountPage } from '../../ui-page/harness';+import { RegionScreenshotOverlay } from '../../../src/renderer/features/region-screenshot/RegionScreenshotOverlay';+import type { RegionScreenshotBegin, RegionScreenshotHit } from '../../../src/shared/regionScreenshot';+import type { RegionScreenshotOverlayBridge } from '../../../src/renderer/features/region-screenshot/bridge';++const native = window as unknown as {+ nativeBegin: RegionScreenshotBegin;+ nativeElements: (x: number, y: number) => Promise<RegionScreenshotHit>;+};+mountPage({ render: () => {+ let begin: Parameters<RegionScreenshotOverlayBridge['onBegin']>[0] | undefined;+ const bridge = {+ onBegin: (callback: typeof begin) => { begin = callback; return () => { begin = undefined; }; },+ onEnd: () => () => {}, onClear: () => () => {},+ shellReady: () => queueMicrotask(() => begin?.({ ...native.nativeBegin,+ frame: { ...native.nativeBegin.frame, data: new Uint8Array(native.nativeBegin.frame.data) },+ })),+ elementsAt: (_token: string, _window: number, x: number, y: number) => native.nativeElements(x, y),+ painted() {}, focused() {}, selectionStarted() {}, abandon() {}, cancel() {}, copyText() {},+ } as RegionScreenshotOverlayBridge;+ return <RegionScreenshotOverlay bridge={bridge} />;+} });diff --git a/apps/desktop/scripts/fixtures/region-capture/ui-page.mjs b/apps/desktop/scripts/fixtures/region-capture/ui-page.mjsindex b09da3dabc..5cc66a3a9a 100644--- a/apps/desktop/scripts/fixtures/region-capture/ui-page.mjs+++ b/apps/desktop/scripts/fixtures/region-capture/ui-page.mjs@@ -4,6 +4,7 @@ export default {real: 'RegionScreenshotOverlay, ScrollCaptureOverlay, LiveCapturePanel, ScrollCapture, CaptureRecorder, real Chromium canvas/MediaRecorder and GIF encoding',notCovered: 'Native screen acquisition, global keyboard sampling, macOS permissions and native save dialogs; frame source is synthetic.',cases: {+ snapping: { about: 'Correct a transparent window guess, hover adjacent elements, cycle hierarchy and click to select' },default: { about: 'Scroll overlapping text, finish, export the actual stitched pixels' },idle: { about: 'Repainting before the first scroll keeps the gesture hint, then scrolling still exports a long image' },'preview-motion': { about: 'Prepending reveals new rows without jumping existing content; growth and scrolling share one animation' },@@ -14,6 +15,21 @@ export default {},async ready(page) { await page.locator('canvas').first().waitFor(); },async check(page, ui) {+ if (ui.case === 'snapping') {+ await page.getByText('180 × 140', { exact: true }).waitFor();+ await ui.capture('first-element');+ await page.mouse.move(400, 180);+ await page.getByText('200 × 140', { exact: true }).waitFor();+ await ui.capture('adjacent-element');+ await page.mouse.wheel(0, -120);+ await page.getByText('480 × 360', { exact: true }).waitFor();+ await ui.capture('corrected-window');+ await page.mouse.click(400, 180);+ await page.mouse.move(650, 550);+ assert.equal(await page.getByText('480 × 360', { exact: true }).count(), 1, 'click keeps the resolved window selected');+ await ui.capture('selected');+ return;+ }if (ui.case === 'preview-motion') await page.emulateMedia({ reducedMotion: 'no-preference' });const isScroll = ['default', 'idle', 'preview-motion'].includes(ui.case);const button = async (key) => page.getByRole('button', { name: await ui.t(key), exact: true });diff --git a/apps/desktop/src/main/region-screenshot/__tests__/snapshotHost.test.ts b/apps/desktop/src/main/region-screenshot/__tests__/snapshotHost.test.tsindex cb522174de..98b7c0e308 100644--- a/apps/desktop/src/main/region-screenshot/__tests__/snapshotHost.test.ts+++ b/apps/desktop/src/main/region-screenshot/__tests__/snapshotHost.test.ts@@ -64,6 +64,31 @@ describe('snapshot helper failure isolation', () => {await expect(helper.windows()).resolves.toEqual([]);});+ // Region identification regression: rectangle z-order cannot identify a+ // click-through overlay. Exercise the native protocol through the production+ // host; an opaque unknown window must not expose windows behind it. Unit,+ // in-memory child streams; existing tests cover failures, not target identity.+ it('resolves the native hit only against captured windows, without falling through an occluder', async () => {+ vi.stubGlobal('process', Object.create(process, { platform: { value: 'darwin' } }));+ const captured = [+ { handle: 5, pid: 9, rect: [0, 0, 200, 300] as [number, number, number, number] },+ { handle: 6, pid: 10, rect: [20, 20, 100, 100] as [number, number, number, number] },+ ];+ const request = helper.windowAt(captured, 30, 40, [99]);+ expect(JSON.parse(child.stdin.read().toString())).toMatchObject({ op: 'window-at', x: 30, y: 40, exclude: [99] });+ child.stdout.write(JSON.stringify({ id: 1, hwnd: 6 }) + '\n');+ await expect(request).resolves.toBe(captured[1]);+ const occluded = helper.windowAt(captured, 30, 40, [99]);+ child.stdout.write(JSON.stringify({ id: 2, hwnd: 77 }) + '\n');+ await expect(occluded).resolves.toBeNull();+ const stale = helper.windowAt(captured, 150, 40, [99]);+ child.stdout.write(JSON.stringify({ id: 3, hwnd: 6 }) + '\n');+ await expect(stale).resolves.toBeNull();+ const unavailable = helper.windowAt(captured, 30, 40, [99]);+ child.stdout.write(JSON.stringify({ id: 4, error: 'unknown-op' }) + '\n');+ await expect(unavailable).resolves.toBe(captured[0]);+ });+it('bounds a stuck helper instead of accumulating timed-out queries', async () => {const result = helper.windows();await vi.advanceTimersByTimeAsync(1500);@@ -80,12 +105,31 @@ describe('snapshot helper failure isolation', () => {JSON.stringify({id: 1,windows: [- { hwnd: 5, pid: 9, rect: [0, 0, 20, 30] },+ // Captured child geometry is optional and must stay within its window.+ { hwnd: 5, pid: 9, rect: [0, 0, 20, 30], snapRects: [[0, 5, 20, 30], [-1, 0, 20, 30], [0, 0, -1, 2], null] },+ { hwnd: 7, rect: [0, 0, 20, 30] },{ hwnd: 6, rect: [0, 0, -1, 2] },],}) + '\n',);- await expect(result).resolves.toEqual([{ handle: 5, pid: 9, rect: [0, 0, 20, 30] }]);+ await expect(result).resolves.toEqual([+ { handle: 5, pid: 9, rect: [0, 0, 20, 30], snapRects: [[0, 5, 20, 30]] },+ { handle: 7, pid: null, rect: [0, 0, 20, 30] },+ ]);+ });++ // Browser content regression: the native preferred rectangle must survive+ // the production transport, while malformed/old replies remain usable.+ // Existing tests only cover window geometry; unit, one in-memory response.+ it.each([+ { preferredRect: [10, 20, 190, 280], expected: { preferredRect: [10, 20, 190, 280] } },+ { preferredRect: [100, 100, 190, 280], expected: {} },+ { preferredRect: [0, 0, -1, 30], expected: {} },+ { preferredRect: undefined, expected: {} },+ ])('preserves only a valid content region containing the query point: $preferredRect', async ({ preferredRect, expected }) => {+ const result = helper.elements({ handle: 5, pid: null, rect: [0, 0, 200, 300] }, 30, 40);+ child.stdout.write(JSON.stringify({ id: 1, rects: [[20, 30, 60, 70], [100, 100, 110, 110], null], preferredRect }) + '\n');+ await expect(result).resolves.toEqual({ rects: [[20, 30, 60, 70]], ...expected });});// Unit: resolving through a transparent overlay must not admit a new windowdiff --git a/apps/desktop/src/main/region-screenshot/__tests__/windows.test.ts b/apps/desktop/src/main/region-screenshot/__tests__/windows.test.tsindex 6d7bb87fb0..e999a32aea 100644--- a/apps/desktop/src/main/region-screenshot/__tests__/windows.test.ts+++ b/apps/desktop/src/main/region-screenshot/__tests__/windows.test.ts@@ -101,6 +101,7 @@ function fakeWindow() {show: vi.fn(() => { visible = true; window.emit('show'); }),isDestroyed: () => destroyed,isVisible: () => visible,+ getMediaSourceId: () => 'window:999:0',setVisibleOnAllWorkspaces: vi.fn(),setAlwaysOnTop: vi.fn(),setBounds: vi.fn(),@@ -162,6 +163,44 @@ describe('screenshot window lifecycle', () => {vi.unstubAllGlobals();});+ // Region detection: native hit-testing can replace the transparent front+ // rectangle. Existing lifecycle tests never queried elements. Minimal input:+ // two overlapping windows on an offset display; unit, no OS UI or real waits.+ it('returns the corrected window in display coordinates and drops a query after the session closes', async () => {+ vi.stubGlobal('process', Object.create(process, { platform: { value: 'darwin' } }));+ const monitor = { ...display, bounds: { x: -100, y: 100, width: 100, height: 100 } } as Display;+ const overlay = pool.ensure(monitor)!;+ const owner = overlay.window.webContents;+ pool.markShellReady(owner, owner.mainFrame);+ const captured = [+ { handle: 1, pid: 1, rect: [-100, 100, 0, 200] as [number, number, number, number], snapRects: [[-100, 120, 0, 200] as [number, number, number, number]] },+ { handle: 2, pid: 2, rect: [-90, 110, -10, 190] as [number, number, number, number] },+ ];+ const hit = vi.fn(async () => captured[1]);+ const elements = vi.fn(async () => ({ rects: [[-80, 120, -50, 150]], preferredRect: [-80, 120, -50, 150] }));+ const session = new RegionScreenshotSession({ pool, captures: [{ display: monitor, image }], helper: { windowAt: hit, elements } as unknown as SnapshotHelper, windows: captured, lastRegions: new Map(), onClosed: vi.fn(), onResult: vi.fn() });+ await session.open();+ // Snapshot defaults must reach the first paint on an offset display, before+ // any ELEMENTS_AT reply; previous coverage only inspected query conversion.+ const begin = vi.mocked(owner.send).mock.calls.find(([name]) => name === 'region-screenshot:begin')![1];+ expect(begin.windows[0].snapRects).toEqual([{ x: 0, y: 20, width: 100, height: 80 }]);+ await expect(session.elementsAt(owner, owner.mainFrame, session.token, 0, 25, 25)).resolves.toEqual({+ window: { id: 1, rect: { x: 10, y: 10, width: 80, height: 80 } },+ rects: [{ x: 20, y: 20, width: 30, height: 30 }],+ preferredRect: { x: 20, y: 20, width: 30, height: 30 },+ });+ expect(hit).toHaveBeenCalledWith(captured, -75, 125, [999]);+ expect(elements).toHaveBeenCalledWith(captured[1], -75, 125);+ hit.mockClear();+ await expect(session.elementsAt(owner, owner.mainFrame, 'old-token', 0, 25, 25)).resolves.toBeNull();+ await expect(session.elementsAt(owner, owner.mainFrame, session.token, 0, 125, 25)).resolves.toBeNull();+ expect(hit).not.toHaveBeenCalled();+ const pending = session.elementsAt(owner, owner.mainFrame, session.token, 0, 25, 25);+ session.cancel(owner, owner.mainFrame, session.token);+ await expect(pending).resolves.toBeNull();+ expect(elements).toHaveBeenCalledTimes(1);+ });+// 用户反馈:打开截图时整屏缩放。实机验证 Aura 动画;unit 保护临时开关// 不泄漏到其它窗口、不清除用户启动参数,且异常也恢复。tier: unit。it.each([diff --git a/apps/desktop/src/main/region-screenshot/index.ts b/apps/desktop/src/main/region-screenshot/index.tsindex dd6c90156f..ac4a27bd8b 100644--- a/apps/desktop/src/main/region-screenshot/index.ts+++ b/apps/desktop/src/main/region-screenshot/index.ts@@ -210,7 +210,7 @@ function registerIpc(): void {ipcMain.handle(C.ELEMENTS_AT,(event, token: unknown, windowId: unknown, x: unknown, y: unknown) =>- active?.elementsAt(event.sender, event.senderFrame, token, windowId, x, y) ?? [],+ active?.elementsAt(event.sender, event.senderFrame, token, windowId, x, y) ?? null,);ipcMain.handle(C.EXPORT,diff --git a/apps/desktop/src/main/region-screenshot/session.ts b/apps/desktop/src/main/region-screenshot/session.tsindex e362f0d3d0..c714ec03ed 100644--- a/apps/desktop/src/main/region-screenshot/session.ts+++ b/apps/desktop/src/main/region-screenshot/session.ts@@ -29,6 +29,7 @@ import {type RegionScreenshotBegin,type RegionScreenshotBitmap,type RegionScreenshotWindow,+ type RegionScreenshotHit,} from '../../shared/regionScreenshot';import { hideScreenshotOverlay, showScreenshotOverlay, type OverlayPool, type OverlayWindow } from './overlays';import type { SnapshotHelper, SnapshotWindow } from './snapshotHost';@@ -204,7 +205,11 @@ export class RegionScreenshotSession {const windows: RegionScreenshotWindow[] = [];this.deps.windows.forEach((window, id) => {const rect = intersectRegion(toLocal(toGlobalDip(window.rect), display), viewport);- if (rect) windows.push({ id, rect });+ if (rect) windows.push({ id, rect,+ ...(window.snapRects ? { snapRects: window.snapRects+ .map((region) => intersectRegion(toLocal(toGlobalDip(region), display), viewport))+ .filter((region): region is RegionRect => region !== null) } : {}),+ });});const bitmap = image.toBitmap();return {@@ -284,18 +289,30 @@ export class RegionScreenshotSession {windowId: unknown,x: unknown,y: unknown,- ): Promise<RegionRect[]> {+ ): Promise<RegionScreenshotHit | null> {const member = this.member(sender, frame, token);- if (!member || !Number.isInteger(windowId)) return [];- if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x + y)) return [];- const window = this.deps.windows[windowId as number];- if (!window) return [];+ if (!member || !Number.isInteger(windowId)) return null;+ if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x + y)) return null;+ if (!this.deps.windows[windowId as number]) return null;const { display } = member.capture;+ if (x < 0 || y < 0 || x >= display.bounds.width || y >= display.bounds.height) return null;const global = { x: x + display.bounds.x, y: y + display.bounds.y };const point = process.platform === 'win32' ? screen.dipToScreenPoint(global) : global;- const rects = await this.deps.helper.elements(window, point.x, point.y);- if (this.closed) return [];- return rects.map((rect) => toLocal(toGlobalDip(rect), display));+ const excluded = process.platform === 'darwin'+ ? [...this.members.values()].filter(({ overlay }) => !overlay.window.isDestroyed())+ .map(({ overlay }) => Number(overlay.window.getMediaSourceId().split(':')[1]))+ .filter((id) => Number.isSafeInteger(id) && id > 0)+ : [];+ const window = await this.deps.helper.windowAt(this.deps.windows, point.x, point.y, excluded);+ if (this.closed) return null;+ if (!window) return { window: null, rects: [] };+ const hit = await this.deps.helper.elements(window, point.x, point.y);+ if (this.closed) return null;+ return {+ window: { id: this.deps.windows.indexOf(window), rect: toLocal(toGlobalDip(window.rect), display) },+ rects: hit.rects.map((rect) => toLocal(toGlobalDip(rect), display)),+ ...(hit.preferredRect ? { preferredRect: toLocal(toGlobalDip(hit.preferredRect), display) } : {}),+ };}copyText(diff --git a/apps/desktop/src/main/region-screenshot/snapshotHost.ts b/apps/desktop/src/main/region-screenshot/snapshotHost.tsindex 0cc76155c1..7cba8ed2ae 100644--- a/apps/desktop/src/main/region-screenshot/snapshotHost.ts+++ b/apps/desktop/src/main/region-screenshot/snapshotHost.ts@@ -32,6 +32,7 @@ export interface SnapshotWindow {pid: number | null;/** Windows 为物理像素, macOS 为全局 DIP; [left, top, right, bottom]。 */rect: [number, number, number, number];+ snapRects?: [number, number, number, number][];}const binaryName =@@ -220,24 +221,52 @@ export class SnapshotHelper {const windows: SnapshotWindow[] = [];for (const raw of list) {if (!raw || typeof raw !== 'object') continue;- const { hwnd, pid, rect } = raw as Record<string, unknown>;+ const { hwnd, pid, rect, snapRects } = raw as Record<string, unknown>;if (typeof hwnd !== 'number' || !isRectTuple(rect)) continue;- windows.push({ handle: hwnd, pid: typeof pid === 'number' ? pid : null, rect });+ windows.push({ handle: hwnd, pid: typeof pid === 'number' ? pid : null, rect,+ ...(Array.isArray(snapRects) ? { snapRects: snapRects.slice(0, 256).filter(isRectTuple)+ .filter((r) => r[0] >= rect[0] && r[1] >= rect[1] && r[2] <= rect[2] && r[3] <= rect[3]) } : {}),+ });}return windows;}+ /** Resolve only captured targets; an unknown native hit is an occluder, not a reason to look below it. */+ async windowAt(+ windows: readonly SnapshotWindow[], x: number, y: number, exclude: readonly number[],+ ): Promise<SnapshotWindow | null> {+ const contains = ({ rect }: SnapshotWindow) => x >= rect[0] && x < rect[2] && y >= rect[1] && y < rect[3];+ if (process.platform === 'darwin') {+ const response = await this.request({ op: 'window-at', x, y, exclude });+ if (typeof response?.hwnd === 'number') {+ return windows.find((window) => window.handle === response.hwnd && contains(window)) ?? null;+ }+ }+ // Windows already excludes click-through windows during enumeration. Old+ // native helpers and unavailable helpers retain rectangle-only snapping.+ return windows.find(contains) ?? null;+ }+/** 指针下的界面元素 (由内到外), 坐标系同 {@link SnapshotWindow.rect}。 */- async elements(window: SnapshotWindow, x: number, y: number): Promise<SnapshotWindow['rect'][]> {+ async elements(window: SnapshotWindow, x: number, y: number): Promise<{+ rects: SnapshotWindow['rect'][]; preferredRect?: SnapshotWindow['rect'];+ }> {const response = await this.request({op: 'elements',hwnd: window.handle,+ rect: window.rect,...(window.pid !== null ? { pid: window.pid } : {}),x: Math.round(x),y: Math.round(y),});const rects = Array.isArray(response?.rects) ? response.rects : [];- return rects.filter(isRectTuple);+ const preferred = response?.preferredRect;+ const valid = (rect: unknown): rect is SnapshotWindow['rect'] => isRectTuple(rect)+ && rect[0] <= x && x < rect[2] && rect[1] <= y && y < rect[3];+ return {+ rects: rects.filter(valid),+ ...(valid(preferred) ? { preferredRect: preferred } : {}),+ };}/** macOS: 截图前的前台应用进程号, 结束后把焦点还给它。 */diff --git a/apps/desktop/src/main/remote-agent/__tests__/guestHost.test.ts b/apps/desktop/src/main/remote-agent/__tests__/guestHost.test.tsindex 684a38894c..83cafb46bb 100644--- a/apps/desktop/src/main/remote-agent/__tests__/guestHost.test.ts+++ b/apps/desktop/src/main/remote-agent/__tests__/guestHost.test.ts@@ -600,9 +600,12 @@ describe('guest provider access and usage', () => {const started: Started[] = [];const host = makeHost(started, { bindGuestProviderRoute: bind });await host.handle(GUEST, { op: 'open', runId: RUN_1, agentKind: 'claude-code', payload: { json: openPayload('task-1') } });- await vi.waitFor(() => expect(currents).toHaveLength(1));+ await vi.waitFor(() => expect(currents).toHaveLength(1), { timeout: 10_000 });await host.handle(GUEST, { op: 'open', runId: RUN_2, agentKind: 'claude-code', payload: { json: openPayload('task-1') } });- await vi.waitFor(() => expect(started).toHaveLength(1));+ // Windows CI run 40667219234 exceeded the default 1s disk-preparation wait.+ // Observe RUN_2's completed handshake before checking that RUN_1 stays stale.+ await waitForStarted(host, GUEST, RUN_2);+ expect(started).toHaveLength(1);expect(currents[0]!()).toBe(false);openGate();await new Promise((resolve) => setTimeout(resolve, 20));diff --git a/apps/desktop/src/preload/regionScreenshotPreload.ts b/apps/desktop/src/preload/regionScreenshotPreload.tsindex fe2042e9e3..8b99bb1ed4 100644--- a/apps/desktop/src/preload/regionScreenshotPreload.ts+++ b/apps/desktop/src/preload/regionScreenshotPreload.ts@@ -3,6 +3,7 @@ import { contextBridge, ipcRenderer } from 'electron';import {REGION_SCREENSHOT_CHANNEL as C,type RegionRect,+ type RegionScreenshotHit,type RegionScreenshotBegin,type RegionScreenshotExport,type RegionScreenshotBitmap,@@ -36,7 +37,7 @@ contextBridge.exposeInMainWorld('regionScreenshot', {copyText: (token: string, text: string): void => ipcRenderer.send(C.COPY_TEXT, token, text),cancel: (token: string): void => ipcRenderer.send(C.CANCEL, token),abandon: (): void => ipcRenderer.send(C.ABANDON),- elementsAt: (token: string, windowId: number, x: number, y: number): Promise<RegionRect[]> =>+ elementsAt: (token: string, windowId: number, x: number, y: number): Promise<RegionScreenshotHit | null> =>ipcRenderer.invoke(C.ELEMENTS_AT, token, windowId, x, y),exportImage: (token: string, payload: RegionScreenshotExport): Promise<boolean> =>ipcRenderer.invoke(C.EXPORT, token, payload),diff --git a/apps/desktop/src/renderer/features/region-screenshot/RegionScreenshotOverlay.tsx b/apps/desktop/src/renderer/features/region-screenshot/RegionScreenshotOverlay.tsxindex 1ce07a4208..62d435353a 100644--- a/apps/desktop/src/renderer/features/region-screenshot/RegionScreenshotOverlay.tsx+++ b/apps/desktop/src/renderer/features/region-screenshot/RegionScreenshotOverlay.tsx@@ -36,6 +36,8 @@ import {type RegionScreenshotBegin,type RegionScreenshotBitmap,type RegionScreenshotWindow,+ type RegionScreenshotHit,+ intersectRegion,type RegionSize,} from '../../../shared/regionScreenshot';import {@@ -180,7 +182,7 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridgconst longResultRef = useRef(longResult);longResultRef.current = longResult;const [selection, setSelection] = useState<RegionRect | null>(null);- const [hover, setHover] = useState<{ candidates: RegionRect[]; level: number } | null>(null);+ const [hover, setHover] = useState<{ candidates: RegionRect[]; level: number; defaultRect: RegionRect } | null>(null);const [pointer, setPointer] = useState<RegionPoint | null>(null);const [drag, setDrag] = useState<Drag | null>(null);const [tool, setTool] = useState<AnnotationTool | null>(null);@@ -239,8 +241,9 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridgconst elementsRef = useRef<{inFlight: boolean;pending: RegionPoint | null;- cache: { windowId: number; rects: RegionRect[] } | null;- }>({ inFlight: false, pending: null, cache: null });+ cache: { point: RegionPoint; hit: RegionScreenshotHit } | null;+ defaults: Map<number, RegionRect[]>;+ }>({ inFlight: false, pending: null, cache: null, defaults: new Map() });const viewport = session?.viewport ?? { width: window.innerWidth, height: window.innerHeight };const safe = session?.safeArea ?? { x: 0, y: 0, width: viewport.width, height: viewport.height };@@ -272,7 +275,7 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridgsetOverToolbar(false);busyRef.current = false;wheelRef.current = 0;- elementsRef.current = { inFlight: false, pending: null, cache: null };+ elementsRef.current = { inFlight: false, pending: null, cache: null, defaults: new Map() };}, []);useEffect(() => {@@ -470,13 +473,44 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridgconst refreshHover = useCallback((point: RegionPoint) => {const current = state.current.session;if (!current) return;- const window = topWindowAt(current.windows, point);const cache = elementsRef.current.cache;- const elements = cache && window && cache.windowId === window.id ? cache.rects : [];+ // A parent rectangle containing a new point does not prove that its old+ // element chain (or the window under a transparent pixel) is still valid.+ const hit = cache && cache.point.x === point.x && cache.point.y === point.y ? cache.hit : null;+ const rectangularWindow = topWindowAt(current.windows, point);+ // Keep a corrected target (including an explicit null for an unknown+ // occluder) while this point is pending under the same captured front window.+ // This is only a display fallback: requestElements still queries the new+ // point, and its reply is authoritative. Leaving a known corrected target's+ // bounds or entering a different front window ends retention immediately.+ const pendingHit = !hit && cache && rectangularWindow+ && cache.hit.window?.id !== rectangularWindow.id+ && topWindowAt(current.windows, cache.point)?.id === rectangularWindow.id+ && (!cache.hit.window || rectContains(cache.hit.window.rect, point)) ? cache.hit : null;+ const window = hit ? hit.window : pendingHit ? pendingHit.window : rectangularWindow;+ const snapshot = current.windows.find((candidate) => candidate.id === window?.id);+ // Default regions belong to the frozen capture, not a point query. UIA may+ // finish later or fail without making the outline bounce back to the window.+ // Older helpers/macOS can retain a known content region while a new query is+ // pending, but never reuse its fine element chain at a different point.+ const remembered = window && elementsRef.current.defaults.get(window.id)?.find((rect) => rectContains(rect, point));+ const native = snapshot?.snapRects?.find((rect) => rectContains(rect, point));+ const preferred = native && window && !sameRegion(native, window.rect) ? native+ : remembered ?? hit?.preferredRect+ ?? (snapshot?.snapRects ? window?.rect : undefined);+ const elements = [...(hit?.rects ?? []), ...(snapshot?.snapRects ?? []), ...(preferred ? [preferred] : [])]+ .sort((a, b) => a.width * a.height - b.width * b.height);const candidates = snapCandidates(point, window, elements, current.viewport);+ const clipped = preferred && intersectRegion(preferred, current.viewport);+ const defaultLevel = clipped ? Math.max(0, candidates.findIndex((rect) => sameRegion(rect, clipped))) : 0;+ const defaultRect = candidates[defaultLevel];setHover((previous) => {- const keep = previous && previous.candidates[0] && sameRegion(previous.candidates[0], candidates[0]);- return { candidates, level: keep ? Math.min(previous.level, candidates.length - 1) : 0 };+ const selected = previous && sameRegion(previous.defaultRect, defaultRect)+ ? candidates.findIndex((rect) => sameRegion(rect, previous.candidates[previous.level])) : -1;+ const level = selected >= 0 ? selected : defaultLevel;+ const keep = previous && previous.level === level && previous.candidates.length === candidates.length+ && previous.candidates.every((rect, index) => sameRegion(rect, candidates[index]));+ return keep && sameRegion(previous.defaultRect, defaultRect) ? previous : { candidates, level, defaultRect };});}, []);@@ -494,18 +528,29 @@ export function RegionScreenshotOverlay({ bridge, imageEditor = false }: { bridgtracker.inFlight = true;void bridge.elementsAt(current.token, window.id, point.x, point.y)- .then((rects) => {- if (state.current.session?.token !== current.token) return;- tracker.cache = { windowId: window.id, rects: Array.isArray(rects) ? rects : [] };+ .then((hit) => {+ if (state.current.session?.token !== current.token || elementsRef.current !== tracker || !hit) return;const latest = state.current.pointer;- if (latest && !state.current.selection && !state.current.drag) refreshHover(latest);+ if (latest && latest.x === point.x && latest.y === point.y && !state.current.selection && !state.current.drag) {+ // Some browsers have no content HWND. Learn their document regions+ // once per capture; an empty/partial UIA reply must not erase them.+ if (hit.window && hit.preferredRect && !sameRegion(hit.preferredRect, hit.window.rect)) {+ const regions = tracker.defaults.get(hit.window.id) ?? [];+ if (regions.length < 256 && !regions.some((rect) => sameRegion(rect, hit.preferredRect!))) {+ tracker.defaults.set(hit.window.id, [...regions, hit.preferredRect]+ .sort((a, b) => a.width * a.height - b.width * b.height));+ }+ }+ tracker.cache = { point, hit };+ refreshHover(latest);+ }}).catch(() => undefined).finally(() => {tracker.inFlight = false;const pending = tracker.pending;tracker.pending = null;- if (pending && state.current.session?.token === current.token && !state.current.selection) {+ if (pending && elementsRef.current === tracker && state.current.session?.token === current.token && !state.current.selection) {requestElements(pending);}});diff --git a/apps/desktop/src/renderer/features/region-screenshot/__tests__/screenshotRendering.test.tsx b/apps/desktop/src/renderer/features/region-screenshot/__tests__/screenshotRendering.test.tsxindex 3f422bcee5..7868bd5820 100644--- a/apps/desktop/src/renderer/features/region-screenshot/__tests__/screenshotRendering.test.tsx+++ b/apps/desktop/src/renderer/features/region-screenshot/__tests__/screenshotRendering.test.tsx@@ -3,13 +3,20 @@ import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'import { afterEach, describe, expect, it, vi } from 'vitest';import { ScreenshotErrorBoundary } from '../ScreenshotErrorBoundary';import { Magnifier } from '../Magnifier';-import { bitmapToImageData, imageDataToBitmap } from '../bridge';-import type { RegionScreenshotOverlayBridge } from '../bridge';+import { bitmapToImageData, imageDataToBitmap, type RegionScreenshotOverlayBridge } from '../bridge';+import { RegionScreenshotOverlay } from '../RegionScreenshotOverlay';+import type { RegionScreenshotHit } from '../../../../shared/regionScreenshot';import type { RegionCaptureFrame } from '../../../../shared/regionCapture';import { ScrollCaptureOverlay } from '../ScrollCaptureOverlay';vi.mock('@/lib/logger', () => ({ createLogger: () => ({ warn: vi.fn() }) }));vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) }));+vi.mock('@/i18n', () => ({ default: { language: 'en' } }));+vi.mock('@/hooks/useTheme', () => ({ getInitialThemeVariant: () => ({ theme: 'default' }) }));+vi.mock('@/themes/theme-service', () => ({ themeService: { applyTheme: vi.fn() } }));+vi.mock('../ScreenshotToolbar', () => ({ TOOL_SHORTCUTS: {}, ScreenshotModeBar: () => null, ScreenshotStyleBar: () => null, ScreenshotToolbar: () => null }));+vi.mock('../LiveCapturePanel', () => ({ LiveCapturePanel: () => null }));+vi.mock('../ScreenshotEditorViewport', () => ({ ScreenshotEditorViewport: () => null }));// 测试价值:生产 React 边界出错会释放浮层,放大镜坐标与屏幕像素一致。// 纯几何测试没有组件/错误路径;最小输入为 render throw、2x 屏配 3x 定格图。@@ -21,6 +28,177 @@ describe('screenshot rendering recovery', () => {vi.restoreAllMocks();vi.unstubAllGlobals();});+ // Production overlay regression: moving within the same window must not+ // reuse the old parent's chain or apply a delayed hit from another point.+ // Review regression (2026-10-28): a captured click-through front window must+ // not flash back while a corrected hit is pending. Previously this test+ // expected that flash. Overlapping captured windows + deferred replies, tier: unit.+ // Standalone file: 1.52s on XD-PC (2026-10-28).+ it('keeps point-specific hits current and selects the corrected target', async () => {+ vi.stubGlobal('PointerEvent', MouseEvent);+ vi.stubGlobal('ImageData', class {+ data: Uint8ClampedArray;+ constructor(readonly width: number, readonly height: number) { this.data = new Uint8ClampedArray(width * height * 4); }+ });+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null);+ let begin!: Parameters<RegionScreenshotOverlayBridge['onBegin']>[0];+ const replies: ((hit: RegionScreenshotHit) => void)[] = [];+ const bridge = {+ onBegin: (callback: typeof begin) => { begin = callback; return () => {}; },+ onEnd: () => () => {}, onClear: () => () => {},+ shellReady() {}, abandon() {}, painted() {}, focused() {}, selectionStarted() {},+ elementsAt: vi.fn(() => new Promise<RegionScreenshotHit>((resolve) => replies.push(resolve))),+ } as unknown as RegionScreenshotOverlayBridge;+ const { container } = render(<RegionScreenshotOverlay bridge={bridge} />);+ const root = container.firstElementChild!;+ const outer = { id: 0, rect: { x: 0, y: 0, width: 500, height: 400 } };+ const corrected = { id: 1, rect: { x: 20, y: 20, width: 300, height: 300 } };+ const front = { id: 2, rect: { x: 200, y: 100, width: 50, height: 40 } };+ await act(async () => begin({ token: 'one', locale: 'en', viewport: { width: 600, height: 500 }, safeArea: { x: 0, y: 0, width: 600, height: 500 }, scaleFactor: 1, frame: { width: 1, height: 1, data: new Uint8Array(4) }, windows: [front, outer, corrected], cursor: { x: 10, y: 10 }, lastRegion: null }));+ fireEvent.pointerMove(root, { clientX: 40, clientY: 40 });+ await act(async () => replies[0]({ window: outer, rects: [{ x: 0, y: 0, width: 200, height: 200 }] }));+ expect(screen.getByText('500 × 400')).toBeTruthy();+ expect(bridge.elementsAt).toHaveBeenLastCalledWith('one', 0, 40, 40);+ const content = { x: 25, y: 25, width: 250, height: 200 };+ // 2026-10-28 browser regression: default to the document, not its button.+ // The actual renderer must still let a wheel gesture reach finer elements.+ await act(async () => replies[1]({ window: corrected, rects: [{ x: 30, y: 30, width: 80, height: 60 }, content], preferredRect: content }));+ expect(screen.getByText('250 × 200')).toBeTruthy();+ fireEvent.wheel(root, { deltaY: 120 });+ expect(screen.getByText('80 × 60')).toBeTruthy();+ // Keep the default, not the old fine element, during a new point query.+ fireEvent.pointerMove(root, { clientX: 150, clientY: 150 });+ expect(screen.getByText('250 × 200')).toBeTruthy();+ await act(async () => replies[2]({ window: corrected, rects: [] }));+ expect(screen.getByText('250 × 200')).toBeTruthy();+ // An opaque pixel in the same front window must still win once confirmed.+ fireEvent.pointerMove(root, { clientX: 160, clientY: 150 });+ expect(screen.getByText('250 × 200')).toBeTruthy();+ await act(async () => replies[3]({ window: outer, rects: [] }));+ expect(screen.getByText('500 × 400')).toBeTruthy();+ fireEvent.pointerMove(root, { clientX: 180, clientY: 150 });+ await act(async () => replies[4]({ window: corrected, rects: [], preferredRect: content }));+ expect(screen.getByText('250 × 200')).toBeTruthy();+ // Leaving the corrected window stops retaining it even before a reply.+ fireEvent.pointerMove(root, { clientX: 450, clientY: 350 });+ expect(screen.getByText('500 × 400')).toBeTruthy();+ fireEvent.pointerMove(root, { clientX: 220, clientY: 120 });+ expect(screen.getByText('50 × 40')).toBeTruthy(); // a different front window wins immediately+ fireEvent.pointerMove(root, { clientX: 200, clientY: 150 });+ expect(screen.getByText('250 × 200')).toBeTruthy();+ await act(async () => replies[5]({ window: outer, rects: [] }));+ expect(screen.getByText('250 × 200')).toBeTruthy(); // stale opaque hit+ await act(async () => replies[6]({ window: corrected, rects: [] }));+ fireEvent.keyDown(window, { key: 'Enter' });+ fireEvent.pointerMove(root, { clientX: 450, clientY: 350 });+ expect(screen.getByText('250 × 200')).toBeTruthy();+ expect(bridge.elementsAt).toHaveBeenCalledTimes(7);+ });+ // PR #7661 discussion_r4398847280: a native null window must not flash the+ // captured window underneath an unknown occluder at each new pointer position.+ // Existing corrected-target coverage only used non-null windows. Production+ // overlay + deferred null/recovery replies is the minimal input; tier: unit.+ it('retains a confirmed unknown occluder while a new point is pending', async () => {+ vi.stubGlobal('PointerEvent', MouseEvent);+ vi.stubGlobal('ImageData', class {+ data = new Uint8ClampedArray(4);+ constructor(readonly width: number, readonly height: number) {}+ });+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null);+ let begin!: Parameters<RegionScreenshotOverlayBridge['onBegin']>[0];+ const replies: ((hit: RegionScreenshotHit) => void)[] = [];+ const bridge = {+ onBegin: (callback: typeof begin) => { begin = callback; return () => {}; },+ onEnd: () => () => {}, onClear: () => () => {},+ shellReady() {}, abandon() {}, painted() {}, focused() {}, selectionStarted() {},+ elementsAt: vi.fn(() => new Promise<RegionScreenshotHit>((resolve) => replies.push(resolve))),+ } as unknown as RegionScreenshotOverlayBridge;+ const outer = { id: 0, rect: { x: 0, y: 0, width: 500, height: 400 } };+ const front = { id: 1, rect: { x: 400, y: 100, width: 100, height: 100 } };+ const { container } = render(<RegionScreenshotOverlay bridge={bridge} />);+ const root = container.firstElementChild!;+ await act(async () => begin({ token: 'occluded', locale: 'en', viewport: { width: 600, height: 500 }, safeArea: { x: 0, y: 0, width: 600, height: 500 }, scaleFactor: 1, frame: { width: 1, height: 1, data: new Uint8Array(4) }, windows: [front, outer], cursor: { x: 50, y: 80 }, lastRegion: null }));+ await act(async () => replies[0]({ window: null, rects: [] }));+ expect(screen.getByText('600 × 500')).toBeTruthy();+ for (const x of [100, 200]) {+ fireEvent.pointerMove(root, { clientX: x, clientY: 80 });+ expect(screen.getByText('600 × 500')).toBeTruthy();+ }+ // An obsolete non-null hit cannot select the occluded window either.+ await act(async () => replies[1]({ window: outer, rects: [] }));+ expect(screen.getByText('600 × 500')).toBeTruthy();+ expect(bridge.elementsAt).toHaveBeenLastCalledWith('occluded', 0, 200, 80);+ await act(async () => replies[2]({ window: null, rects: [] }));+ fireEvent.pointerMove(root, { clientX: 250, clientY: 80 });+ expect(screen.getByText('600 × 500')).toBeTruthy();+ // A fresh hit restores the known window as soon as the occluder is left.+ await act(async () => replies[3]({ window: outer, rects: [] }));+ expect(screen.getByText('500 × 400')).toBeTruthy();+ fireEvent.pointerMove(root, { clientX: 300, clientY: 80 });+ await act(async () => replies[4]({ window: null, rects: [] }));+ fireEvent.pointerMove(root, { clientX: 450, clientY: 150 });+ expect(screen.getByText('100 × 100')).toBeTruthy(); // a different captured front ends retention+ await act(async () => replies[5]({ window: front, rects: [] }));+ expect(bridge.elementsAt).toHaveBeenCalledTimes(6);+ });+ // 2026-10-28 flicker: final-result assertions missed the whole-window frame+ // inserted during every pending query. Production overlay, two moves before a+ // deferred reply (unit); also cover old helpers without captured child regions.+ // Standalone file: 1.64s on XD-PC (2026-10-28).+ it.each([true, false])('keeps the default outline stable during pending queries (snapshot=%s)', async (snapshot) => {+ vi.stubGlobal('PointerEvent', MouseEvent);+ vi.stubGlobal('ImageData', class {+ data = new Uint8ClampedArray(4);+ constructor(readonly width: number, readonly height: number) {}+ });+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null);+ let begin!: Parameters<RegionScreenshotOverlayBridge['onBegin']>[0];+ const replies: ((hit: RegionScreenshotHit) => void)[] = [];+ const bridge = {+ onBegin: (callback: typeof begin) => { begin = callback; return () => {}; },+ onEnd: () => () => {}, onClear: () => () => {},+ shellReady() {}, abandon() {}, painted() {}, focused() {}, selectionStarted() {},+ elementsAt: vi.fn(() => new Promise<RegionScreenshotHit>((resolve) => replies.push(resolve))),+ } as unknown as RegionScreenshotOverlayBridge;+ const content = { x: 0, y: 50, width: 500, height: 350 };+ const outer = { id: 0, rect: { x: 0, y: 0, width: 500, height: 400 }, ...(snapshot ? { snapRects: [content] } : {}) };+ const front = { id: 1, rect: { x: 400, y: 100, width: 100, height: 100 }, snapRects: [] };+ const { container } = render(<RegionScreenshotOverlay bridge={bridge} />);+ const root = container.firstElementChild!;+ const payload = { token: 'stable', locale: 'en', viewport: { width: 600, height: 500 }, safeArea: { x: 0, y: 0, width: 600, height: 500 }, scaleFactor: 1, frame: { width: 1, height: 1, data: new Uint8Array(4) }, windows: [front, outer], cursor: { x: 50, y: 80 }, lastRegion: null };+ await act(async () => begin(payload));+ if (snapshot) expect(screen.getByText('500 × 350')).toBeTruthy(); // before any native reply+ await act(async () => replies[0]({ window: outer, rects: [], preferredRect: content }));+ for (const x of [100, 200]) {+ fireEvent.pointerMove(root, { clientX: x, clientY: 80 });+ expect(screen.getByText('500 × 350')).toBeTruthy();+ }+ // A late result for x=100 must not change even the remembered default.+ await act(async () => replies[1]({ window: outer, rects: [], preferredRect: outer.rect }));+ fireEvent.pointerMove(root, { clientX: 250, clientY: 80 });+ expect(screen.getByText('500 × 350')).toBeTruthy();+ await act(async () => replies[2]({ window: outer, rects: [], preferredRect: outer.rect }));+ await act(async () => replies[3]({ window: outer, rects: [{ x: 230, y: 60, width: 80, height: 60 }], preferredRect: content }));+ expect(screen.getByText('500 × 350')).toBeTruthy();+ fireEvent.wheel(root, { deltaY: 120 });+ expect(screen.getByText('80 × 60')).toBeTruthy();+ fireEvent.pointerMove(root, { clientX: 250, clientY: 81 });+ await act(async () => replies[4]({ window: outer, rects: [], preferredRect: outer.rect }));+ expect(screen.getByText('500 × 350')).toBeTruthy(); // partial UIA does not erase known content+ fireEvent.pointerMove(root, { clientX: 250, clientY: 20 });+ expect(screen.getByText('500 × 400')).toBeTruthy(); // leave content immediately+ fireEvent.pointerMove(root, { clientX: 450, clientY: 150 });+ expect(screen.getByText('100 × 100')).toBeTruthy(); // top window occludes cached content+ fireEvent.pointerMove(root, { clientX: 250, clientY: 80 });+ expect(screen.getByText('500 × 350')).toBeTruthy();+ fireEvent.keyDown(window, { key: 'Enter' });+ fireEvent.pointerMove(root, { clientX: 550, clientY: 450 });+ expect(screen.getByText('500 × 350')).toBeTruthy();+ // A new capture cannot inherit a region or a late reply from this capture.+ await act(async () => begin({ ...payload, token: 'next', windows: [{ ...outer, snapRects: [] }] }));+ await act(async () => replies[5]({ window: outer, rects: [], preferredRect: content }));+ expect(screen.getByText('500 × 400')).toBeTruthy();+ });// 用户反馈点击自动滚动立刻提示已停止。已有渲染测试未覆盖开关与在途帧的竞态。// 最小输入:开启前发出的帧在开启成功后返回 false;下一帧的真实停止仍须显示。tier: unit。it('ignores a pre-start frame but reports a fresh automatic scrolling failure', async () => {diff --git a/apps/desktop/src/renderer/features/region-screenshot/bridge.ts b/apps/desktop/src/renderer/features/region-screenshot/bridge.tsindex 38b30b1d6d..a06dc525a3 100644--- a/apps/desktop/src/renderer/features/region-screenshot/bridge.ts+++ b/apps/desktop/src/renderer/features/region-screenshot/bridge.ts@@ -1,5 +1,6 @@import type {RegionRect,+ RegionScreenshotHit,RegionScreenshotBegin,RegionScreenshotBitmap,RegionScreenshotExport,@@ -30,7 +31,7 @@ export interface RegionScreenshotOverlayBridge {copyText(token: string, text: string): void;cancel(token: string): void;abandon(): void;- elementsAt(token: string, windowId: number, x: number, y: number): Promise<RegionRect[]>;+ elementsAt(token: string, windowId: number, x: number, y: number): Promise<RegionScreenshotHit | null>;exportImage(token: string, payload: RegionScreenshotExport): Promise<boolean>;}diff --git a/apps/desktop/src/renderer/region-screenshot-entry.tsx b/apps/desktop/src/renderer/region-screenshot-entry.tsxindex 8de78965ae..0e056587fb 100644--- a/apps/desktop/src/renderer/region-screenshot-entry.tsx+++ b/apps/desktop/src/renderer/region-screenshot-entry.tsx@@ -38,7 +38,7 @@ const bridge: RegionScreenshotOverlayBridge | undefined = editor ? {abandon: () => { if (editorToken) editor.cancel(editorToken); },// Rendering adapters only: none of these unused capture capabilities cross preload.onClear: () => () => {}, focused: () => {}, selectionStarted: () => {}, copyText: () => {},- elementsAt: async () => [], autoScroll: async () => false, restoreEditor: async () => false,+ elementsAt: async () => null, autoScroll: async () => false, restoreEditor: async () => false,holdCapture: async () => {}, startLive: async () => null, liveFrame: async () => null, saveRecording: async () => false,recordingControls: async () => null, previewRecording: async () => false,} : window.regionScreenshot;diff --git a/apps/desktop/src/shared/regionScreenshot.ts b/apps/desktop/src/shared/regionScreenshot.tsindex 32e04f82a8..90a301ae8f 100644--- a/apps/desktop/src/shared/regionScreenshot.ts+++ b/apps/desktop/src/shared/regionScreenshot.ts@@ -23,7 +23,7 @@ export const REGION_SCREENSHOT_CHANNEL = {CLEAR: 'region-screenshot:clear',/** main → 浮层: 会话结束, 清空画面与全部交互态后隐藏。 */END: 'region-screenshot:end',- /** invoke(token, windowId, x, y): 指针下的界面元素, 由内到外 (DIP)。 */+ /** invoke(token, windowId, x, y): 实际命中窗口与由内到外的元素 (DIP)。 */ELEMENTS_AT: 'region-screenshot:elements-at',/** send(token, text): 复制取到的颜色值。 */COPY_TEXT: 'region-screenshot:copy-text',@@ -138,6 +138,16 @@ export interface RegionScreenshotBitmap {export interface RegionScreenshotWindow {id: number;rect: RegionRect;+ /** Captured native regions in hit order; absent on older helpers/platforms. */+ snapRects?: RegionRect[];+}++/** A point query can correct the rectangular window guess through transparent overlays. */+export interface RegionScreenshotHit {+ window: RegionScreenshotWindow | null;+ rects: RegionRect[];+ /** Default document/content region; finer elements remain available via wheel. */+ preferredRect?: RegionRect;}export interface RegionScreenshotBegin {diff --git a/docs/dev-rules/electron-security-and-process-boundaries.md b/docs/dev-rules/electron-security-and-process-boundaries.mdindex d116e6ec21..36cdfc428b 100644--- a/docs/dev-rules/electron-security-and-process-boundaries.md+++ b/docs/dev-rules/electron-security-and-process-boundaries.md@@ -138,6 +138,34 @@ macOS 浮层是不激活应用的面板(`type: 'panel'`):拿键盘焦点应用保持前台,菜单栏不切换,滚动截图时触控板滚动与惯性照常交给它;不得改回`app.focus({ steal: true })`。+截图自动选区只查询几何信息。macOS 按原生命中规则跳过透明位置与忽略鼠标的窗口,+只显式排除本次截图浮层;命中未收入截图快照的窗口时不继续向下穿透。辅助功能查询+优先命中目标应用并沿父级取区域,失败时在有界预算内回退到目标窗口树。+本机 `ELEMENTS_AT` 返回纠正后的窗口与元素;主进程保留 sender/顶层 frame/token 和+显示器内坐标校验,只接受快照中的目标。精细元素缓存绑定查询位置,迟到结果不覆盖新位置或默认区域缓存。+此接口与原生 helper 同包更新,不进入 device-link allowlist;旧 helper 缺少命中查询时+沿用窗口矩形吸附。回归见 `snapshotHost.test.ts`、`windows.test.ts` 与 `screenshotRendering.test.tsx`。+首次悬停优先网页/文档内容区,浏览器工具栏与标签栏回到整窗,内部元素保留供滚轮切换。+Windows 在截图准备阶段保存原生子窗口区域(每窗最多 256 个、最多 40 层,按原生层级/z 序,+裁到祖先边界),随窗口列表可选 `snapRects` 下发;main 校验并转换为显示器局部 DIP。+移动时从快照同步命中默认区域,异步元素查询只补滚轮候选,不在等待时退回整窗;+相同默认区域保留仍有效的滚轮层级,相同候选和层级不重复更新。旧 helper/macOS 没有此字段,+或 Chromium 没有原生内容子窗口时,将有效的内容区按窗口补入本次截图的默认区域缓存(每窗最多 256 个)。+等待查询和空/不完整回复不清除仍包含指针的已知内容区;退出/重新截图即清空。+不复用旧位置的精细元素链。若原生命中曾纠正透明/鼠标穿透前窗,新位置仍在纠正窗口内、+且矩形命中的前窗未变,等待查询期间沿用已确认目标的默认区域显示;此显示保留不替代新位置的+原生命中,当前回复确认不透明前窗后立即切换。离开纠正窗口或进入另一前窗时立即按新边界显示;+未知遮挡物的显式空命中也须保留,不能在同一快照前窗内移动时因新坐标尚未回复而选中底层窗口;+新位置的有效回复恢复已知窗口,进入另一快照前窗则结束保留。+双窗口/未知遮挡中间帧、迟到回复和不透明区域切换回归见 `screenshotRendering.test.tsx`。+Windows 同时在有界预算内遍历重叠的 UIA 分支,不能因首个同尺寸+空容器而漏掉正文;macOS 沿 AXWebArea/滚动容器获取内容区。helper 可选返回 `preferredRect`,+主进程验证并转换到显示器局部 DIP;旧 helper 省略时保持原有元素吸附。+Windows 真实 Chrome+原生 helper+生产浮层的黑盒回归入口为+`node apps/desktop/scripts/check-region-snapping.mjs`,副屏可加 `--position=2700,100`;+记录 Light/Dark 连续移动时的 DOM 提交与绘制帧,并延后真实元素响应验证过程中无整窗闪回及跨边界切换;+只适配测试传输,不替代生产 IPC 授权用例或 macOS 实机验证。+滚动截图与区域录制复用该浮层和会话 token,开始后把控制区移到选区之外。滚动截图在原透明浮层显示选区轮廓与侧边预览,只在预览和自动滚动按钮处接收鼠标。长图编辑交给独立无标题栏窗口,复用辅助窗口 controller,原浮层结束;首份图片绘制后才显示。