π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
fix(capture-native): name the smallest element under the pointer
Luis Ortiz5ccca3619958
diff --git a/packages/capture-native/src/hit.rs b/packages/capture-native/src/hit.rsnew file mode 100644index 0000000..b22e1f6--- /dev/null+++ b/packages/capture-native/src/hit.rs@@ -0,0 +1,42 @@+pub type Bounds = (i32, i32, i32, i32);++pub fn smallest_under(boxes: &[Bounds], x: i32, y: i32) -> Option<usize> {+ boxes+ .iter()+ .enumerate()+ .filter(|(_, &(left, top, right, bottom))| {+ right > left && bottom > top && (left..right).contains(&x) && (top..bottom).contains(&y)+ })+ .min_by_key(|(_, &(left, top, right, bottom))| i64::from(right - left) * i64::from(bottom - top))+ .map(|(index, _)| index)+}++#[cfg(test)]+mod tests {+ use super::smallest_under;++ const PANEL: (i32, i32, i32, i32) = (0, 0, 400, 600);+ const ITEM: (i32, i32, i32, i32) = (10, 50, 390, 90);+ const ICON: (i32, i32, i32, i32) = (12, 55, 40, 85);++ #[test]+ fn takes_the_smallest_box_under_the_point() {+ assert_eq!(smallest_under(&[PANEL, ITEM, ICON], 100, 70), Some(1));+ assert_eq!(smallest_under(&[PANEL, ITEM, ICON], 20, 70), Some(2));+ }++ #[test]+ fn ignores_empty_boxes_and_boxes_elsewhere() {+ assert_eq!(+ smallest_under(&[PANEL, (100, 70, 100, 70), (0, 0, 0, 0)], 100, 70),+ Some(0)+ );+ assert_eq!(smallest_under(&[PANEL, ITEM], 100, 700), None);+ assert_eq!(smallest_under(&[ITEM], 390, 70), None);+ }++ #[test]+ fn keeps_the_outer_element_when_sizes_tie() {+ assert_eq!(smallest_under(&[ITEM, ITEM], 100, 70), Some(0));+ }+}diff --git a/packages/capture-native/src/lib.rs b/packages/capture-native/src/lib.rsindex 5c0d995..e32ddb2 100644--- a/packages/capture-native/src/lib.rs+++ b/packages/capture-native/src/lib.rs@@ -4,6 +4,8 @@ use napi::bindgen_prelude::AsyncTask;use napi::{Env, Result, Task};use napi_derive::napi;+#[cfg(any(windows, test))]+mod hit;#[cfg(windows)]mod win;diff --git a/packages/capture-native/src/win.rs b/packages/capture-native/src/win.rsindex b4d8fc3..aa28eef 100644--- a/packages/capture-native/src/win.rs+++ b/packages/capture-native/src/win.rs@@ -14,6 +14,7 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{};use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};+use crate::hit::smallest_under;use crate::{ElementNode, ElementRect, UiElement};const ROLES: &[(UIA_CONTROLTYPE_ID, &str)] = &[@@ -169,15 +170,49 @@ fn children(walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement) ->out}+fn narrowest(+ uia: &IUIAutomation,+ walker: &IUIAutomationTreeWalker,+ hit: &IUIAutomationElement,+ point: POINT,+) -> Option<IUIAutomationElement> {+ if unsafe { hit.CurrentControlType() }.ok()? == UIA_EditControlTypeId {+ return None;+ }+ unsafe { walker.GetFirstChildElement(hit) }.ok()?;+ let request = unsafe { uia.CreateCacheRequest() }.ok()?;+ unsafe { request.AddProperty(UIA_BoundingRectanglePropertyId) }.ok()?;+ let everything = unsafe { uia.CreateTrueCondition() }.ok()?;+ let found = unsafe { hit.FindAllBuildCache(TreeScope_Subtree, &everything, &request) }.ok()?;+ let elements: Vec<IUIAutomationElement> = (0..unsafe { found.Length() }.ok()?)+ .filter_map(|index| unsafe { found.GetElement(index) }.ok())+ .collect();+ let boxes: Vec<_> = elements+ .iter()+ .map(|element| {+ unsafe { element.CachedBoundingRectangle() }+ .map(|rect| (rect.left, rect.top, rect.right, rect.bottom))+ .unwrap_or_default()+ })+ .collect();+ smallest_under(&boxes, point.x, point.y).map(|index| elements[index].clone())+}+pub fn element_at_point(x: i32, y: i32) -> Result<Option<UiElement>> {let uia = automation().map_err(|error| napi::Error::from_reason(error.message()))?;- let found = unsafe { uia.ElementFromPoint(POINT { x, y }) }- .map_err(|error| napi::Error::from_reason(error.message()))?;+ let point = POINT { x, y };+ let hit =+ unsafe { uia.ElementFromPoint(point) }.map_err(|error| napi::Error::from_reason(error.message()))?;+ let walker = unsafe { uia.ControlViewWalker() }.ok();+ let found = walker+ .as_ref()+ .and_then(|walker| narrowest(&uia, walker, &hit, point))+ .unwrap_or(hit);let mut element = describe(&found);if element.name.is_none() {- if let Ok(walker) = unsafe { uia.ControlViewWalker() } {- element.ancestors = ancestors(&walker, &found);- element.children = children(&walker, &found);+ if let Some(walker) = &walker {+ element.ancestors = ancestors(walker, &found);+ element.children = children(walker, &found);}}Ok(Some(element))