Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix(capture-native): name the smallest element under the pointer

Luis Ortiz5ccca3619958
diff --git a/packages/capture-native/src/hit.rs b/packages/capture-native/src/hit.rs
new file mode 100644
index 0000000..b22e1f6
--- /dev/null
+++ b/packages/capture-native/src/hit.rs
@@ -0,0 +1,42 @@
+pub type Bounds = (i32, i32, i32, i32);
+
+pub fn smallest_under(boxes: &[Bounds], x: i32, y: i32) -> Option<usize> {
+ boxes
+ .iter()
+ .enumerate()
+ .filter(|(_, &(left, top, right, bottom))| {
+ right > left && bottom > top && (left..right).contains(&x) && (top..bottom).contains(&y)
+ })
+ .min_by_key(|(_, &(left, top, right, bottom))| i64::from(right - left) * i64::from(bottom - top))
+ .map(|(index, _)| index)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::smallest_under;
+
+ const PANEL: (i32, i32, i32, i32) = (0, 0, 400, 600);
+ const ITEM: (i32, i32, i32, i32) = (10, 50, 390, 90);
+ const ICON: (i32, i32, i32, i32) = (12, 55, 40, 85);
+
+ #[test]
+ fn takes_the_smallest_box_under_the_point() {
+ assert_eq!(smallest_under(&[PANEL, ITEM, ICON], 100, 70), Some(1));
+ assert_eq!(smallest_under(&[PANEL, ITEM, ICON], 20, 70), Some(2));
+ }
+
+ #[test]
+ fn ignores_empty_boxes_and_boxes_elsewhere() {
+ assert_eq!(
+ smallest_under(&[PANEL, (100, 70, 100, 70), (0, 0, 0, 0)], 100, 70),
+ Some(0)
+ );
+ assert_eq!(smallest_under(&[PANEL, ITEM], 100, 700), None);
+ assert_eq!(smallest_under(&[ITEM], 390, 70), None);
+ }
+
+ #[test]
+ fn keeps_the_outer_element_when_sizes_tie() {
+ assert_eq!(smallest_under(&[ITEM, ITEM], 100, 70), Some(0));
+ }
+}
diff --git a/packages/capture-native/src/lib.rs b/packages/capture-native/src/lib.rs
index 5c0d995..e32ddb2 100644
--- a/packages/capture-native/src/lib.rs
+++ b/packages/capture-native/src/lib.rs
@@ -4,6 +4,8 @@ use napi::bindgen_prelude::AsyncTask;
use napi::{Env, Result, Task};
use napi_derive::napi;
+#[cfg(any(windows, test))]
+mod hit;
#[cfg(windows)]
mod win;
diff --git a/packages/capture-native/src/win.rs b/packages/capture-native/src/win.rs
index b4d8fc3..aa28eef 100644
--- a/packages/capture-native/src/win.rs
+++ b/packages/capture-native/src/win.rs
@@ -14,6 +14,7 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{
};
use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};
+use crate::hit::smallest_under;
use crate::{ElementNode, ElementRect, UiElement};
const ROLES: &[(UIA_CONTROLTYPE_ID, &str)] = &[
@@ -169,15 +170,49 @@ fn children(walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement) ->
out
}
+fn narrowest(
+ uia: &IUIAutomation,
+ walker: &IUIAutomationTreeWalker,
+ hit: &IUIAutomationElement,
+ point: POINT,
+) -> Option<IUIAutomationElement> {
+ if unsafe { hit.CurrentControlType() }.ok()? == UIA_EditControlTypeId {
+ return None;
+ }
+ unsafe { walker.GetFirstChildElement(hit) }.ok()?;
+ let request = unsafe { uia.CreateCacheRequest() }.ok()?;
+ unsafe { request.AddProperty(UIA_BoundingRectanglePropertyId) }.ok()?;
+ let everything = unsafe { uia.CreateTrueCondition() }.ok()?;
+ let found = unsafe { hit.FindAllBuildCache(TreeScope_Subtree, &everything, &request) }.ok()?;
+ let elements: Vec<IUIAutomationElement> = (0..unsafe { found.Length() }.ok()?)
+ .filter_map(|index| unsafe { found.GetElement(index) }.ok())
+ .collect();
+ let boxes: Vec<_> = elements
+ .iter()
+ .map(|element| {
+ unsafe { element.CachedBoundingRectangle() }
+ .map(|rect| (rect.left, rect.top, rect.right, rect.bottom))
+ .unwrap_or_default()
+ })
+ .collect();
+ smallest_under(&boxes, point.x, point.y).map(|index| elements[index].clone())
+}
+
pub fn element_at_point(x: i32, y: i32) -> Result<Option<UiElement>> {
let uia = automation().map_err(|error| napi::Error::from_reason(error.message()))?;
- let found = unsafe { uia.ElementFromPoint(POINT { x, y }) }
- .map_err(|error| napi::Error::from_reason(error.message()))?;
+ let point = POINT { x, y };
+ let hit =
+ unsafe { uia.ElementFromPoint(point) }.map_err(|error| napi::Error::from_reason(error.message()))?;
+ let walker = unsafe { uia.ControlViewWalker() }.ok();
+ let found = walker
+ .as_ref()
+ .and_then(|walker| narrowest(&uia, walker, &hit, point))
+ .unwrap_or(hit);
let mut element = describe(&found);
if element.name.is_none() {
- if let Ok(walker) = unsafe { uia.ControlViewWalker() } {
- element.ancestors = ancestors(&walker, &found);
- element.children = children(&walker, &found);
+ if let Some(walker) = &walker {
+ element.ancestors = ancestors(walker, &found);
+ element.children = children(walker, &found);
}
}
Ok(Some(element))