Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(desktop): report the elements around an unnamed control

Luis Ortize044eb95949f
diff --git a/apps/desktop/scripts/check-pipeline.ts b/apps/desktop/scripts/check-pipeline.ts
index 3becc11..60635ba 100644
--- a/apps/desktop/scripts/check-pipeline.ts
+++ b/apps/desktop/scripts/check-pipeline.ts
@@ -180,6 +180,8 @@ app.whenReady().then(async () => {
ariaLabel: 'Save',
altText: null,
password: false,
+ ancestors: [],
+ children: [],
rect: { x: region.x + 40, y: region.y + 30, width: 120, height: 32 },
};
let field: ScreenElement | null = {
@@ -189,6 +191,8 @@ app.whenReady().then(async () => {
ariaLabel: 'Search',
altText: null,
password: false,
+ ancestors: [],
+ children: [],
rect: { x: region.x + 20, y: region.y + 20, width: 200, height: 24 },
};
let seen: CaptureRequest | null = null;
@@ -243,6 +247,8 @@ app.whenReady().then(async () => {
ariaLabel: 'Search',
altText: null,
password: false,
+ ancestors: [],
+ children: [],
rect: null,
};
await metaRecorder.captureTyping();
@@ -256,6 +262,8 @@ app.whenReady().then(async () => {
ariaLabel: 'Password',
altText: null,
password: true,
+ ancestors: [],
+ children: [],
rect: { x: region.x + 20, y: region.y + 60, width: 200, height: 24 },
};
await metaRecorder.captureTyping();
@@ -355,6 +363,8 @@ app.whenReady().then(async () => {
ariaLabel: 'Document',
altText: null,
password: false,
+ ancestors: [],
+ children: [],
rect: { x: region.x + 10, y: region.y + 10, width: 400, height: 300 },
});
diff --git a/apps/desktop/src/main/capture/element.ts b/apps/desktop/src/main/capture/element.ts
index 86c886a..c26be67 100644
--- a/apps/desktop/src/main/capture/element.ts
+++ b/apps/desktop/src/main/capture/element.ts
@@ -1,4 +1,4 @@
-import type { UiElement } from '@mimik/capture-native';
+import type { ElementNode, UiElement } from '@mimik/capture-native';
import { screen } from 'electron';
import { toDip } from './focused-window';
@@ -12,6 +12,8 @@ export interface ScreenElement {
altText: string | null;
password: boolean;
rect: { x: number; y: number; width: number; height: number } | null;
+ ancestors: { role: string | null; name: string | null }[];
+ children: { role: string | null; name: string | null }[];
}
const TEXT_ROLES = new Set(['textbox', 'combobox', 'document']);
@@ -59,6 +61,8 @@ function toPhysical(point: { x: number; y: number }): { x: number; y: number } {
}
}
+const node = (found: ElementNode) => ({ role: found.role ?? null, name: found.name ?? null });
+
function describe(found: UiElement): ScreenElement {
return {
role: found.role ?? null,
@@ -68,6 +72,8 @@ function describe(found: UiElement): ScreenElement {
altText: found.helpText ?? null,
password: found.isPassword,
rect: toDip(found.rect ?? null),
+ ancestors: (found.ancestors ?? []).map(node),
+ children: (found.children ?? []).map(node),
};
}
diff --git a/apps/desktop/src/main/capture/recorder.ts b/apps/desktop/src/main/capture/recorder.ts
index 517b97e..8fef083 100644
--- a/apps/desktop/src/main/capture/recorder.ts
+++ b/apps/desktop/src/main/capture/recorder.ts
@@ -400,6 +400,8 @@ export class DesktopRecorder {
...(target?.password ? { inputType: 'password' } : {}),
devicePixelRatio: scale,
clickPoint: local,
+ ...(target?.ancestors.length ? { ancestors: target.ancestors } : {}),
+ ...(target?.children.length ? { children: target.children } : {}),
...(found.ok ? { app: found.window.app, window: { title: found.window.title } } : {}),
},
image: {
diff --git a/packages/capture-native/index.d.ts b/packages/capture-native/index.d.ts
index 190d3f1..fc4b879 100644
--- a/packages/capture-native/index.d.ts
+++ b/packages/capture-native/index.d.ts
@@ -11,6 +11,11 @@ export declare const __napiBindingTarget: 'native' | 'wasm32-wasi' | 'wasm32-was
export declare function elementAtPoint(x: number, y: number): Promise<UiElement | null>
+export interface ElementNode {
+ role?: string
+ name?: string
+}
+
export interface ElementRect {
x: number
y: number
@@ -36,4 +41,6 @@ export interface UiElement {
helpText?: string
isPassword: boolean
rect?: ElementRect
+ ancestors: Array<ElementNode>
+ children: Array<ElementNode>
}
diff --git a/packages/capture-native/src/lib.rs b/packages/capture-native/src/lib.rs
index 1ede4b8..01c19b8 100644
--- a/packages/capture-native/src/lib.rs
+++ b/packages/capture-native/src/lib.rs
@@ -15,6 +15,12 @@ pub struct ElementRect {
pub height: f64,
}
+#[napi(object)]
+pub struct ElementNode {
+ pub role: Option<String>,
+ pub name: Option<String>,
+}
+
#[napi(object)]
pub struct UiElement {
pub role: Option<String>,
@@ -24,6 +30,8 @@ pub struct UiElement {
pub help_text: Option<String>,
pub is_password: bool,
pub rect: Option<ElementRect>,
+ pub ancestors: Vec<ElementNode>,
+ pub children: Vec<ElementNode>,
}
pub struct ElementLookup {
diff --git a/packages/capture-native/src/win.rs b/packages/capture-native/src/win.rs
index 4117e3b..d43f239 100644
--- a/packages/capture-native/src/win.rs
+++ b/packages/capture-native/src/win.rs
@@ -14,7 +14,7 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{
};
use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};
-use crate::{ElementRect, UiElement};
+use crate::{ElementNode, ElementRect, UiElement};
const ROLES: &[(UIA_CONTROLTYPE_ID, &str)] = &[
(UIA_ButtonControlTypeId, "button"),
@@ -128,14 +128,59 @@ fn describe(found: &IUIAutomationElement) -> UiElement {
.map(|flag| flag.as_bool())
.unwrap_or(false),
rect: rect_of(found),
+ ancestors: Vec::new(),
+ children: Vec::new(),
}
}
+const MAX_ANCESTORS: usize = 4;
+const MAX_CHILDREN: usize = 12;
+
+fn node(element: &IUIAutomationElement) -> ElementNode {
+ ElementNode {
+ role: role(unsafe { element.CurrentControlType() }.unwrap_or_default()),
+ name: text(unsafe { element.CurrentName() }),
+ }
+}
+
+fn ancestors(walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement) -> Vec<ElementNode> {
+ let mut out = Vec::new();
+ let mut current = element.clone();
+ while out.len() < MAX_ANCESTORS {
+ let Ok(parent) = (unsafe { walker.GetParentElement(&current) }) else {
+ break;
+ };
+ out.push(node(&parent));
+ current = parent;
+ }
+ out
+}
+
+fn children(walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement) -> Vec<ElementNode> {
+ let mut out = Vec::new();
+ let mut next = unsafe { walker.GetFirstChildElement(element) };
+ while let Ok(child) = next {
+ if out.len() == MAX_CHILDREN {
+ break;
+ }
+ out.push(node(&child));
+ next = unsafe { walker.GetNextSiblingElement(&child) };
+ }
+ out
+}
+
pub fn element_at_point(x: i32, y: i32) -> Result<Option<UiElement>> {
- let found = automation()
- .and_then(|uia| unsafe { uia.ElementFromPoint(POINT { x, y }) })
+ let uia = automation().map_err(|error| napi::Error::from_reason(error.message()))?;
+ let found = unsafe { uia.ElementFromPoint(POINT { x, y }) }
.map_err(|error| napi::Error::from_reason(error.message()))?;
- Ok(Some(describe(&found)))
+ let mut element = describe(&found);
+ if element.name.is_none() {
+ if let Ok(walker) = unsafe { uia.ControlViewWalker() } {
+ element.ancestors = ancestors(&walker, &found);
+ element.children = children(&walker, &found);
+ }
+ }
+ Ok(Some(element))
}
pub fn focused_element() -> Result<Option<UiElement>> {
diff --git a/packages/core/src/guides/types.ts b/packages/core/src/guides/types.ts
index de50cb1..5d7ddb1 100644
--- a/packages/core/src/guides/types.ts
+++ b/packages/core/src/guides/types.ts
@@ -97,6 +97,11 @@ export type SettingsKey = keyof Settings;
export type ElementSource = 'dom' | 'ax' | 'uia' | 'screen';
+export interface ElementNode {
+ role: string | null;
+ name: string | null;
+}
+
export interface ElementMeta {
source?: ElementSource;
textContent: string | null;
@@ -115,6 +120,8 @@ export interface ElementMeta {
dataTestId?: string | null;
app?: { name: string; id?: string };
window?: { title: string | null };
+ ancestors?: ElementNode[];
+ children?: ElementNode[];
}
export function elementSource(meta: ElementMeta): ElementSource {