π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(desktop): report the elements around an unnamed control
Luis Ortize044eb95949f
diff --git a/apps/desktop/scripts/check-pipeline.ts b/apps/desktop/scripts/check-pipeline.tsindex 3becc11..60635ba 100644--- a/apps/desktop/scripts/check-pipeline.ts+++ b/apps/desktop/scripts/check-pipeline.ts@@ -180,6 +180,8 @@ app.whenReady().then(async () => {ariaLabel: 'Save',altText: null,password: false,+ ancestors: [],+ children: [],rect: { x: region.x + 40, y: region.y + 30, width: 120, height: 32 },};let field: ScreenElement | null = {@@ -189,6 +191,8 @@ app.whenReady().then(async () => {ariaLabel: 'Search',altText: null,password: false,+ ancestors: [],+ children: [],rect: { x: region.x + 20, y: region.y + 20, width: 200, height: 24 },};let seen: CaptureRequest | null = null;@@ -243,6 +247,8 @@ app.whenReady().then(async () => {ariaLabel: 'Search',altText: null,password: false,+ ancestors: [],+ children: [],rect: null,};await metaRecorder.captureTyping();@@ -256,6 +262,8 @@ app.whenReady().then(async () => {ariaLabel: 'Password',altText: null,password: true,+ ancestors: [],+ children: [],rect: { x: region.x + 20, y: region.y + 60, width: 200, height: 24 },};await metaRecorder.captureTyping();@@ -355,6 +363,8 @@ app.whenReady().then(async () => {ariaLabel: 'Document',altText: null,password: false,+ ancestors: [],+ children: [],rect: { x: region.x + 10, y: region.y + 10, width: 400, height: 300 },});diff --git a/apps/desktop/src/main/capture/element.ts b/apps/desktop/src/main/capture/element.tsindex 86c886a..c26be67 100644--- a/apps/desktop/src/main/capture/element.ts+++ b/apps/desktop/src/main/capture/element.ts@@ -1,4 +1,4 @@-import type { UiElement } from '@mimik/capture-native';+import type { ElementNode, UiElement } from '@mimik/capture-native';import { screen } from 'electron';import { toDip } from './focused-window';@@ -12,6 +12,8 @@ export interface ScreenElement {altText: string | null;password: boolean;rect: { x: number; y: number; width: number; height: number } | null;+ ancestors: { role: string | null; name: string | null }[];+ children: { role: string | null; name: string | null }[];}const TEXT_ROLES = new Set(['textbox', 'combobox', 'document']);@@ -59,6 +61,8 @@ function toPhysical(point: { x: number; y: number }): { x: number; y: number } {}}+const node = (found: ElementNode) => ({ role: found.role ?? null, name: found.name ?? null });+function describe(found: UiElement): ScreenElement {return {role: found.role ?? null,@@ -68,6 +72,8 @@ function describe(found: UiElement): ScreenElement {altText: found.helpText ?? null,password: found.isPassword,rect: toDip(found.rect ?? null),+ ancestors: (found.ancestors ?? []).map(node),+ children: (found.children ?? []).map(node),};}diff --git a/apps/desktop/src/main/capture/recorder.ts b/apps/desktop/src/main/capture/recorder.tsindex 517b97e..8fef083 100644--- a/apps/desktop/src/main/capture/recorder.ts+++ b/apps/desktop/src/main/capture/recorder.ts@@ -400,6 +400,8 @@ export class DesktopRecorder {...(target?.password ? { inputType: 'password' } : {}),devicePixelRatio: scale,clickPoint: local,+ ...(target?.ancestors.length ? { ancestors: target.ancestors } : {}),+ ...(target?.children.length ? { children: target.children } : {}),...(found.ok ? { app: found.window.app, window: { title: found.window.title } } : {}),},image: {diff --git a/packages/capture-native/index.d.ts b/packages/capture-native/index.d.tsindex 190d3f1..fc4b879 100644--- a/packages/capture-native/index.d.ts+++ b/packages/capture-native/index.d.ts@@ -11,6 +11,11 @@ export declare const __napiBindingTarget: 'native' | 'wasm32-wasi' | 'wasm32-wasexport declare function elementAtPoint(x: number, y: number): Promise<UiElement | null>+export interface ElementNode {+ role?: string+ name?: string+}+export interface ElementRect {x: numbery: number@@ -36,4 +41,6 @@ export interface UiElement {helpText?: stringisPassword: booleanrect?: ElementRect+ ancestors: Array<ElementNode>+ children: Array<ElementNode>}diff --git a/packages/capture-native/src/lib.rs b/packages/capture-native/src/lib.rsindex 1ede4b8..01c19b8 100644--- a/packages/capture-native/src/lib.rs+++ b/packages/capture-native/src/lib.rs@@ -15,6 +15,12 @@ pub struct ElementRect {pub height: f64,}+#[napi(object)]+pub struct ElementNode {+ pub role: Option<String>,+ pub name: Option<String>,+}+#[napi(object)]pub struct UiElement {pub role: Option<String>,@@ -24,6 +30,8 @@ pub struct UiElement {pub help_text: Option<String>,pub is_password: bool,pub rect: Option<ElementRect>,+ pub ancestors: Vec<ElementNode>,+ pub children: Vec<ElementNode>,}pub struct ElementLookup {diff --git a/packages/capture-native/src/win.rs b/packages/capture-native/src/win.rsindex 4117e3b..d43f239 100644--- a/packages/capture-native/src/win.rs+++ b/packages/capture-native/src/win.rs@@ -14,7 +14,7 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{};use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};-use crate::{ElementRect, UiElement};+use crate::{ElementNode, ElementRect, UiElement};const ROLES: &[(UIA_CONTROLTYPE_ID, &str)] = &[(UIA_ButtonControlTypeId, "button"),@@ -128,14 +128,59 @@ fn describe(found: &IUIAutomationElement) -> UiElement {.map(|flag| flag.as_bool()).unwrap_or(false),rect: rect_of(found),+ ancestors: Vec::new(),+ children: Vec::new(),}}+const MAX_ANCESTORS: usize = 4;+const MAX_CHILDREN: usize = 12;++fn node(element: &IUIAutomationElement) -> ElementNode {+ ElementNode {+ role: role(unsafe { element.CurrentControlType() }.unwrap_or_default()),+ name: text(unsafe { element.CurrentName() }),+ }+}++fn ancestors(walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement) -> Vec<ElementNode> {+ let mut out = Vec::new();+ let mut current = element.clone();+ while out.len() < MAX_ANCESTORS {+ let Ok(parent) = (unsafe { walker.GetParentElement(¤t) }) else {+ break;+ };+ out.push(node(&parent));+ current = parent;+ }+ out+}++fn children(walker: &IUIAutomationTreeWalker, element: &IUIAutomationElement) -> Vec<ElementNode> {+ let mut out = Vec::new();+ let mut next = unsafe { walker.GetFirstChildElement(element) };+ while let Ok(child) = next {+ if out.len() == MAX_CHILDREN {+ break;+ }+ out.push(node(&child));+ next = unsafe { walker.GetNextSiblingElement(&child) };+ }+ out+}+pub fn element_at_point(x: i32, y: i32) -> Result<Option<UiElement>> {- let found = automation()- .and_then(|uia| unsafe { uia.ElementFromPoint(POINT { x, y }) })+ let uia = automation().map_err(|error| napi::Error::from_reason(error.message()))?;+ let found = unsafe { uia.ElementFromPoint(POINT { x, y }) }.map_err(|error| napi::Error::from_reason(error.message()))?;- Ok(Some(describe(&found)))+ let mut element = describe(&found);+ if element.name.is_none() {+ if let Ok(walker) = unsafe { uia.ControlViewWalker() } {+ element.ancestors = ancestors(&walker, &found);+ element.children = children(&walker, &found);+ }+ }+ Ok(Some(element))}pub fn focused_element() -> Result<Option<UiElement>> {diff --git a/packages/core/src/guides/types.ts b/packages/core/src/guides/types.tsindex de50cb1..5d7ddb1 100644--- a/packages/core/src/guides/types.ts+++ b/packages/core/src/guides/types.ts@@ -97,6 +97,11 @@ export type SettingsKey = keyof Settings;export type ElementSource = 'dom' | 'ax' | 'uia' | 'screen';+export interface ElementNode {+ role: string | null;+ name: string | null;+}+export interface ElementMeta {source?: ElementSource;textContent: string | null;@@ -115,6 +120,8 @@ export interface ElementMeta {dataTestId?: string | null;app?: { name: string; id?: string };window?: { title: string | null };+ ancestors?: ElementNode[];+ children?: ElementNode[];}export function elementSource(meta: ElementMeta): ElementSource {