Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

refactor(core): make ElementMeta source-agnostic across DOM and accessibility trees

Luis Ortize3983296e767
diff --git a/AGENTS.md b/AGENTS.md
index df1347a..4188dd3 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -176,6 +176,34 @@ Extraction walks up from the target element to find:
- Nearest heading
- Sibling interactive elements in the same container (max 10)
+## Element Metadata
+
+`ElementMeta` describes the thing the user acted on, and it is written by more than one kind of
+capture. `source` says which β€” `dom` in the extension, `ax` on macOS, `uia` on Windows. It is absent
+on records captured before the field existed, so read it through `elementSource(meta)`, which
+defaults to `dom`.
+
+The identity fields are shared, and every source populates them:
+
+| Field | DOM | macOS AX | Windows UIAutomation |
+|---|---|---|---|
+| `role` | `role` attribute, else tag name | `AXRole` | `ControlType` |
+| `name` | `name` attribute | `AXTitle` | `Name` |
+| `textContent` | trimmed text | `AXValue` / `AXSelectedText` | `Value` |
+| `ariaLabel` | `aria-label` | `AXDescription` / `AXARIAValueText` | `HelpText` |
+| `placeholder` | `placeholder` | `AXPlaceholderValue` | `Placeholder` |
+| `altText` | `img.alt` | `AXHelp` | `HelpText` |
+| `rect` | `getBoundingClientRect()` | `AXPosition` + `AXSize` | `BoundingRectangle` |
+
+`tag`, `cssSelector`, `href`, `inputType` and `dataTestId` are DOM-only and absent elsewhere.
+`app` and `window` are the reverse β€” desktop only.
+
+Consumers must not branch on `source`. Read the shared fields first and treat the DOM-only ones as
+refinements: `buildFallbackDescription` reaches the same wording through `role === 'checkbox'` that
+it reaches through `tag === 'input' && inputType === 'checkbox'`. Guide Me is the exception by
+nature β€” it replays against a live DOM, so `finder.ts` scores `cssSelector` only when present and
+falls back to `*` without a tag.
+
## Export Formats
| Format | Generator | Details |
diff --git a/packages/core/src/capture/__tests__/step-description.test.ts b/packages/core/src/capture/__tests__/step-description.test.ts
index fb9b282..10e80fb 100644
--- a/packages/core/src/capture/__tests__/step-description.test.ts
+++ b/packages/core/src/capture/__tests__/step-description.test.ts
@@ -109,3 +109,44 @@ describe('buildFallbackDescription', () => {
expect(result).toBe(`steps.click[${'A'.repeat(80)}]`);
});
});
+
+describe('accessibility-tree sources', () => {
+ function axMeta(overrides: Partial<ElementMeta> = {}): ElementMeta {
+ return {
+ source: 'ax',
+ textContent: null,
+ ariaLabel: null,
+ placeholder: null,
+ altText: null,
+ name: null,
+ role: null,
+ rect: { x: 0, y: 0, width: 0, height: 0 },
+ devicePixelRatio: 2,
+ app: { name: 'Excel', id: 'com.microsoft.Excel' },
+ window: { title: 'Budget.xlsx' },
+ ...overrides,
+ };
+ }
+
+ it('describes a click with no DOM fields at all', () => {
+ expect(buildFallbackDescription('click', axMeta({ name: 'Save' }))).toBe('steps.click[Save]');
+ });
+
+ it('reaches the same checkbox wording through role as the DOM does through tag', () => {
+ const viaRole = buildFallbackDescription('click', axMeta({ role: 'checkbox', name: 'Wrap text' }));
+ const viaTag = buildFallbackDescription(
+ 'click',
+ makeMeta({ tag: 'input', inputType: 'checkbox', ariaLabel: 'Wrap text' }),
+ );
+ expect(viaRole).toBe('steps.toggleCheckbox[Wrap text]');
+ expect(viaTag).toBe(viaRole);
+ });
+
+ it('falls back to role when nothing nameable is present', () => {
+ expect(buildFallbackDescription('click', axMeta({ role: 'button' }))).toBe('steps.click[button]');
+ });
+
+ it('describes typing without an inputType', () => {
+ expect(buildFallbackDescription('input', axMeta({ name: 'Cell B4' }))).toBe('steps.typeInto[Cell B4]');
+ });
+});
diff --git a/packages/core/src/capture/dom/element-meta.ts b/packages/core/src/capture/dom/element-meta.ts
index 5b4ac41..e95ec8a 100644
--- a/packages/core/src/capture/dom/element-meta.ts
+++ b/packages/core/src/capture/dom/element-meta.ts
@@ -44,6 +44,7 @@ export function extractElementMeta(el: HTMLElement, atEvent?: FrozenRect): Eleme
cssSelector = el.tagName?.toLowerCase() ?? 'unknown';
}
return {
+ source: 'dom',
tag: el.tagName?.toLowerCase() ?? 'unknown',
cssSelector,
textContent: getCleanText(el),
diff --git a/packages/core/src/capture/step-description.ts b/packages/core/src/capture/step-description.ts
index 0fcd846..98c4a39 100644
--- a/packages/core/src/capture/step-description.ts
+++ b/packages/core/src/capture/step-description.ts
@@ -9,7 +9,8 @@ export function buildFallbackDescription(action: string, meta: ElementMeta): str
meta.altText ||
meta.name ||
meta.role ||
- meta.tag;
+ meta.tag ||
+ '';
if (action.startsWith('keydown:')) {
const key = action.split(':')[1];
diff --git a/packages/core/src/guideme/finder.ts b/packages/core/src/guideme/finder.ts
index c9c04ca..dd6c5b7 100644
--- a/packages/core/src/guideme/finder.ts
+++ b/packages/core/src/guideme/finder.ts
@@ -59,6 +59,7 @@ function getCandidateValue(el: HTMLElement, signal: string): string | null {
}
function scoreCssSelector(meta: ElementMeta, candidate: HTMLElement): number {
+ if (!meta.cssSelector) return 0;
try {
const matched = document.querySelector(meta.cssSelector);
return matched === candidate ? 1 : 0;
@@ -75,7 +76,7 @@ function scoreCandidate(
for (const [key, weight] of Object.entries(WEIGHTS)) {
if (key === 'cssSelector') {
- activeSignals.push({ key, weight });
+ if (meta.cssSelector) activeSignals.push({ key, weight });
continue;
}
const storedValue = meta[key as keyof ElementMeta];
@@ -114,7 +115,7 @@ function scoreCandidate(
}
function findElement(meta: ElementMeta): FindResult {
- const candidates = document.querySelectorAll<HTMLElement>(meta.tag);
+ const candidates = document.querySelectorAll<HTMLElement>(meta.tag ?? '*');
let bestResult: FindResult = { element: null, score: 0, matchDetails: {} };
for (const candidate of candidates) {
diff --git a/packages/core/src/guides/types.ts b/packages/core/src/guides/types.ts
index 3a34bb2..c89468a 100644
--- a/packages/core/src/guides/types.ts
+++ b/packages/core/src/guides/types.ts
@@ -90,21 +90,30 @@ export interface Settings {
export type SettingsKey = keyof Settings;
+export type ElementSource = 'dom' | 'ax' | 'uia';
+
export interface ElementMeta {
- tag: string;
- cssSelector: string;
+ source?: ElementSource;
textContent: string | null;
ariaLabel: string | null;
placeholder: string | null;
altText: string | null;
name: string | null;
role: string | null;
- href: string | null;
- inputType: string | null;
- dataTestId: string | null;
rect: { x: number; y: number; width: number; height: number };
devicePixelRatio: number;
clickPoint?: { x: number; y: number };
+ tag?: string;
+ cssSelector?: string;
+ href?: string | null;
+ inputType?: string | null;
+ dataTestId?: string | null;
+ app?: { name: string; id?: string };
+ window?: { title: string | null };
+}
+
+export function elementSource(meta: ElementMeta): ElementSource {
+ return meta.source ?? 'dom';
}
export interface Snapshot {