π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
refactor(core): make ElementMeta source-agnostic across DOM and accessibility trees
Luis Ortize3983296e767
diff --git a/AGENTS.md b/AGENTS.mdindex df1347a..4188dd3 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -176,6 +176,34 @@ Extraction walks up from the target element to find:- Nearest heading- Sibling interactive elements in the same container (max 10)+## Element Metadata++`ElementMeta` describes the thing the user acted on, and it is written by more than one kind of+capture. `source` says which β `dom` in the extension, `ax` on macOS, `uia` on Windows. It is absent+on records captured before the field existed, so read it through `elementSource(meta)`, which+defaults to `dom`.++The identity fields are shared, and every source populates them:++| Field | DOM | macOS AX | Windows UIAutomation |+|---|---|---|---|+| `role` | `role` attribute, else tag name | `AXRole` | `ControlType` |+| `name` | `name` attribute | `AXTitle` | `Name` |+| `textContent` | trimmed text | `AXValue` / `AXSelectedText` | `Value` |+| `ariaLabel` | `aria-label` | `AXDescription` / `AXARIAValueText` | `HelpText` |+| `placeholder` | `placeholder` | `AXPlaceholderValue` | `Placeholder` |+| `altText` | `img.alt` | `AXHelp` | `HelpText` |+| `rect` | `getBoundingClientRect()` | `AXPosition` + `AXSize` | `BoundingRectangle` |++`tag`, `cssSelector`, `href`, `inputType` and `dataTestId` are DOM-only and absent elsewhere.+`app` and `window` are the reverse β desktop only.++Consumers must not branch on `source`. Read the shared fields first and treat the DOM-only ones as+refinements: `buildFallbackDescription` reaches the same wording through `role === 'checkbox'` that+it reaches through `tag === 'input' && inputType === 'checkbox'`. Guide Me is the exception by+nature β it replays against a live DOM, so `finder.ts` scores `cssSelector` only when present and+falls back to `*` without a tag.+## Export Formats| Format | Generator | Details |diff --git a/packages/core/src/capture/__tests__/step-description.test.ts b/packages/core/src/capture/__tests__/step-description.test.tsindex fb9b282..10e80fb 100644--- a/packages/core/src/capture/__tests__/step-description.test.ts+++ b/packages/core/src/capture/__tests__/step-description.test.ts@@ -109,3 +109,44 @@ describe('buildFallbackDescription', () => {expect(result).toBe(`steps.click[${'A'.repeat(80)}]`);});});++describe('accessibility-tree sources', () => {+ function axMeta(overrides: Partial<ElementMeta> = {}): ElementMeta {+ return {+ source: 'ax',+ textContent: null,+ ariaLabel: null,+ placeholder: null,+ altText: null,+ name: null,+ role: null,+ rect: { x: 0, y: 0, width: 0, height: 0 },+ devicePixelRatio: 2,+ app: { name: 'Excel', id: 'com.microsoft.Excel' },+ window: { title: 'Budget.xlsx' },+ ...overrides,+ };+ }++ it('describes a click with no DOM fields at all', () => {+ expect(buildFallbackDescription('click', axMeta({ name: 'Save' }))).toBe('steps.click[Save]');+ });++ it('reaches the same checkbox wording through role as the DOM does through tag', () => {+ const viaRole = buildFallbackDescription('click', axMeta({ role: 'checkbox', name: 'Wrap text' }));+ const viaTag = buildFallbackDescription(+ 'click',+ makeMeta({ tag: 'input', inputType: 'checkbox', ariaLabel: 'Wrap text' }),+ );+ expect(viaRole).toBe('steps.toggleCheckbox[Wrap text]');+ expect(viaTag).toBe(viaRole);+ });++ it('falls back to role when nothing nameable is present', () => {+ expect(buildFallbackDescription('click', axMeta({ role: 'button' }))).toBe('steps.click[button]');+ });++ it('describes typing without an inputType', () => {+ expect(buildFallbackDescription('input', axMeta({ name: 'Cell B4' }))).toBe('steps.typeInto[Cell B4]');+ });+});diff --git a/packages/core/src/capture/dom/element-meta.ts b/packages/core/src/capture/dom/element-meta.tsindex 5b4ac41..e95ec8a 100644--- a/packages/core/src/capture/dom/element-meta.ts+++ b/packages/core/src/capture/dom/element-meta.ts@@ -44,6 +44,7 @@ export function extractElementMeta(el: HTMLElement, atEvent?: FrozenRect): ElemecssSelector = el.tagName?.toLowerCase() ?? 'unknown';}return {+ source: 'dom',tag: el.tagName?.toLowerCase() ?? 'unknown',cssSelector,textContent: getCleanText(el),diff --git a/packages/core/src/capture/step-description.ts b/packages/core/src/capture/step-description.tsindex 0fcd846..98c4a39 100644--- a/packages/core/src/capture/step-description.ts+++ b/packages/core/src/capture/step-description.ts@@ -9,7 +9,8 @@ export function buildFallbackDescription(action: string, meta: ElementMeta): strmeta.altText ||meta.name ||meta.role ||- meta.tag;+ meta.tag ||+ '';if (action.startsWith('keydown:')) {const key = action.split(':')[1];diff --git a/packages/core/src/guideme/finder.ts b/packages/core/src/guideme/finder.tsindex c9c04ca..dd6c5b7 100644--- a/packages/core/src/guideme/finder.ts+++ b/packages/core/src/guideme/finder.ts@@ -59,6 +59,7 @@ function getCandidateValue(el: HTMLElement, signal: string): string | null {}function scoreCssSelector(meta: ElementMeta, candidate: HTMLElement): number {+ if (!meta.cssSelector) return 0;try {const matched = document.querySelector(meta.cssSelector);return matched === candidate ? 1 : 0;@@ -75,7 +76,7 @@ function scoreCandidate(for (const [key, weight] of Object.entries(WEIGHTS)) {if (key === 'cssSelector') {- activeSignals.push({ key, weight });+ if (meta.cssSelector) activeSignals.push({ key, weight });continue;}const storedValue = meta[key as keyof ElementMeta];@@ -114,7 +115,7 @@ function scoreCandidate(}function findElement(meta: ElementMeta): FindResult {- const candidates = document.querySelectorAll<HTMLElement>(meta.tag);+ const candidates = document.querySelectorAll<HTMLElement>(meta.tag ?? '*');let bestResult: FindResult = { element: null, score: 0, matchDetails: {} };for (const candidate of candidates) {diff --git a/packages/core/src/guides/types.ts b/packages/core/src/guides/types.tsindex 3a34bb2..c89468a 100644--- a/packages/core/src/guides/types.ts+++ b/packages/core/src/guides/types.ts@@ -90,21 +90,30 @@ export interface Settings {export type SettingsKey = keyof Settings;+export type ElementSource = 'dom' | 'ax' | 'uia';+export interface ElementMeta {- tag: string;- cssSelector: string;+ source?: ElementSource;textContent: string | null;ariaLabel: string | null;placeholder: string | null;altText: string | null;name: string | null;role: string | null;- href: string | null;- inputType: string | null;- dataTestId: string | null;rect: { x: number; y: number; width: number; height: number };devicePixelRatio: number;clickPoint?: { x: number; y: number };+ tag?: string;+ cssSelector?: string;+ href?: string | null;+ inputType?: string | null;+ dataTestId?: string | null;+ app?: { name: string; id?: string };+ window?: { title: string | null };+}++export function elementSource(meta: ElementMeta): ElementSource {+ return meta.source ?? 'dom';}export interface Snapshot {