🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots 📖
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
fix(core): name typing by its text and skip id-like and invisible names
Luis Ortiz4129b30ed863
diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.tsindex 35b48b1..801c1c6 100644--- a/apps/desktop/src/renderer/capture-sink.ts+++ b/apps/desktop/src/renderer/capture-sink.ts@@ -64,7 +64,7 @@ export class DesktopCaptureSink implements CaptureSink {screenshotId = screenshot.id;}- const description = buildFallbackDescription(data.action, meta);+ const description = buildFallbackDescription(data.action, meta, data.inputValue);const pending = (await credentials()) !== null;await createStep({diff --git a/packages/core/src/capture/__tests__/step-description.test.ts b/packages/core/src/capture/__tests__/step-description.test.tsindex fc88977..e951d8a 100644--- a/packages/core/src/capture/__tests__/step-description.test.ts+++ b/packages/core/src/capture/__tests__/step-description.test.ts@@ -192,6 +192,25 @@ describe('accessibility-tree sources', () => {expect(buildFallbackDescription('click', axMeta({ role: 'pane', textContent: '42' }))).toBe('steps.clickHere');});+ it('never names a control by an identifier', () => {+ for (const name of ['fl-post-111', 'SaveButton', 'btn_submit', 'item3']) {+ expect(buildFallbackDescription('click', axMeta({ role: 'button', name }))).toBe('steps.clickHere');+ }+ expect(buildFallbackDescription('click', axMeta({ role: 'button', name: 'Save' }))).toBe('steps.click[Save]');+ });++ it('names what was typed when the value is known', () => {+ const meta = axMeta({ role: 'textbox', ariaLabel: 'Search box' });+ expect(buildFallbackDescription('input', meta, ' noticias\nde hoy ')).toBe('steps.type[noticias de hoy]');+ expect(buildFallbackDescription('input', meta)).toBe('steps.typeInto[Search box]');+ expect(buildFallbackDescription('input', { ...meta, inputType: 'password' }, 'hunter2')).toBe('steps.typeSecret');+ });++ it('drops invisible characters from names', () => {+ const meta = axMeta({ role: 'link', ariaLabel: '\u2068Policía\u2069 \u2068abate\u200b\u2069' });+ expect(buildFallbackDescription('click', meta)).toBe('steps.click[Policía abate]');+ });+it('describes typing without an inputType', () => {expect(buildFallbackDescription('input', axMeta({ name: 'Cell B4' }))).toBe('steps.typeInto[Cell B4]');});diff --git a/packages/core/src/capture/step-description.ts b/packages/core/src/capture/step-description.tsindex bbe30fa..652f95f 100644--- a/packages/core/src/capture/step-description.ts+++ b/packages/core/src/capture/step-description.ts@@ -28,17 +28,20 @@ const HOST_SURFACES = new Set(['intermediate d3d window',]);const NAME_LIMIT = 200;+const INVISIBLE = /[\p{Cf}\uFFFC\uFFFD]/gu;+const IDENTIFIER = /^(?=.*(?:[\d_.:[\]-]|[a-z][A-Z]))\S+$/;function elementName(meta: ElementMeta): string {const usable = (value: string | null | undefined) => {- const text = (value ?? '').trim();+ const text = (value ?? '').replace(INVISIBLE, '').trim();const lower = text.toLowerCase();return HOST_SURFACES.has(lower) || lower.startsWith('chrome_widgetwin_') ? '' : text;};const role = meta.role ?? '';const value = usable(meta.textContent?.slice(0, 80));const hideValue = VERB_BY_ROLE[role] === 'enter' || (BOUNDARIES.has(role) && /^\d+$/.test(value));- const own = [meta.ariaLabel, meta.placeholder, hideValue ? null : value, meta.altText, meta.name]+ const handle = IDENTIFIER.test(meta.name ?? '') ? null : meta.name;+ const own = [meta.ariaLabel, meta.placeholder, hideValue ? null : value, meta.altText, handle].map(usable).find(Boolean);if (own) return own.slice(0, NAME_LIMIT);@@ -55,7 +58,7 @@ function elementName(meta: ElementMeta): string {return around ? usable(around.name).slice(0, NAME_LIMIT) : '';}-export function buildFallbackDescription(action: string, meta: ElementMeta): string {+export function buildFallbackDescription(action: string, meta: ElementMeta, typed?: string): string {const name = elementName(meta);const target = name || meta.role || meta.tag || '';@@ -79,6 +82,7 @@ export function buildFallbackDescription(action: string, meta: ElementMeta): strreturn i18n.t('steps.click', [name]);case 'input':if (meta.inputType === 'password') return i18n.t('steps.typeSecret');+ if (typed?.trim()) return i18n.t('steps.type', [typed.replace(/\s+/g, ' ').trim().slice(0, NAME_LIMIT)]);if (meta.inputType) return i18n.t('steps.typeIntoField', [meta.inputType, target]);return i18n.t('steps.typeInto', [target]);case 'copy':diff --git a/src/locales/de.yml b/src/locales/de.ymlindex 9ca25f0..f4e254c 100644--- a/src/locales/de.yml+++ b/src/locales/de.yml@@ -455,6 +455,7 @@ steps:rightClickHere: "Hier rechtsklicken"typeIntoField: 'In Feld $1 eingeben "$2"'typeInto: 'In "$1" eingeben'+ type: '"$1" eingeben'typeSecret: "Passwort eingeben"copyFrom: 'Aus "$1" kopieren'pasteInto: 'In "$1" einfügen'diff --git a/src/locales/en.yml b/src/locales/en.ymlindex a869d3d..e40392f 100644--- a/src/locales/en.yml+++ b/src/locales/en.yml@@ -524,6 +524,7 @@ steps:rightClickHere: "Right-click here"typeIntoField: 'Type into $1 field "$2"'typeInto: 'Type into "$1"'+ type: 'Type "$1"'typeSecret: "Type password"copyFrom: 'Copy from "$1"'pasteInto: 'Paste into "$1"'diff --git a/src/locales/es.yml b/src/locales/es.ymlindex 5486cd1..0bc4f05 100644--- a/src/locales/es.yml+++ b/src/locales/es.yml@@ -524,6 +524,7 @@ steps:rightClickHere: "Clic derecho aquí"typeIntoField: 'Escribir en el campo $1 "$2"'typeInto: 'Escribir en "$1"'+ type: 'Escribir "$1"'typeSecret: "Escribir contraseña"copyFrom: 'Copiar de "$1"'pasteInto: 'Pegar en "$1"'diff --git a/src/locales/fr.yml b/src/locales/fr.ymlindex 0d986a0..79f6d45 100644--- a/src/locales/fr.yml+++ b/src/locales/fr.yml@@ -524,6 +524,7 @@ steps:rightClickHere: "Clic droit ici"typeIntoField: 'Saisir dans le champ $1 "$2"'typeInto: 'Saisir dans "$1"'+ type: 'Saisir "$1"'typeSecret: "Saisir le mot de passe"copyFrom: 'Copier depuis "$1"'pasteInto: 'Coller dans "$1"'diff --git a/src/locales/pt-BR.yml b/src/locales/pt-BR.ymlindex 79b2400..f7f8628 100644--- a/src/locales/pt-BR.yml+++ b/src/locales/pt-BR.yml@@ -524,6 +524,7 @@ steps:rightClickHere: "Clicar com o botão direito aqui"typeIntoField: 'Digitar no campo $1 "$2"'typeInto: 'Digitar em "$1"'+ type: 'Digitar "$1"'typeSecret: "Digitar senha"copyFrom: 'Copiar de "$1"'pasteInto: 'Colar em "$1"'diff --git a/src/locales/zh-CN.yml b/src/locales/zh-CN.ymlindex c89e211..9deec4e 100644--- a/src/locales/zh-CN.yml+++ b/src/locales/zh-CN.yml@@ -524,6 +524,7 @@ steps:rightClickHere: "右键点击此处"typeIntoField: '在 $1 字段“$2”中输入'typeInto: '在“$1”中输入'+ type: "输入“$1”"typeSecret: "输入密码"copyFrom: '从“$1”复制'pasteInto: '粘贴到“$1”'