Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots 📖

fix(core): name typing by its text and skip id-like and invisible names

Luis Ortiz4129b30ed863
diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.ts
index 35b48b1..801c1c6 100644
--- a/apps/desktop/src/renderer/capture-sink.ts
+++ b/apps/desktop/src/renderer/capture-sink.ts
@@ -64,7 +64,7 @@ export class DesktopCaptureSink implements CaptureSink {
screenshotId = screenshot.id;
}
- const description = buildFallbackDescription(data.action, meta);
+ const description = buildFallbackDescription(data.action, meta, data.inputValue);
const pending = (await credentials()) !== null;
await createStep({
diff --git a/packages/core/src/capture/__tests__/step-description.test.ts b/packages/core/src/capture/__tests__/step-description.test.ts
index fc88977..e951d8a 100644
--- a/packages/core/src/capture/__tests__/step-description.test.ts
+++ b/packages/core/src/capture/__tests__/step-description.test.ts
@@ -192,6 +192,25 @@ describe('accessibility-tree sources', () => {
expect(buildFallbackDescription('click', axMeta({ role: 'pane', textContent: '42' }))).toBe('steps.clickHere');
});
+ it('never names a control by an identifier', () => {
+ for (const name of ['fl-post-111', 'SaveButton', 'btn_submit', 'item3']) {
+ expect(buildFallbackDescription('click', axMeta({ role: 'button', name }))).toBe('steps.clickHere');
+ }
+ expect(buildFallbackDescription('click', axMeta({ role: 'button', name: 'Save' }))).toBe('steps.click[Save]');
+ });
+
+ it('names what was typed when the value is known', () => {
+ const meta = axMeta({ role: 'textbox', ariaLabel: 'Search box' });
+ expect(buildFallbackDescription('input', meta, ' noticias\nde hoy ')).toBe('steps.type[noticias de hoy]');
+ expect(buildFallbackDescription('input', meta)).toBe('steps.typeInto[Search box]');
+ expect(buildFallbackDescription('input', { ...meta, inputType: 'password' }, 'hunter2')).toBe('steps.typeSecret');
+ });
+
+ it('drops invisible characters from names', () => {
+ const meta = axMeta({ role: 'link', ariaLabel: '\u2068Policía\u2069 \u2068abate\u200b\u2069' });
+ expect(buildFallbackDescription('click', meta)).toBe('steps.click[Policía abate]');
+ });
+
it('describes typing without an inputType', () => {
expect(buildFallbackDescription('input', axMeta({ name: 'Cell B4' }))).toBe('steps.typeInto[Cell B4]');
});
diff --git a/packages/core/src/capture/step-description.ts b/packages/core/src/capture/step-description.ts
index bbe30fa..652f95f 100644
--- a/packages/core/src/capture/step-description.ts
+++ b/packages/core/src/capture/step-description.ts
@@ -28,17 +28,20 @@ const HOST_SURFACES = new Set([
'intermediate d3d window',
]);
const NAME_LIMIT = 200;
+const INVISIBLE = /[\p{Cf}\uFFFC\uFFFD]/gu;
+const IDENTIFIER = /^(?=.*(?:[\d_.:[\]-]|[a-z][A-Z]))\S+$/;
function elementName(meta: ElementMeta): string {
const usable = (value: string | null | undefined) => {
- const text = (value ?? '').trim();
+ const text = (value ?? '').replace(INVISIBLE, '').trim();
const lower = text.toLowerCase();
return HOST_SURFACES.has(lower) || lower.startsWith('chrome_widgetwin_') ? '' : text;
};
const role = meta.role ?? '';
const value = usable(meta.textContent?.slice(0, 80));
const hideValue = VERB_BY_ROLE[role] === 'enter' || (BOUNDARIES.has(role) && /^\d+$/.test(value));
- const own = [meta.ariaLabel, meta.placeholder, hideValue ? null : value, meta.altText, meta.name]
+ const handle = IDENTIFIER.test(meta.name ?? '') ? null : meta.name;
+ const own = [meta.ariaLabel, meta.placeholder, hideValue ? null : value, meta.altText, handle]
.map(usable)
.find(Boolean);
if (own) return own.slice(0, NAME_LIMIT);
@@ -55,7 +58,7 @@ function elementName(meta: ElementMeta): string {
return around ? usable(around.name).slice(0, NAME_LIMIT) : '';
}
-export function buildFallbackDescription(action: string, meta: ElementMeta): string {
+export function buildFallbackDescription(action: string, meta: ElementMeta, typed?: string): string {
const name = elementName(meta);
const target = name || meta.role || meta.tag || '';
@@ -79,6 +82,7 @@ export function buildFallbackDescription(action: string, meta: ElementMeta): str
return i18n.t('steps.click', [name]);
case 'input':
if (meta.inputType === 'password') return i18n.t('steps.typeSecret');
+ if (typed?.trim()) return i18n.t('steps.type', [typed.replace(/\s+/g, ' ').trim().slice(0, NAME_LIMIT)]);
if (meta.inputType) return i18n.t('steps.typeIntoField', [meta.inputType, target]);
return i18n.t('steps.typeInto', [target]);
case 'copy':
diff --git a/src/locales/de.yml b/src/locales/de.yml
index 9ca25f0..f4e254c 100644
--- a/src/locales/de.yml
+++ b/src/locales/de.yml
@@ -455,6 +455,7 @@ steps:
rightClickHere: "Hier rechtsklicken"
typeIntoField: 'In Feld $1 eingeben "$2"'
typeInto: 'In "$1" eingeben'
+ type: '"$1" eingeben'
typeSecret: "Passwort eingeben"
copyFrom: 'Aus "$1" kopieren'
pasteInto: 'In "$1" einfügen'
diff --git a/src/locales/en.yml b/src/locales/en.yml
index a869d3d..e40392f 100644
--- a/src/locales/en.yml
+++ b/src/locales/en.yml
@@ -524,6 +524,7 @@ steps:
rightClickHere: "Right-click here"
typeIntoField: 'Type into $1 field "$2"'
typeInto: 'Type into "$1"'
+ type: 'Type "$1"'
typeSecret: "Type password"
copyFrom: 'Copy from "$1"'
pasteInto: 'Paste into "$1"'
diff --git a/src/locales/es.yml b/src/locales/es.yml
index 5486cd1..0bc4f05 100644
--- a/src/locales/es.yml
+++ b/src/locales/es.yml
@@ -524,6 +524,7 @@ steps:
rightClickHere: "Clic derecho aquí"
typeIntoField: 'Escribir en el campo $1 "$2"'
typeInto: 'Escribir en "$1"'
+ type: 'Escribir "$1"'
typeSecret: "Escribir contraseña"
copyFrom: 'Copiar de "$1"'
pasteInto: 'Pegar en "$1"'
diff --git a/src/locales/fr.yml b/src/locales/fr.yml
index 0d986a0..79f6d45 100644
--- a/src/locales/fr.yml
+++ b/src/locales/fr.yml
@@ -524,6 +524,7 @@ steps:
rightClickHere: "Clic droit ici"
typeIntoField: 'Saisir dans le champ $1 "$2"'
typeInto: 'Saisir dans "$1"'
+ type: 'Saisir "$1"'
typeSecret: "Saisir le mot de passe"
copyFrom: 'Copier depuis "$1"'
pasteInto: 'Coller dans "$1"'
diff --git a/src/locales/pt-BR.yml b/src/locales/pt-BR.yml
index 79b2400..f7f8628 100644
--- a/src/locales/pt-BR.yml
+++ b/src/locales/pt-BR.yml
@@ -524,6 +524,7 @@ steps:
rightClickHere: "Clicar com o botão direito aqui"
typeIntoField: 'Digitar no campo $1 "$2"'
typeInto: 'Digitar em "$1"'
+ type: 'Digitar "$1"'
typeSecret: "Digitar senha"
copyFrom: 'Copiar de "$1"'
pasteInto: 'Colar em "$1"'
diff --git a/src/locales/zh-CN.yml b/src/locales/zh-CN.yml
index c89e211..9deec4e 100644
--- a/src/locales/zh-CN.yml
+++ b/src/locales/zh-CN.yml
@@ -524,6 +524,7 @@ steps:
rightClickHere: "右键点击此处"
typeIntoField: '在 $1 字段“$2”中输入'
typeInto: '在“$1”中输入'
+ type: "输入“$1”"
typeSecret: "输入密码"
copyFrom: '从“$1”复制'
pasteInto: '粘贴到“$1”'