Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

refactor: build the screenshot row once for both capture surfaces

Luis Ortizb191c7ed3965
diff --git a/AGENTS.md b/AGENTS.md
index 86e2558..c34aae6 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -496,6 +496,16 @@ cannot interleave. For each click it hides the overlays, grabs the display, crop
implements `CaptureSink` and writes through `@mimik/core/guides/service`, exactly as the extension's
`step-pipeline.ts` does.
+The two surfaces reach that write by different routes but build the screenshot row the same way, so
+`screenshotForElement` owns it. It turns an `ElementMeta` and whatever holds the bytes β€” a `Blob` in
+the extension, a `src` on the desktop β€” into a row with the bounds in CSS pixels, the pixel ratio,
+the click point, and the dashed target scaled by that ratio. Scaling the target is the part worth
+having once: a rectangle multiplied in one surface and not the other puts the dashed box on the wrong
+thing, and nothing about that fails a build. Everything else about the two paths genuinely differs β€”
+voice narration, the deferred-description queue and the input finalisation exist only in the
+extension, and the cursor mark, the application name and the fire-and-forget description exist only
+on the desktop β€” so only the row builder is shared.
+
Main cannot `invoke` a renderer, so `ask()` sends a request with a generated reply channel and waits
for `ipcMain.once` on it, with a timeout. The preload's `onRequest` is the other half. Guide creation
and step writes both ride it, because both need IndexedDB, which only the renderer has.
diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.ts
index b478b90..20f5036 100644
--- a/apps/desktop/src/renderer/capture-sink.ts
+++ b/apps/desktop/src/renderer/capture-sink.ts
@@ -18,9 +18,8 @@ import {
saveScreenshot,
updateStepDescription,
} from '@mimik/core/guides/service';
-import type { StoredScreenshot } from '@mimik/core/guides/types';
+import { screenshotForElement } from '@mimik/core/screenshot/record';
import type { CursorMark } from '@mimik/core/screenshot/types';
-import { DEFAULT_TARGET_COLOR } from '@mimik/core/screenshot/types';
import { describeStep } from './ai';
export type DesktopCaptureStepData = CaptureStepData & { cursor?: CursorMark; inputValue?: string };
@@ -39,29 +38,19 @@ export class DesktopCaptureSink implements CaptureSink {
let screenshotId: string | undefined;
if (data.image) {
const { targetColor } = await localStorage.get(['targetColor']);
- const ratio = meta.devicePixelRatio;
- const screenshot: StoredScreenshot = {
- id: data.image.screenshotId,
- src: data.image.src,
- stepId,
- mimeType: 'image/png',
- width: data.image.width,
- height: data.image.height,
- bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },
- pixelRatio: ratio,
- clickPoint: meta.clickPoint,
- edits: {
+ const screenshot = screenshotForElement(
+ {
+ id: data.image.screenshotId,
+ src: data.image.src,
+ stepId,
+ mimeType: 'image/png',
+ width: data.image.width,
+ height: data.image.height,
cursor: data.cursor ?? null,
- target: {
- x: meta.rect.x * ratio,
- y: meta.rect.y * ratio,
- width: meta.rect.width * ratio,
- height: meta.rect.height * ratio,
- border: 'dashed',
- color: targetColor || DEFAULT_TARGET_COLOR,
- },
+ targetColor,
},
- };
+ meta,
+ );
await saveScreenshot(screenshot);
screenshotId = screenshot.id;
}
diff --git a/packages/core/src/screenshot/__tests__/record.test.ts b/packages/core/src/screenshot/__tests__/record.test.ts
new file mode 100644
index 0000000..cbf5ce1
--- /dev/null
+++ b/packages/core/src/screenshot/__tests__/record.test.ts
@@ -0,0 +1,42 @@
+import { describe, expect, it } from 'vitest';
+import type { ElementMeta } from '@/core/guides/types';
+import { screenshotForElement } from '@/core/screenshot/record';
+
+const meta: ElementMeta = {
+ textContent: null,
+ ariaLabel: null,
+ placeholder: null,
+ altText: null,
+ name: null,
+ role: null,
+ rect: { x: 10, y: 20, width: 100, height: 40 },
+ devicePixelRatio: 2,
+ clickPoint: { x: 60, y: 40 },
+};
+
+const bytes = { id: 'ss-1', stepId: 'step-1', mimeType: 'image/png', width: 800, height: 600 };
+
+describe('screenshotForElement', () => {
+ it('keeps bounds in css pixels and scales the target by the pixel ratio', () => {
+ const shot = screenshotForElement(bytes, meta);
+ expect(shot.bounds).toEqual({ x: 10, y: 20, width: 100, height: 40 });
+ expect(shot.pixelRatio).toBe(2);
+ expect(shot.edits?.target).toEqual({
+ x: 20,
+ y: 40,
+ width: 200,
+ height: 80,
+ border: 'dashed',
+ color: '#4F46E5',
+ });
+ });
+
+ it('prefers the configured target colour', () => {
+ expect(screenshotForElement({ ...bytes, targetColor: '#F43F5E' }, meta).edits?.target?.color).toBe('#F43F5E');
+ });
+
+ it('only writes a cursor when one is supplied', () => {
+ expect(screenshotForElement(bytes, meta).edits).not.toHaveProperty('cursor');
+ expect(screenshotForElement({ ...bytes, cursor: null }, meta).edits?.cursor).toBeNull();
+ });
+});
diff --git a/packages/core/src/screenshot/record.ts b/packages/core/src/screenshot/record.ts
new file mode 100644
index 0000000..3d0af1e
--- /dev/null
+++ b/packages/core/src/screenshot/record.ts
@@ -0,0 +1,36 @@
+import type { ElementMeta, StoredScreenshot } from '@/core/guides/types';
+import { type CursorMark, DEFAULT_TARGET_COLOR } from '@/core/screenshot/types';
+
+export interface ScreenshotBytes {
+ id: string;
+ stepId: string;
+ mimeType: string;
+ width: number;
+ height: number;
+ blob?: Blob;
+ src?: string;
+ cursor?: CursorMark | null;
+ targetColor?: string;
+}
+
+export function screenshotForElement(bytes: ScreenshotBytes, meta: ElementMeta): StoredScreenshot {
+ const { cursor, targetColor, ...rest } = bytes;
+ const ratio = meta.devicePixelRatio;
+ return {
+ ...rest,
+ bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },
+ pixelRatio: ratio,
+ clickPoint: meta.clickPoint,
+ edits: {
+ ...(cursor === undefined ? {} : { cursor }),
+ target: {
+ x: meta.rect.x * ratio,
+ y: meta.rect.y * ratio,
+ width: meta.rect.width * ratio,
+ height: meta.rect.height * ratio,
+ border: 'dashed',
+ color: targetColor || DEFAULT_TARGET_COLOR,
+ },
+ },
+ };
+}
diff --git a/src/entrypoints/background/step-pipeline.ts b/src/entrypoints/background/step-pipeline.ts
index 9548509..3447abb 100644
--- a/src/entrypoints/background/step-pipeline.ts
+++ b/src/entrypoints/background/step-pipeline.ts
@@ -13,8 +13,8 @@ import {
updateStepDescription,
updateStepInputValue,
} from '@/core/guides/service';
-import type { ElementMeta, Screenshot } from '@/core/guides/types';
-import { DEFAULT_TARGET_COLOR } from '@/core/screenshot/types';
+import type { ElementMeta } from '@/core/guides/types';
+import { screenshotForElement } from '@/core/screenshot/record';
import { captureVisibleTab, localStorage } from '@/lib/browser-api';
import type { CaptureStepData, CaptureStepResponse } from '@/lib/messaging';
import { getActor } from './actor';
@@ -29,27 +29,18 @@ async function takeScreenshot(stepId: string, meta: ElementMeta): Promise<string
const dataUrl = await captureVisibleTab('jpeg', 90);
const blob = await fetch(dataUrl).then((r) => r.blob());
const img = await createImageBitmap(blob);
- const screenshot: Screenshot = {
- id: crypto.randomUUID(),
- stepId,
- blob,
- mimeType: 'image/jpeg',
- width: img.width,
- height: img.height,
- bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },
- pixelRatio: meta.devicePixelRatio,
- clickPoint: meta.clickPoint,
- edits: {
- target: {
- x: meta.rect.x * meta.devicePixelRatio,
- y: meta.rect.y * meta.devicePixelRatio,
- width: meta.rect.width * meta.devicePixelRatio,
- height: meta.rect.height * meta.devicePixelRatio,
- border: 'dashed',
- color: (targetColor as string) || DEFAULT_TARGET_COLOR,
- },
+ const screenshot = screenshotForElement(
+ {
+ id: crypto.randomUUID(),
+ stepId,
+ blob,
+ mimeType: 'image/jpeg',
+ width: img.width,
+ height: img.height,
+ targetColor: targetColor as string,
},
- };
+ meta,
+ );
img.close();
await saveScreenshot(screenshot);
return screenshot.id;