π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
refactor: build the screenshot row once for both capture surfaces
Luis Ortizb191c7ed3965
diff --git a/AGENTS.md b/AGENTS.mdindex 86e2558..c34aae6 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -496,6 +496,16 @@ cannot interleave. For each click it hides the overlays, grabs the display, cropimplements `CaptureSink` and writes through `@mimik/core/guides/service`, exactly as the extension's`step-pipeline.ts` does.+The two surfaces reach that write by different routes but build the screenshot row the same way, so+`screenshotForElement` owns it. It turns an `ElementMeta` and whatever holds the bytes β a `Blob` in+the extension, a `src` on the desktop β into a row with the bounds in CSS pixels, the pixel ratio,+the click point, and the dashed target scaled by that ratio. Scaling the target is the part worth+having once: a rectangle multiplied in one surface and not the other puts the dashed box on the wrong+thing, and nothing about that fails a build. Everything else about the two paths genuinely differs β+voice narration, the deferred-description queue and the input finalisation exist only in the+extension, and the cursor mark, the application name and the fire-and-forget description exist only+on the desktop β so only the row builder is shared.+Main cannot `invoke` a renderer, so `ask()` sends a request with a generated reply channel and waitsfor `ipcMain.once` on it, with a timeout. The preload's `onRequest` is the other half. Guide creationand step writes both ride it, because both need IndexedDB, which only the renderer has.diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.tsindex b478b90..20f5036 100644--- a/apps/desktop/src/renderer/capture-sink.ts+++ b/apps/desktop/src/renderer/capture-sink.ts@@ -18,9 +18,8 @@ import {saveScreenshot,updateStepDescription,} from '@mimik/core/guides/service';-import type { StoredScreenshot } from '@mimik/core/guides/types';+import { screenshotForElement } from '@mimik/core/screenshot/record';import type { CursorMark } from '@mimik/core/screenshot/types';-import { DEFAULT_TARGET_COLOR } from '@mimik/core/screenshot/types';import { describeStep } from './ai';export type DesktopCaptureStepData = CaptureStepData & { cursor?: CursorMark; inputValue?: string };@@ -39,29 +38,19 @@ export class DesktopCaptureSink implements CaptureSink {let screenshotId: string | undefined;if (data.image) {const { targetColor } = await localStorage.get(['targetColor']);- const ratio = meta.devicePixelRatio;- const screenshot: StoredScreenshot = {- id: data.image.screenshotId,- src: data.image.src,- stepId,- mimeType: 'image/png',- width: data.image.width,- height: data.image.height,- bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },- pixelRatio: ratio,- clickPoint: meta.clickPoint,- edits: {+ const screenshot = screenshotForElement(+ {+ id: data.image.screenshotId,+ src: data.image.src,+ stepId,+ mimeType: 'image/png',+ width: data.image.width,+ height: data.image.height,cursor: data.cursor ?? null,- target: {- x: meta.rect.x * ratio,- y: meta.rect.y * ratio,- width: meta.rect.width * ratio,- height: meta.rect.height * ratio,- border: 'dashed',- color: targetColor || DEFAULT_TARGET_COLOR,- },+ targetColor,},- };+ meta,+ );await saveScreenshot(screenshot);screenshotId = screenshot.id;}diff --git a/packages/core/src/screenshot/__tests__/record.test.ts b/packages/core/src/screenshot/__tests__/record.test.tsnew file mode 100644index 0000000..cbf5ce1--- /dev/null+++ b/packages/core/src/screenshot/__tests__/record.test.ts@@ -0,0 +1,42 @@+import { describe, expect, it } from 'vitest';+import type { ElementMeta } from '@/core/guides/types';+import { screenshotForElement } from '@/core/screenshot/record';++const meta: ElementMeta = {+ textContent: null,+ ariaLabel: null,+ placeholder: null,+ altText: null,+ name: null,+ role: null,+ rect: { x: 10, y: 20, width: 100, height: 40 },+ devicePixelRatio: 2,+ clickPoint: { x: 60, y: 40 },+};++const bytes = { id: 'ss-1', stepId: 'step-1', mimeType: 'image/png', width: 800, height: 600 };++describe('screenshotForElement', () => {+ it('keeps bounds in css pixels and scales the target by the pixel ratio', () => {+ const shot = screenshotForElement(bytes, meta);+ expect(shot.bounds).toEqual({ x: 10, y: 20, width: 100, height: 40 });+ expect(shot.pixelRatio).toBe(2);+ expect(shot.edits?.target).toEqual({+ x: 20,+ y: 40,+ width: 200,+ height: 80,+ border: 'dashed',+ color: '#4F46E5',+ });+ });++ it('prefers the configured target colour', () => {+ expect(screenshotForElement({ ...bytes, targetColor: '#F43F5E' }, meta).edits?.target?.color).toBe('#F43F5E');+ });++ it('only writes a cursor when one is supplied', () => {+ expect(screenshotForElement(bytes, meta).edits).not.toHaveProperty('cursor');+ expect(screenshotForElement({ ...bytes, cursor: null }, meta).edits?.cursor).toBeNull();+ });+});diff --git a/packages/core/src/screenshot/record.ts b/packages/core/src/screenshot/record.tsnew file mode 100644index 0000000..3d0af1e--- /dev/null+++ b/packages/core/src/screenshot/record.ts@@ -0,0 +1,36 @@+import type { ElementMeta, StoredScreenshot } from '@/core/guides/types';+import { type CursorMark, DEFAULT_TARGET_COLOR } from '@/core/screenshot/types';++export interface ScreenshotBytes {+ id: string;+ stepId: string;+ mimeType: string;+ width: number;+ height: number;+ blob?: Blob;+ src?: string;+ cursor?: CursorMark | null;+ targetColor?: string;+}++export function screenshotForElement(bytes: ScreenshotBytes, meta: ElementMeta): StoredScreenshot {+ const { cursor, targetColor, ...rest } = bytes;+ const ratio = meta.devicePixelRatio;+ return {+ ...rest,+ bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },+ pixelRatio: ratio,+ clickPoint: meta.clickPoint,+ edits: {+ ...(cursor === undefined ? {} : { cursor }),+ target: {+ x: meta.rect.x * ratio,+ y: meta.rect.y * ratio,+ width: meta.rect.width * ratio,+ height: meta.rect.height * ratio,+ border: 'dashed',+ color: targetColor || DEFAULT_TARGET_COLOR,+ },+ },+ };+}diff --git a/src/entrypoints/background/step-pipeline.ts b/src/entrypoints/background/step-pipeline.tsindex 9548509..3447abb 100644--- a/src/entrypoints/background/step-pipeline.ts+++ b/src/entrypoints/background/step-pipeline.ts@@ -13,8 +13,8 @@ import {updateStepDescription,updateStepInputValue,} from '@/core/guides/service';-import type { ElementMeta, Screenshot } from '@/core/guides/types';-import { DEFAULT_TARGET_COLOR } from '@/core/screenshot/types';+import type { ElementMeta } from '@/core/guides/types';+import { screenshotForElement } from '@/core/screenshot/record';import { captureVisibleTab, localStorage } from '@/lib/browser-api';import type { CaptureStepData, CaptureStepResponse } from '@/lib/messaging';import { getActor } from './actor';@@ -29,27 +29,18 @@ async function takeScreenshot(stepId: string, meta: ElementMeta): Promise<stringconst dataUrl = await captureVisibleTab('jpeg', 90);const blob = await fetch(dataUrl).then((r) => r.blob());const img = await createImageBitmap(blob);- const screenshot: Screenshot = {- id: crypto.randomUUID(),- stepId,- blob,- mimeType: 'image/jpeg',- width: img.width,- height: img.height,- bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },- pixelRatio: meta.devicePixelRatio,- clickPoint: meta.clickPoint,- edits: {- target: {- x: meta.rect.x * meta.devicePixelRatio,- y: meta.rect.y * meta.devicePixelRatio,- width: meta.rect.width * meta.devicePixelRatio,- height: meta.rect.height * meta.devicePixelRatio,- border: 'dashed',- color: (targetColor as string) || DEFAULT_TARGET_COLOR,- },+ const screenshot = screenshotForElement(+ {+ id: crypto.randomUUID(),+ stepId,+ blob,+ mimeType: 'image/jpeg',+ width: img.width,+ height: img.height,+ targetColor: targetColor as string,},- };+ meta,+ );img.close();await saveScreenshot(screenshot);return screenshot.id;