π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat: capture engine with event detection, screenshots, and AI descriptions
Luis Ortiz9b79886fdd64
diff --git a/entrypoints/background.ts b/entrypoints/background.tsindex 2916480..1c27c3f 100644--- a/entrypoints/background.ts+++ b/entrypoints/background.ts@@ -1,33 +1,342 @@import { defineBackground } from 'wxt/utils/define-background';import { createActor } from 'xstate';import { captureMachine } from '../src/background/machine';+import { captureAnnotated } from '../src/background/screenshot';+import { buildFallbackDescription } from '../src/background/step-description';+import { getAIDescription } from '../src/background/ai-description';+import { db } from '../src/shared/db-schema';export default defineBackground(() => {chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true });+ let actorReadyResolve!: () => void;+ const actorReady = new Promise<void>(resolve => {+ actorReadyResolve = resolve;+ });+let actor: ReturnType<typeof createActor<typeof captureMachine>>;async function startActor() {- const result = await chrome.storage.session.get('machineSnapshot');- const snapshot = result.machineSnapshot ?? undefined;- actor = createActor(captureMachine, { snapshot });- actor.start();+ try {+ const result = await chrome.storage.session.get('machineSnapshot');+ const snapshot = result.machineSnapshot ?? undefined;+ actor = createActor(captureMachine, { snapshot });+ actor.start();+ } catch (err) {+ console.warn('[Mimik] Failed to restore snapshot, starting fresh', err);+ await chrome.storage.session.remove('machineSnapshot');+ actor = createActor(captureMachine);+ actor.start();+ }actor.subscribe(() => {const persisted = actor.getPersistedSnapshot();chrome.storage.session.set({ machineSnapshot: persisted });});++ actorReadyResolve();}- startActor();+ startActor().catch((err) => {+ console.error('[Mimik] startActor failed', err);+ actor = createActor(captureMachine);+ actor.start();+ actorReadyResolve();+ });- chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => {+ let captureQueue: Promise<void> = Promise.resolve();++ function enqueueCaptureStep(fn: () => Promise<void>): void {+ captureQueue = captureQueue.then(fn).catch(err =>+ console.error('[Mimik] Capture error', err)+ );+ }++ function isInjectableTab(tab: chrome.tabs.Tab): boolean {+ const url = tab.url || tab.pendingUrl || '';+ if (!url || url.startsWith('chrome://') || url.startsWith('chrome-extension://') ||+ url.startsWith('chrome.google.com/webstore') || url.startsWith('about:')) return false;+ return /^https?:/.test(url);+ }++ async function injectContentScript(tabId: number): Promise<void> {+ try {+ await chrome.tabs.sendMessage(tabId, { type: 'PING' });+ } catch {+ try {+ await chrome.scripting.executeScript({+ target: { tabId, allFrames: true },+ files: ['content-scripts/content.js'],+ });+ } catch (err) {+ }+ }+ }++ async function injectAllTabs(): Promise<void> {+ try {+ const windows = await chrome.windows.getAll({ populate: true });+ for (const win of windows) {+ for (const tab of win.tabs || []) {+ if (tab.id && isInjectableTab(tab)) {+ injectContentScript(tab.id);+ }+ }+ }+ } catch (err) {+ console.warn('[Mimik] injectAllTabs failed', err);+ }+ }++ async function broadcastStartCapture(guideId: string): Promise<void> {+ try {+ const tabs = await chrome.tabs.query({});+ for (const tab of tabs) {+ if (tab.id && isInjectableTab(tab)) {+ chrome.tabs.sendMessage(tab.id, { type: 'START_CAPTURE', guideId }).catch(() => {});+ }+ }+ } catch (err) {+ console.warn('[Mimik] broadcastStartCapture failed', err);+ }+ }++ async function broadcastStopCapture(): Promise<void> {+ try {+ const tabs = await chrome.tabs.query({});+ for (const tab of tabs) {+ if (tab.id) {+ chrome.tabs.sendMessage(tab.id, { type: 'STOP_CAPTURE' }).catch(() => {});+ }+ }+ } catch (err) {+ console.warn('[Mimik] broadcastStopCapture failed', err);+ }+ }++ injectAllTabs();++ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {if (msg.type === 'PING') {sendResponse({ alive: true });+ return true;}+if (msg.type === 'GET_STATE') {- sendResponse({ state: actor?.getSnapshot().value ?? 'unknown' });+ (async () => {+ try {+ await actorReady;+ const snap = actor.getSnapshot();+ sendResponse({+ state: snap.value ?? 'unknown',+ stepCount: snap.context.stepCount,+ currentGuideId: snap.context.currentGuideId,+ });+ } catch (err) {+ console.error('[Mimik] GET_STATE failed', err);+ sendResponse({ state: 'unknown' });+ }+ })();+ return true;+ }++ if (msg.type === 'START_RECORDING') {+ (async () => {+ try {+ await actorReady;+ actor.send({ type: 'START_RECORDING', url: msg.url });++ const guideId = actor.getSnapshot().context.currentGuideId!;++ await db.guides.add({+ id: guideId,+ title: 'Untitled Guide',+ createdAt: Date.now(),+ updatedAt: Date.now(),+ stepIds: [],+ });++ await broadcastStartCapture(guideId);++ sendResponse({ guideId });+ } catch (err) {+ console.error('[Mimik] START_RECORDING failed', err);+ sendResponse({ error: String(err) });+ }+ })();+ return true;+ }++ if (msg.type === 'STOP_RECORDING') {+ (async () => {+ try {+ await actorReady;++ await broadcastStopCapture();++ actor.send({ type: 'STOP_RECORDING' });+ sendResponse({ success: true });+ } catch (err) {+ console.error('[Mimik] STOP_RECORDING failed', err);+ sendResponse({ error: String(err) });+ }+ })();+ return true;+ }++ if (msg.type === 'USER_ACTION') {+ (async () => {+ await actorReady;++ const snap = actor.getSnapshot();+ if (snap.value !== 'recording') {+ sendResponse({ ignored: true });+ return;+ }++ actor.send({ type: 'USER_ACTION' });++ const stepCount = actor.getSnapshot().context.stepCount;+ const guideId = snap.context.currentGuideId!;+ const stepId = crypto.randomUUID();+ const tabId = sender.tab?.id;++ if (!tabId) {+ sendResponse({ error: 'no tabId' });+ return;+ }++ sendResponse({ stepId });++ enqueueCaptureStep(async () => {+ const fallbackDescription = buildFallbackDescription(msg.action, msg.elementMeta);+ const currentUrl = snap.context.currentUrl;++ let screenshotId: string | undefined;+ let screenshotBlob: Blob | undefined;+ try {+ const screenshot = await captureAnnotated(tabId, stepId, msg.elementMeta);+ screenshotId = screenshot.id;+ screenshotBlob = screenshot.blob;+ } catch (err) {+ console.warn('[Mimik] Screenshot capture failed, storing step without screenshot', err);+ }++ await db.steps.add({+ id: stepId,+ guideId,+ index: stepCount,+ description: fallbackDescription,+ action: msg.action,+ url: currentUrl,+ timestamp: Date.now(),+ screenshotId,+ });++ const guide = await db.guides.get(guideId);+ if (guide) {+ await db.guides.update(guideId, {+ stepIds: [...guide.stepIds, stepId],+ updatedAt: Date.now(),+ });+ }++ if (screenshotBlob) {+ (async () => {+ try {+ const settings = await chrome.storage.local.get(['aiApiKey', 'aiProvider']);+ if (settings.aiApiKey) {+ const provider = (settings.aiProvider as 'openai' | 'anthropic') || 'openai';+ const aiDescription = await getAIDescription(+ screenshotBlob,+ msg.action,+ msg.elementMeta,+ provider,+ settings.aiApiKey+ );+ if (aiDescription) {+ await db.steps.update(stepId, { description: aiDescription });+ }+ }+ } catch (err) {+ console.error('[Mimik] AI description update failed', err);+ }+ })();+ }+ });+ })();+ return true;+ }++ if (msg.type === 'RRWEB_CHUNK') {+ (async () => {+ await db.rrwebEvents.add({+ id: crypto.randomUUID(),+ guideId: msg.guideId,+ events: msg.events,+ timestamp: msg.timestamp,+ });+ sendResponse({ stored: true });+ })();+ return true;+ }++ return false;+ });++ chrome.webNavigation.onCompleted.addListener(async (details) => {+ if (details.frameId !== 0) return;+ await actorReady;+ const state = actor.getSnapshot();+ if (state.value === 'recording') {+ actor.send({ type: 'SPA_NAVIGATE', url: details.url });+ }+ });++ chrome.webNavigation.onHistoryStateUpdated.addListener(async (details) => {+ if (details.frameId !== 0) return;+ await actorReady;+ const state = actor.getSnapshot();+ if (state.value === 'recording') {+ actor.send({ type: 'SPA_NAVIGATE', url: details.url });+ }+ });++ chrome.tabs.onActivated.addListener(async (activeInfo) => {+ await actorReady;+ const state = actor.getSnapshot();+ if (state.value !== 'recording') return;++ const guideId = state.context.currentGuideId;+ if (!guideId) return;++ try {+ await chrome.tabs.sendMessage(activeInfo.tabId, { type: 'PING' });+ } catch {+ try {+ const tab = await chrome.tabs.get(activeInfo.tabId);+ if (isInjectableTab(tab)) {+ await injectContentScript(activeInfo.tabId);+ }+ } catch {+ }+ }+ });++ chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, tab) => {+ if (changeInfo.status !== 'complete') return;+ await actorReady;+ const state = actor.getSnapshot();+ if (state.value !== 'recording') return;+ if (!isInjectableTab(tab)) return;++ try {+ await chrome.tabs.sendMessage(tabId, { type: 'PING' });+ } catch {+ try {+ await injectContentScript(tabId);+ } catch {+ }}- return true;});});diff --git a/entrypoints/content.ts b/entrypoints/content.tsindex 82953f7..d4b840a 100644--- a/entrypoints/content.ts+++ b/entrypoints/content.ts@@ -1,8 +1,98 @@import { defineContentScript } from 'wxt/utils/define-content-script';+import { startCapture } from '../src/content/events';+import { startRrwebRecording } from '../src/content/rrweb-recorder';+import { updateUrl } from '../src/content/spa-nav';++const CLEANUP_EVENT = `mimik_remove_content_script_${chrome.runtime.id}`;export default defineContentScript({matches: ['<all_urls>'],+ allFrames: true,+ matchAboutBlank: true,+ runAt: 'document_idle',+main() {+ document.dispatchEvent(new CustomEvent(CLEANUP_EVENT));++ let stopCapture: (() => void) | null = null;+ let stopRrweb: (() => void) | null = null;+ let destroyed = false;++ function beginCapture(guideId: string) {+ if (destroyed) return;+ stopCapture?.();+ stopRrweb?.();+ stopCapture = startCapture(guideId);+ stopRrweb = startRrwebRecording(guideId);+ }++ function endCapture() {+ stopCapture?.();+ stopRrweb?.();+ stopCapture = null;+ stopRrweb = null;+ }++ function cleanup() {+ destroyed = true;+ endCapture();+ chrome.runtime.onMessage.removeListener(messageHandler);+ document.removeEventListener(CLEANUP_EVENT, cleanup);+ }++ document.addEventListener(CLEANUP_EVENT, cleanup);++ window.addEventListener('beforeunload', () => {+ endCapture();+ });++ function messageHandler(msg: any, _sender: any, sendResponse: (r: any) => void) {+ if (destroyed) return false;++ if (msg.type === 'PING') {+ sendResponse({ alive: true });+ return true;+ }++ if (msg.type === 'GET_ROUTE') {+ sendResponse({ alive: true, capturing: !!stopCapture });+ return true;+ }++ if (msg.type === 'START_CAPTURE' && msg.guideId) {+ beginCapture(msg.guideId);+ sendResponse({ started: true });+ return true;+ }++ if (msg.type === 'STOP_CAPTURE') {+ endCapture();+ sendResponse({ stopped: true });+ return true;+ }++ if (msg.type === 'SPA_NAVIGATE' && msg.url) {+ updateUrl(msg.url);+ sendResponse({ updated: true });+ return true;+ }++ return false;+ }++ chrome.runtime.onMessage.addListener(messageHandler);++ try {+ chrome.runtime.sendMessage({ type: 'GET_STATE' }, (response) => {+ if (chrome.runtime.lastError) return; // Extension context invalid+ if (destroyed) return;+ if (response?.state === 'recording' && response.currentGuideId) {+ beginCapture(response.currentGuideId);+ }+ });+ } catch {+ }+console.log('[Mimik] Content script loaded');},});diff --git a/package-lock.json b/package-lock.jsonindex e5010a1..d01bc22 100644--- a/package-lock.json+++ b/package-lock.json@@ -9,14 +9,18 @@"version": "1.0.0","license": "ISC","dependencies": {+ "@anthropic-ai/sdk": "^0.80.0","@tailwindcss/vite": "^4.2.2","@wxt-dev/module-react": "^1.2.2","@xstate/react": "^6.1.0","clsx": "^2.1.1",+ "css-selector-generator": "^3.8.1","dexie": "^4.3.0","lucide-react": "^0.577.0",+ "openai": "^6.32.0","react": "^19.2.4","react-dom": "^19.2.4",+ "rrweb": "^1.1.3","tailwind-merge": "^3.5.0","tailwindcss": "^4.2.2","wxt": "^0.20.20",@@ -142,6 +146,26 @@"url": "https://github.com/sponsors/jonschlinkert"}},+ "node_modules/@anthropic-ai/sdk": {+ "version": "0.80.0",+ "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.80.0.tgz",+ "integrity": "sha512-WeXLn7zNVk3yjeshn+xZHvld6AoFUOR3Sep6pSoHho5YbSi6HwcirqgPA5ccFuW8QTVJAAU7N8uQQC6Wa9TG+g==",+ "license": "MIT",+ "dependencies": {+ "json-schema-to-ts": "^3.1.1"+ },+ "bin": {+ "anthropic-ai-sdk": "bin/cli"+ },+ "peerDependencies": {+ "zod": "^3.25.0 || ^4.0.0"+ },+ "peerDependenciesMeta": {+ "zod": {+ "optional": true+ }+ }+ },"node_modules/@asamuzakjp/css-color": {"version": "5.0.1","resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.0.1.tgz",@@ -1641,6 +1665,12 @@"assertion-error": "^2.0.1"}},+ "node_modules/@types/css-font-loading-module": {+ "version": "0.0.7",+ "resolved": "https://registry.npmjs.org/@types/css-font-loading-module/-/css-font-loading-module-0.0.7.tgz",+ "integrity": "sha512-nl09VhutdjINdWyXxHWN/w9zlNCfr60JUqJbd24YXUuCwgeL0TpFSdElCwb6cxfB6ybE19Gjj4g0jsgkXxKv1Q==",+ "license": "MIT"+ },"node_modules/@types/deep-eql": {"version": "4.0.2","resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz",@@ -1929,6 +1959,12 @@"url": "https://github.com/sponsors/wxt-dev"}},+ "node_modules/@xstate/fsm": {+ "version": "1.6.5",+ "resolved": "https://registry.npmjs.org/@xstate/fsm/-/fsm-1.6.5.tgz",+ "integrity": "sha512-b5o1I6aLNeYlU/3CPlj/Z91ybk1gUsKT+5NAJI+2W4UjvS5KLG28K9v5UvNoFVjHV8PajVZ00RH3vnjyQO7ZAw==",+ "license": "MIT"+ },"node_modules/@xstate/react": {"version": "6.1.0","resolved": "https://registry.npmjs.org/@xstate/react/-/react-6.1.0.tgz",@@ -2148,6 +2184,15 @@"node": "18 || 20 || >=22"}},+ "node_modules/base64-arraybuffer": {+ "version": "1.0.2",+ "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz",+ "integrity": "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==",+ "license": "MIT",+ "engines": {+ "node": ">= 0.6.0"+ }+ },"node_modules/bidi-js": {"version": "1.0.3","resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",@@ -2741,6 +2786,12 @@"url": "https://github.com/sponsors/fb55"}},+ "node_modules/css-selector-generator": {+ "version": "3.8.1",+ "resolved": "https://registry.npmjs.org/css-selector-generator/-/css-selector-generator-3.8.1.tgz",+ "integrity": "sha512-3gT7VgYfhH6uLNdho9Z4lpaoUcFc6pciJwgfBbBrmuvNnTnugJ9+WJRFJkW+cGh2uHCxaIyBAk2pfhiqRP047A==",+ "license": "MIT"+ },"node_modules/css-tree": {"version": "3.2.1","resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz",@@ -3269,6 +3320,12 @@}}},+ "node_modules/fflate": {+ "version": "0.4.8",+ "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz",+ "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==",+ "license": "MIT"+ },"node_modules/filesize": {"version": "11.0.13","resolved": "https://registry.npmjs.org/filesize/-/filesize-11.0.13.tgz",@@ -3863,6 +3920,19 @@"node": "^14.17.0 || ^16.13.0 || >=18.0.0"}},+ "node_modules/json-schema-to-ts": {+ "version": "3.1.1",+ "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz",+ "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==",+ "license": "MIT",+ "dependencies": {+ "@babel/runtime": "^7.18.3",+ "ts-algebra": "^2.0.0"+ },+ "engines": {+ "node": ">=16"+ }+ },"node_modules/json5": {"version": "2.2.3","resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",@@ -4576,6 +4646,12 @@"url": "https://github.com/sponsors/ljharb"}},+ "node_modules/mitt": {+ "version": "1.2.0",+ "resolved": "https://registry.npmjs.org/mitt/-/mitt-1.2.0.tgz",+ "integrity": "sha512-r6lj77KlwqLhIUku9UWYes7KJtsczvolZkzp8hbaDPPaE24OmWl5s539Mytlj22siEQKosZ26qCBgda2PKwoJw==",+ "license": "MIT"+ },"node_modules/mlly": {"version": "1.8.2","resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz",@@ -4882,6 +4958,27 @@"url": "https://github.com/sponsors/sindresorhus"}},+ "node_modules/openai": {+ "version": "6.32.0",+ "resolved": "https://registry.npmjs.org/openai/-/openai-6.32.0.tgz",+ "integrity": "sha512-j3k+BjydAf8yQlcOI7WUQMQTbbF5GEIMAE2iZYCOzwwB3S2pCheaWYp+XZRNAch4jWVc52PMDGRRjutao3lLCg==",+ "license": "Apache-2.0",+ "bin": {+ "openai": "bin/cli"+ },+ "peerDependencies": {+ "ws": "^8.18.0",+ "zod": "^3.25 || ^4.0"+ },+ "peerDependenciesMeta": {+ "ws": {+ "optional": true+ },+ "zod": {+ "optional": true+ }+ }+ },"node_modules/os-shim": {"version": "0.1.3","resolved": "https://registry.npmjs.org/os-shim/-/os-shim-0.1.3.tgz",@@ -5483,6 +5580,26 @@"integrity": "sha512-UkVDEFk1w3mveXeKgaTuYfKWtPbvgck1dT8TUG3bnccrH0XtLTuAyfCoks4Q/M5ZGToSVJTIQYCzy2g/atAOeg==","license": "MIT"},+ "node_modules/rrweb": {+ "version": "1.1.3",+ "resolved": "https://registry.npmjs.org/rrweb/-/rrweb-1.1.3.tgz",+ "integrity": "sha512-F2qp8LteJLyycsv+lCVJqtVpery63L3U+/ogqMA0da8R7Jx57o6gT+HpjrzdeeGMIBZR7kKNaKyJwDupTTu5KA==",+ "license": "MIT",+ "dependencies": {+ "@types/css-font-loading-module": "0.0.7",+ "@xstate/fsm": "^1.4.0",+ "base64-arraybuffer": "^1.0.1",+ "fflate": "^0.4.4",+ "mitt": "^1.1.3",+ "rrweb-snapshot": "^1.1.14"+ }+ },+ "node_modules/rrweb-snapshot": {+ "version": "1.1.14",+ "resolved": "https://registry.npmjs.org/rrweb-snapshot/-/rrweb-snapshot-1.1.14.tgz",+ "integrity": "sha512-eP5pirNjP5+GewQfcOQY4uBiDnpqxNRc65yKPW0eSoU1XamDfc4M8oqpXGMyUyvLyxFDB0q0+DChuxxiU2FXBQ==",+ "license": "MIT"+ },"node_modules/run-applescript": {"version": "7.1.0","resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz",@@ -5997,6 +6114,12 @@"node": ">=20"}},+ "node_modules/ts-algebra": {+ "version": "2.0.0",+ "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz",+ "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",+ "license": "MIT"+ },"node_modules/tslib": {"version": "2.8.1","resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",@@ -6576,7 +6699,7 @@"version": "8.19.0","resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz","integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==",- "dev": true,+ "devOptional": true,"license": "MIT","engines": {"node": ">=10.0.0"diff --git a/package.json b/package.jsonindex 5721fa1..1c868b6 100644--- a/package.json+++ b/package.json@@ -10,14 +10,18 @@"author": "","license": "ISC","dependencies": {+ "@anthropic-ai/sdk": "^0.80.0","@tailwindcss/vite": "^4.2.2","@wxt-dev/module-react": "^1.2.2","@xstate/react": "^6.1.0","clsx": "^2.1.1",+ "css-selector-generator": "^3.8.1","dexie": "^4.3.0","lucide-react": "^0.577.0",+ "openai": "^6.32.0","react": "^19.2.4","react-dom": "^19.2.4",+ "rrweb": "^1.1.3","tailwind-merge": "^3.5.0","tailwindcss": "^4.2.2","wxt": "^0.20.20",diff --git a/src/background/ai-description.ts b/src/background/ai-description.tsnew file mode 100644index 0000000..61e2f5a--- /dev/null+++ b/src/background/ai-description.ts@@ -0,0 +1,56 @@+import OpenAI from 'openai';+import type { ElementMeta } from '../shared/types';++export async function getAIDescription(+ blob: Blob,+ action: string,+ meta: ElementMeta,+ provider: 'openai' | 'anthropic',+ apiKey: string+): Promise<string | null> {+ const arrayBuffer = await blob.arrayBuffer();+ const base64 = btoa(String.fromCharCode(...new Uint8Array(arrayBuffer)));++ const targetText = meta.textContent || meta.ariaLabel || meta.placeholder || meta.tag;+ const prompt = `This is a browser screenshot. The highlighted element (blue border) is a ${meta.tag} with text "${targetText}". The user performed a "${action}" action on it. Write a single concise sentence describing this step (e.g., "Click the Submit button"). No preamble.`;++ try {+ if (provider === 'openai') {+ const client = new OpenAI({ apiKey, dangerouslyAllowBrowser: true });+ const response = await client.chat.completions.create({+ model: 'gpt-4o-mini',+ max_tokens: 100,+ messages: [{+ role: 'user',+ content: [+ { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${base64}` } },+ { type: 'text', text: prompt },+ ],+ }],+ });+ return response.choices[0]?.message?.content?.trim() || null;+ } else {+ const { default: Anthropic } = await import('@anthropic-ai/sdk');+ const client = new Anthropic({ apiKey });+ const response = await client.messages.create({+ model: 'claude-3-5-haiku-20241022',+ max_tokens: 100,+ messages: [{+ role: 'user',+ content: [+ {+ type: 'image',+ source: { type: 'base64', media_type: 'image/jpeg', data: base64 },+ },+ { type: 'text', text: prompt },+ ],+ }],+ });+ const block = response.content[0];+ return block.type === 'text' ? block.text.trim() : null;+ }+ } catch (err) {+ console.error('[Mimik] AI description failed, using fallback', err);+ return null;+ }+}diff --git a/src/background/machine.ts b/src/background/machine.tsindex 4dd7353..2e4c29d 100644--- a/src/background/machine.ts+++ b/src/background/machine.ts@@ -6,6 +6,7 @@ export const captureMachine = createMachine({context: {currentGuideId: null as string | null,stepCount: 0,+ currentUrl: '' as string,},states: {idle: {@@ -15,6 +16,7 @@ export const captureMachine = createMachine({actions: assign({currentGuideId: () => crypto.randomUUID(),stepCount: 0,+ currentUrl: ({ event }) => (event as { type: 'START_RECORDING'; url?: string }).url ?? '',}),},},@@ -26,6 +28,17 @@ export const captureMachine = createMachine({actions: assign({currentGuideId: null,stepCount: 0,+ currentUrl: '',+ }),+ },+ USER_ACTION: {+ actions: assign({+ stepCount: ({ context }) => context.stepCount + 1,+ }),+ },+ SPA_NAVIGATE: {+ actions: assign({+ currentUrl: ({ event }) => (event as { type: 'SPA_NAVIGATE'; url: string }).url,}),},},diff --git a/src/background/screenshot.ts b/src/background/screenshot.tsindex a5ea850..b6fc520 100644--- a/src/background/screenshot.ts+++ b/src/background/screenshot.ts@@ -1,27 +1,64 @@import { db } from '../shared/db-schema';-import type { Screenshot } from '../shared/types';+import type { Screenshot, ElementMeta } from '../shared/types';export async function captureAndStore(tabId: number, stepId: string): Promise<Screenshot> {+ const dataUrl = await chrome.tabs.captureVisibleTab(null, { format: 'jpeg', quality: 85 });+ const blob = await fetch(dataUrl).then(r => r.blob());+ const screenshot: Screenshot = {+ id: crypto.randomUUID(), stepId, blob, mimeType: 'image/jpeg', width: 0, height: 0,+ };+ await db.screenshots.add(screenshot);+ return screenshot;+}++export function dataUrlToBlob(dataUrl: string): Promise<Blob> {+ return fetch(dataUrl).then(r => r.blob());+}++export async function captureAnnotated(+ tabId: number,+ stepId: string,+ elementMeta: ElementMeta+): Promise<Screenshot> {const dataUrl = await chrome.tabs.captureVisibleTab(null, {format: 'jpeg',quality: 85,});- const blob = await fetch(dataUrl).then(r => r.blob());+ const annotatedBlob = await drawHighlight(dataUrl, elementMeta);+ const img = await createImageBitmap(annotatedBlob);const screenshot: Screenshot = {id: crypto.randomUUID(),stepId,- blob,+ blob: annotatedBlob,mimeType: 'image/jpeg',- width: 0,- height: 0,+ width: img.width,+ height: img.height,};await db.screenshots.add(screenshot);return screenshot;}-export function dataUrlToBlob(dataUrl: string): Promise<Blob> {- return fetch(dataUrl).then(r => r.blob());+export async function drawHighlight(dataUrl: string, meta: ElementMeta): Promise<Blob> {+ const img = await createImageBitmap(await fetch(dataUrl).then(r => r.blob()));++ const canvas = new OffscreenCanvas(img.width, img.height);+ const ctx = canvas.getContext('2d')!;+ ctx.drawImage(img, 0, 0);++ const dpr = meta.devicePixelRatio || 1;+ const x = meta.rect.x * dpr;+ const y = meta.rect.y * dpr;+ const w = meta.rect.width * dpr;+ const h = meta.rect.height * dpr;++ ctx.strokeStyle = '#2563EB';+ ctx.lineWidth = 3 * dpr;+ ctx.fillStyle = 'rgba(37, 99, 235, 0.15)';+ ctx.fillRect(x, y, w, h);+ ctx.strokeRect(x, y, w, h);++ return canvas.convertToBlob({ type: 'image/jpeg', quality: 0.85 });}diff --git a/src/background/step-description.ts b/src/background/step-description.tsnew file mode 100644index 0000000..51bb59b--- /dev/null+++ b/src/background/step-description.ts@@ -0,0 +1,33 @@+import type { ElementMeta } from '../shared/types';++export function buildFallbackDescription(action: string, meta: ElementMeta): string {+ const target = meta.ariaLabel+ || meta.placeholder+ || meta.textContent?.slice(0, 80)+ || meta.altText+ || meta.name+ || meta.role+ || meta.tag;++ if (action.startsWith('keydown:')) {+ const key = action.split(':')[1];+ return `Press ${key} on ${target}`;+ }++ switch (action) {+ case 'click':+ if (meta.tag === 'input' && meta.inputType === 'checkbox') return `Toggle checkbox ${target}`;+ if (meta.tag === 'input' && meta.inputType === 'radio') return `Select radio ${target}`;+ if (meta.href) return `Click link "${target}"`;+ return `Click ${target}`;+ case 'input':+ if (meta.inputType) return `Type into ${meta.inputType} field ${target}`;+ return `Type into ${target}`;+ case 'scroll':+ return 'Scroll the page';+ case 'navigate':+ return 'Navigate to page';+ default:+ return `${action} ${target}`;+ }+}diff --git a/src/content/element-meta.ts b/src/content/element-meta.tsnew file mode 100644index 0000000..c4ca16e--- /dev/null+++ b/src/content/element-meta.ts@@ -0,0 +1,27 @@+import { getCssSelector } from 'css-selector-generator';+import type { ElementMeta } from '../shared/types';++export function extractElementMeta(el: HTMLElement): ElementMeta {+ const rect = el.getBoundingClientRect();+ let cssSelector: string;+ try {+ cssSelector = getCssSelector(el);+ } catch {+ cssSelector = el.tagName?.toLowerCase() ?? 'unknown';+ }+ return {+ tag: el.tagName?.toLowerCase() ?? 'unknown',+ cssSelector,+ textContent: el.textContent?.trim().slice(0, 200) || null,+ ariaLabel: el.getAttribute('aria-label'),+ placeholder: el.getAttribute('placeholder'),+ altText: el instanceof HTMLImageElement ? el.alt : null,+ name: el.getAttribute('name'),+ role: el.getAttribute('role') || (el.tagName?.toLowerCase() ?? null),+ href: el instanceof HTMLAnchorElement ? el.href : null,+ inputType: el instanceof HTMLInputElement ? el.type : null,+ dataTestId: el.getAttribute('data-testid') || el.getAttribute('data-test-id') || el.getAttribute('data-qa') || null,+ rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },+ devicePixelRatio: window.devicePixelRatio,+ };+}diff --git a/src/content/events.ts b/src/content/events.tsnew file mode 100644index 0000000..f3d8177--- /dev/null+++ b/src/content/events.ts@@ -0,0 +1,104 @@+import { extractElementMeta } from './element-meta';++const FOCUSABLE_SELECTOR = 'a[href], button, input, select, textarea, [role="button"], [role="link"], [role="tab"], [role="menuitem"], [role="checkbox"], [role="radio"], [tabindex]';++const DEDUP_MS = 200;+let lastClickTarget: Element | null = null;+let lastClickTime = 0;++function findFocusableAncestor(el: Element): HTMLElement {+ const focusable = el.closest(FOCUSABLE_SELECTOR);+ if (focusable instanceof HTMLElement) return focusable;++ if (el instanceof HTMLElement) return el;+ if (el.parentElement instanceof HTMLElement) return el.parentElement;++ return document.body;+}++export function startCapture(guideId: string): () => void {+ let scrollTimer: ReturnType<typeof setTimeout> | null = null;++ function sendAction(action: string, meta: ReturnType<typeof extractElementMeta>) {+ try {+ chrome.runtime.sendMessage({+ type: 'USER_ACTION',+ guideId,+ action,+ elementMeta: meta,+ });+ } catch (err) {+ console.warn('[Mimik] Failed to send action', err);+ }+ }++ function handleClick(e: MouseEvent) {+ const rawTarget = e.target;+ if (!rawTarget || !(rawTarget instanceof Element)) return;++ const target = findFocusableAncestor(rawTarget);+ if (target.closest('[data-mimik-ignore]')) return;++ const now = Date.now();+ if (target === lastClickTarget && now - lastClickTime < DEDUP_MS) return;+ lastClickTarget = target;+ lastClickTime = now;++ try {+ const meta = extractElementMeta(target);+ sendAction('click', meta);+ } catch (err) {+ console.warn('[Mimik] Failed to capture click', err);+ }+ }++ function handleKeydown(e: KeyboardEvent) {+ if (!['Enter', 'Escape', 'Tab'].includes(e.key)) return;+ const target = e.target instanceof HTMLElement ? e.target : document.activeElement;+ if (!target || !(target instanceof HTMLElement)) return;++ try {+ const meta = extractElementMeta(target);+ sendAction(`keydown:${e.key}`, meta);+ } catch (err) {+ console.warn('[Mimik] Failed to capture keydown', err);+ }+ }++ function handleInput(e: Event) {+ if (!(e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement || e.target instanceof HTMLSelectElement)) return;+ try {+ const meta = extractElementMeta(e.target as HTMLElement);+ sendAction('input', meta);+ } catch (err) {+ console.warn('[Mimik] Failed to capture input', err);+ }+ }++ function handleScroll() {+ if (scrollTimer) clearTimeout(scrollTimer);+ scrollTimer = setTimeout(() => {+ sendAction('scroll', {+ tag: 'window', cssSelector: 'window', textContent: null,+ ariaLabel: null, placeholder: null, altText: null,+ name: null, role: 'scrollbar', href: null,+ inputType: null, dataTestId: null,+ rect: { x: 0, y: 0, width: window.innerWidth, height: window.innerHeight },+ devicePixelRatio: window.devicePixelRatio,+ });+ }, 500);+ }++ window.addEventListener('click', handleClick, { capture: true });+ window.addEventListener('keydown', handleKeydown, { capture: true });+ window.addEventListener('input', handleInput, { capture: true });+ window.addEventListener('scroll', handleScroll, { capture: true, passive: true });++ return () => {+ window.removeEventListener('click', handleClick, { capture: true });+ window.removeEventListener('keydown', handleKeydown, { capture: true });+ window.removeEventListener('input', handleInput, { capture: true });+ window.removeEventListener('scroll', handleScroll, { capture: true });+ if (scrollTimer) clearTimeout(scrollTimer);+ };+}diff --git a/src/content/rrweb-recorder.ts b/src/content/rrweb-recorder.tsnew file mode 100644index 0000000..236f826--- /dev/null+++ b/src/content/rrweb-recorder.ts@@ -0,0 +1,41 @@+// @ts-expect-error rrweb 1.1.3 type resolution+import { record } from 'rrweb';++const MAX_BUFFER_SIZE = 10_000;++export function startRrwebRecording(guideId: string): () => void {+ let eventBuffer: unknown[] = [];+ let flushTimer: ReturnType<typeof setInterval>;++ function flushChunk() {+ if (eventBuffer.length === 0) return;+ const chunk = eventBuffer.splice(0);+ chrome.runtime.sendMessage({+ type: 'RRWEB_CHUNK',+ guideId,+ events: chunk,+ timestamp: Date.now(),+ });+ }++ const stopRecord = record({+ emit(event: unknown, isCheckout?: boolean) {+ eventBuffer.push(event);+ if (eventBuffer.length > MAX_BUFFER_SIZE) {+ eventBuffer = eventBuffer.slice(-MAX_BUFFER_SIZE);+ }+ if (isCheckout) {+ flushChunk();+ }+ },+ checkoutEveryNms: 30_000,+ });++ flushTimer = setInterval(flushChunk, 30_000);++ return () => {+ flushChunk();+ clearInterval(flushTimer);+ stopRecord?.();+ };+}diff --git a/src/content/spa-nav.ts b/src/content/spa-nav.tsnew file mode 100644index 0000000..8c4fa5e--- /dev/null+++ b/src/content/spa-nav.ts@@ -0,0 +1,9 @@+let currentUrl = window.location.href;++export function getCurrentUrl(): string {+ return currentUrl;+}++export function updateUrl(url: string): void {+ currentUrl = url;+}diff --git a/src/shared/db-schema.ts b/src/shared/db-schema.tsindex 425d0d0..c5fc11d 100644--- a/src/shared/db-schema.ts+++ b/src/shared/db-schema.ts@@ -1,10 +1,11 @@import Dexie, { type EntityTable } from 'dexie';-import type { Guide, Step, Screenshot } from './types';+import type { Guide, Step, Screenshot, RrwebEventChunk } from './types';export class MimikDB extends Dexie {guides!: EntityTable<Guide, 'id'>;steps!: EntityTable<Step, 'id'>;screenshots!: EntityTable<Screenshot, 'id'>;+ rrwebEvents!: EntityTable<RrwebEventChunk, 'id'>;constructor() {super('mimik');@@ -13,6 +14,12 @@ export class MimikDB extends Dexie {steps: 'id, guideId, index',screenshots: 'id, stepId',});+ this.version(2).stores({+ guides: 'id, createdAt, updatedAt',+ steps: 'id, guideId, index',+ screenshots: 'id, stepId',+ rrwebEvents: 'id, guideId, timestamp',+ });}}diff --git a/src/shared/messages.ts b/src/shared/messages.tsindex 340d01a..e0fd119 100644--- a/src/shared/messages.ts+++ b/src/shared/messages.ts@@ -1,3 +1,5 @@+import type { ElementMeta } from './types';+export type MessageType = 'PING' | 'CAPTURE_SCREENSHOT' | 'GET_STATE';export interface PingMessage { type: 'PING'; }@@ -9,4 +11,25 @@ export interface CaptureScreenshotResponse { screenshotId: string; }export interface GetStateMessage { type: 'GET_STATE'; }export interface GetStateResponse { state: string; }-export type ExtensionMessage = PingMessage | CaptureScreenshotMessage | GetStateMessage;+export interface StartRecordingMessage { type: 'START_RECORDING'; url: string; }+export interface StopRecordingMessage { type: 'STOP_RECORDING'; }+export interface UserActionMessage {+ type: 'USER_ACTION';+ guideId: string;+ action: 'click' | 'input' | 'scroll';+ elementMeta: ElementMeta;+}+export interface RrwebChunkMessage {+ type: 'RRWEB_CHUNK';+ guideId: string;+ events: unknown[];+ timestamp: number;+}+export interface SpaNavigateMessage {+ type: 'SPA_NAVIGATE';+ url: string;+ guideId: string;+}++export type ExtensionMessage = PingMessage | CaptureScreenshotMessage | GetStateMessage+ | StartRecordingMessage | StopRecordingMessage | UserActionMessage | RrwebChunkMessage | SpaNavigateMessage;diff --git a/src/shared/types.ts b/src/shared/types.tsindex eda0651..cab6a8d 100644--- a/src/shared/types.ts+++ b/src/shared/types.ts@@ -11,10 +11,10 @@ export interface Step {guideId: string;index: number;description: string;- action: 'click' | 'input' | 'scroll' | 'navigate';+ action: string;url: string;timestamp: number;- screenshotId: string;+ screenshotId?: string;}export interface Screenshot {@@ -30,3 +30,14 @@ export interface Settings {aiApiKey: string;aiProvider: 'openai' | 'anthropic';}++export interface ElementMeta {+ tag: string; cssSelector: string; textContent: string | null;+ ariaLabel: string | null; placeholder: string | null; altText: string | null;+ name: string | null; role: string | null; href: string | null;+ inputType: string | null; dataTestId: string | null;+ rect: { x: number; y: number; width: number; height: number };+ devicePixelRatio: number;+}++export interface RrwebEventChunk { id: string; guideId: string; events: unknown[]; timestamp: number; }diff --git a/src/sidepanel/App.tsx b/src/sidepanel/App.tsxindex 68cc1c0..d017333 100644--- a/src/sidepanel/App.tsx+++ b/src/sidepanel/App.tsx@@ -1,12 +1,53 @@-import { useState, useEffect } from 'react';+import { useState, useEffect, useCallback } from 'react';export default function App() {const [isAlive, setIsAlive] = useState(false);+ const [isRecording, setIsRecording] = useState(false);+ const [stepCount, setStepCount] = useState(0);useEffect(() => {chrome.runtime.sendMessage({ type: 'PING' }, (response) => {if (response?.alive) setIsAlive(true);});++ const interval = setInterval(() => {+ chrome.runtime.sendMessage({ type: 'GET_STATE' }, (response) => {+ if (response?.state === 'recording') {+ setIsRecording(true);+ if (response.stepCount !== undefined) {+ setStepCount(response.stepCount);+ }+ } else {+ setIsRecording(false);+ }+ });+ }, 1000);++ return () => clearInterval(interval);+ }, []);++ const handleStartRecording = useCallback(async () => {+ const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });+ const url = tab?.url || '';++ chrome.runtime.sendMessage(+ { type: 'START_RECORDING', url },+ (response) => {+ if (response?.guideId) {+ setIsRecording(true);+ setStepCount(0);+ }+ }+ );+ }, []);++ const handleStopRecording = useCallback(() => {+ chrome.runtime.sendMessage({ type: 'STOP_RECORDING' }, (response) => {+ if (response?.success) {+ setIsRecording(false);+ setStepCount(0);+ }+ });}, []);return (@@ -17,15 +58,31 @@ export default function App() {</p><div className="mt-4 p-3 rounded-lg bg-gray-50 border border-gray-200"><p className="text-xs text-gray-500">- Service Worker: {isAlive ? 'β Connected' : '... Connecting'}+ Service Worker: {isAlive ? 'Connected' : '... Connecting'}</p>+ {isRecording && (+ <p className="text-xs text-red-600 mt-1 font-medium">+ Recording... {stepCount} step{stepCount !== 1 ? 's' : ''} captured+ </p>+ )}</div>- <button- className="mt-4 w-full py-2 px-4 bg-blue-600 text-white rounded-lg font-medium hover:bg-blue-700 transition-colors"- disabled- >- Start Recording (Phase 2)- </button>++ {!isRecording ? (+ <button+ onClick={handleStartRecording}+ className="mt-4 w-full py-2 px-4 bg-red-600 text-white rounded-lg font-medium hover:bg-red-700 transition-colors"+ disabled={!isAlive}+ >+ Start Recording+ </button>+ ) : (+ <button+ onClick={handleStopRecording}+ className="mt-4 w-full py-2 px-4 bg-gray-600 text-white rounded-lg font-medium hover:bg-gray-700 transition-colors"+ >+ Stop Recording+ </button>+ )}</div>);}diff --git a/tests/ai-description.test.ts b/tests/ai-description.test.tsnew file mode 100644index 0000000..fbd7fc5--- /dev/null+++ b/tests/ai-description.test.ts@@ -0,0 +1,108 @@+import { describe, it, expect, vi, beforeEach } from 'vitest';+import { getAIDescription } from '../src/background/ai-description';+import type { ElementMeta } from '../src/shared/types';++const mockOpenAICreate = vi.fn().mockResolvedValue({+ choices: [{ message: { content: 'Click the Submit button' } }],+});++const mockAnthropicCreate = vi.fn().mockResolvedValue({+ content: [{ type: 'text', text: 'Click the Submit button' }],+});++vi.mock('openai', () => ({+ default: vi.fn().mockImplementation(function () {+ return {+ chat: {+ completions: {+ create: mockOpenAICreate,+ },+ },+ };+ }),+}));++vi.mock('@anthropic-ai/sdk', () => ({+ default: vi.fn().mockImplementation(function () {+ return {+ messages: {+ create: mockAnthropicCreate,+ },+ };+ }),+}));++function makeMeta(): ElementMeta {+ return {+ tag: 'button',+ cssSelector: '#submit-btn',+ textContent: 'Submit',+ ariaLabel: null,+ placeholder: null,+ altText: null,+ name: null,+ role: 'button',+ href: null,+ inputType: null,+ dataTestId: null,+ rect: { x: 100, y: 200, width: 80, height: 32 },+ devicePixelRatio: 1,+ };+}++function makeBlob(): Blob {+ return new Blob(['test'], { type: 'image/jpeg' });+}++describe('getAIDescription', () => {+ const blob = makeBlob();+ const meta = makeMeta();++ beforeEach(() => {+ mockOpenAICreate.mockResolvedValue({+ choices: [{ message: { content: 'Click the Submit button' } }],+ });+ mockAnthropicCreate.mockResolvedValue({+ content: [{ type: 'text', text: 'Click the Submit button' }],+ });+ });++ it('OpenAI provider returns description string from mocked completions.create', async () => {+ const result = await getAIDescription(blob, 'click', meta, 'openai', 'test-key');+ expect(result).toBe('Click the Submit button');+ });++ it('Anthropic provider returns description string from mocked messages.create', async () => {+ const result = await getAIDescription(blob, 'click', meta, 'anthropic', 'test-key');+ expect(result).toBe('Click the Submit button');+ });++ it('getAIDescription returns null when OpenAI API throws error', async () => {+ mockOpenAICreate.mockRejectedValueOnce(new Error('API Error'));+ const result = await getAIDescription(blob, 'click', meta, 'openai', 'bad-key');+ expect(result).toBeNull();+ });++ it('getAIDescription returns null when Anthropic API throws error', async () => {+ mockAnthropicCreate.mockRejectedValueOnce(new Error('API Error'));+ const result = await getAIDescription(blob, 'click', meta, 'anthropic', 'bad-key');+ expect(result).toBeNull();+ });++ it('Blob is converted to base64 and included in prompt', async () => {+ let capturedParams: unknown = null;+ mockOpenAICreate.mockImplementationOnce((params: unknown) => {+ capturedParams = params;+ return Promise.resolve({+ choices: [{ message: { content: 'Click the Submit button' } }],+ });+ });+ await getAIDescription(blob, 'click', meta, 'openai', 'test-key');+ const params = capturedParams as { messages: Array<{ content: Array<{ type: string; image_url?: { url: string } }> }> };+ expect(params).not.toBeNull();+ const content = params.messages[0].content;+ const imageContent = content.find((c) => c.type === 'image_url');+ expect(imageContent).toBeDefined();+ expect(imageContent?.image_url?.url).toMatch(/^data:image\/jpeg;base64,/);+ });+});diff --git a/tests/capture-machine.test.ts b/tests/capture-machine.test.tsnew file mode 100644index 0000000..395e63b--- /dev/null+++ b/tests/capture-machine.test.ts@@ -0,0 +1,90 @@+import { describe, it, expect } from 'vitest';+import { createActor } from 'xstate';+import { captureMachine } from '../src/background/machine';++describe('captureMachine Phase 2', () => {+ it('idle -> recording on START_RECORDING; context.currentGuideId is a UUID string', () => {+ const actor = createActor(captureMachine);+ actor.start();+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });+ const snapshot = actor.getSnapshot();+ expect(snapshot.value).toBe('recording');+ expect(typeof snapshot.context.currentGuideId).toBe('string');+ expect(snapshot.context.currentGuideId).toMatch(+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/+ );+ actor.stop();+ });++ it('recording -> idle on STOP_RECORDING; context resets to null/0/empty', () => {+ const actor = createActor(captureMachine);+ actor.start();+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });+ actor.send({ type: 'STOP_RECORDING' });+ const snapshot = actor.getSnapshot();+ expect(snapshot.value).toBe('idle');+ expect(snapshot.context.currentGuideId).toBeNull();+ expect(snapshot.context.stepCount).toBe(0);+ expect(snapshot.context.currentUrl).toBe('');+ actor.stop();+ });++ it('USER_ACTION in recording state increments stepCount by 1', () => {+ const actor = createActor(captureMachine);+ actor.start();+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });+ expect(actor.getSnapshot().context.stepCount).toBe(0);+ actor.send({ type: 'USER_ACTION' });+ expect(actor.getSnapshot().context.stepCount).toBe(1);+ actor.send({ type: 'USER_ACTION' });+ expect(actor.getSnapshot().context.stepCount).toBe(2);+ actor.stop();+ });++ it('USER_ACTION in idle state does nothing (no transition)', () => {+ const actor = createActor(captureMachine);+ actor.start();+ expect(actor.getSnapshot().value).toBe('idle');+ actor.send({ type: 'USER_ACTION' });+ expect(actor.getSnapshot().value).toBe('idle');+ expect(actor.getSnapshot().context.stepCount).toBe(0);+ actor.stop();+ });++ it('SPA_NAVIGATE in recording state updates currentUrl', () => {+ const actor = createActor(captureMachine);+ actor.start();+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });+ actor.send({ type: 'SPA_NAVIGATE', url: 'https://example.com/page2' });+ expect(actor.getSnapshot().context.currentUrl).toBe('https://example.com/page2');+ expect(actor.getSnapshot().value).toBe('recording');+ actor.stop();+ });++ it('START_RECORDING sets currentUrl from event payload', () => {+ const actor = createActor(captureMachine);+ actor.start();+ actor.send({ type: 'START_RECORDING', url: 'https://example.com/start' });+ expect(actor.getSnapshot().context.currentUrl).toBe('https://example.com/start');+ actor.stop();+ });++ it('getPersistedSnapshot() roundtrip preserves currentUrl and stepCount', () => {+ const actor = createActor(captureMachine);+ actor.start();+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });+ actor.send({ type: 'USER_ACTION' });+ actor.send({ type: 'USER_ACTION' });+ actor.send({ type: 'SPA_NAVIGATE', url: 'https://example.com/dashboard' });++ const persisted = actor.getPersistedSnapshot();+ actor.stop();++ const restored = createActor(captureMachine, { snapshot: persisted });+ restored.start();+ const snapshot = restored.getSnapshot();+ expect(snapshot.context.currentUrl).toBe('https://example.com/dashboard');+ expect(snapshot.context.stepCount).toBe(2);+ restored.stop();+ });+});diff --git a/tests/db-migration.test.ts b/tests/db-migration.test.tsnew file mode 100644index 0000000..7aded97--- /dev/null+++ b/tests/db-migration.test.ts@@ -0,0 +1,62 @@+import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import 'fake-indexeddb/auto';+import { MimikDB } from '../src/shared/db-schema';+import type { RrwebEventChunk, Guide, Step, Screenshot } from '../src/shared/types';++describe('Dexie v2 migration', () => {+ let db: MimikDB;++ beforeEach(async () => {+ db = new MimikDB();+ await db.open();+ });++ afterEach(async () => {+ await db.delete();+ });++ it('creates rrwebEvents table (db.rrwebEvents.add succeeds)', async () => {+ const chunk: RrwebEventChunk = {+ id: crypto.randomUUID(),+ guideId: 'guide-1',+ events: [{ type: 1, data: {} }],+ timestamp: Date.now(),+ };+ await expect(db.rrwebEvents.add(chunk)).resolves.toBeDefined();+ });++ it('preserves existing guides table after v2 migration', async () => {+ const guide: Guide = {+ id: crypto.randomUUID(),+ title: 'Migration Test Guide',+ createdAt: Date.now(),+ updatedAt: Date.now(),+ stepIds: [],+ };+ await db.guides.add(guide);+ const retrieved = await db.guides.get(guide.id);+ expect(retrieved).toBeDefined();+ expect(retrieved!.title).toBe('Migration Test Guide');+ });++ it('can store and retrieve RrwebEventChunk with events array intact', async () => {+ const events = [{ type: 2, data: { x: 100, y: 200 } }, { type: 3, data: { text: 'hello' } }];+ const chunk: RrwebEventChunk = {+ id: crypto.randomUUID(),+ guideId: 'guide-2',+ events,+ timestamp: 1234567890,+ };+ await db.rrwebEvents.add(chunk);+ const retrieved = await db.rrwebEvents.get(chunk.id);+ expect(retrieved).toBeDefined();+ expect(retrieved!.guideId).toBe('guide-2');+ expect(retrieved!.timestamp).toBe(1234567890);+ expect(retrieved!.events).toHaveLength(2);+ expect(retrieved!.events[0]).toEqual({ type: 2, data: { x: 100, y: 200 } });+ });++ it('has version 2 with four tables', () => {+ expect(db.tables.map(t => t.name).sort()).toEqual(['guides', 'rrwebEvents', 'screenshots', 'steps']);+ });+});diff --git a/tests/db-schema.test.ts b/tests/db-schema.test.tsindex b49cd04..b16a771 100644--- a/tests/db-schema.test.ts+++ b/tests/db-schema.test.ts@@ -48,7 +48,7 @@ describe('MimikDB Schema', () => {expect(typeof retrieved!.blob).not.toBe('string');});- it('has version 1 with three tables', () => {- expect(db.tables.map(t => t.name).sort()).toEqual(['guides', 'screenshots', 'steps']);+ it('has version 2 with four tables including rrwebEvents', () => {+ expect(db.tables.map(t => t.name).sort()).toEqual(['guides', 'rrwebEvents', 'screenshots', 'steps']);});});diff --git a/tests/rrweb-storage.test.ts b/tests/rrweb-storage.test.tsnew file mode 100644index 0000000..f623391--- /dev/null+++ b/tests/rrweb-storage.test.ts@@ -0,0 +1,61 @@+import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import 'fake-indexeddb/auto';+import { MimikDB } from '../src/shared/db-schema';+import type { RrwebEventChunk } from '../src/shared/types';++describe('rrweb event chunk storage', () => {+ let db: MimikDB;++ beforeEach(async () => {+ db = new MimikDB();+ await db.open();+ });++ afterEach(async () => {+ await db.delete();+ });++ it('can add an RrwebEventChunk to db.rrwebEvents and retrieve it by id', async () => {+ const chunk: RrwebEventChunk = {+ id: crypto.randomUUID(),+ guideId: 'guide-123',+ events: [{ type: 2, data: { node: {} } }, { type: 3, data: {} }],+ timestamp: Date.now(),+ };++ await db.rrwebEvents.add(chunk);+ const retrieved = await db.rrwebEvents.get(chunk.id);++ expect(retrieved).toBeDefined();+ expect(retrieved!.guideId).toBe('guide-123');+ expect(retrieved!.events).toHaveLength(2);+ });++ it('can retrieve multiple chunks for the same guideId via where({guideId}).toArray()', async () => {+ const guideId = 'guide-abc';+ const chunk1: RrwebEventChunk = {+ id: crypto.randomUUID(),+ guideId,+ events: [{ type: 1 }],+ timestamp: Date.now(),+ };+ const chunk2: RrwebEventChunk = {+ id: crypto.randomUUID(),+ guideId,+ events: [{ type: 2 }, { type: 3 }],+ timestamp: Date.now() + 1000,+ };+ const otherChunk: RrwebEventChunk = {+ id: crypto.randomUUID(),+ guideId: 'other-guide',+ events: [{ type: 99 }],+ timestamp: Date.now(),+ };++ await db.rrwebEvents.bulkAdd([chunk1, chunk2, otherChunk]);++ const results = await db.rrwebEvents.where({ guideId }).toArray();+ expect(results).toHaveLength(2);+ expect(results.map(r => r.id).sort()).toEqual([chunk1.id, chunk2.id].sort());+ });+});diff --git a/tests/screenshot-annotation.test.ts b/tests/screenshot-annotation.test.tsnew file mode 100644index 0000000..5903136--- /dev/null+++ b/tests/screenshot-annotation.test.ts@@ -0,0 +1,130 @@+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+import type { ElementMeta } from '../src/shared/types';++function makeElementMeta(overrides: Partial<ElementMeta> = {}): ElementMeta {+ return {+ tag: 'button',+ cssSelector: 'button',+ textContent: 'Click me',+ ariaLabel: null,+ placeholder: null,+ altText: null,+ name: null,+ role: 'button',+ href: null,+ inputType: null,+ dataTestId: null,+ rect: { x: 10, y: 20, width: 100, height: 50 },+ devicePixelRatio: 2,+ ...overrides,+ };+}++function mockOffscreenCanvas(width: number, height: number) {+ const calls: Array<{ method: string; args: unknown[] }> = [];+ const ctx = {+ drawImage: (...args: unknown[]) => calls.push({ method: 'drawImage', args }),+ fillRect: (...args: unknown[]) => calls.push({ method: 'fillRect', args }),+ strokeRect: (...args: unknown[]) => calls.push({ method: 'strokeRect', args }),+ set strokeStyle(v: string) { calls.push({ method: 'set:strokeStyle', args: [v] }); },+ set lineWidth(v: number) { calls.push({ method: 'set:lineWidth', args: [v] }); },+ set fillStyle(v: string) { calls.push({ method: 'set:fillStyle', args: [v] }); },+ };+ return {+ width,+ height,+ getContext: () => ctx,+ convertToBlob: async ({ type }: { type: string }) => new Blob(['fake-image'], { type }),+ _calls: calls,+ };+}++describe('drawHighlight β OffscreenCanvas annotation', () => {+ let OffscreenCanvasInstances: ReturnType<typeof mockOffscreenCanvas>[] = [];++ afterEach(() => {+ vi.resetModules();+ });++ beforeEach(() => {+ vi.resetModules();+ OffscreenCanvasInstances = [];++ const instances = OffscreenCanvasInstances;+ (globalThis as unknown as Record<string, unknown>).OffscreenCanvas = class MockOffscreenCanvas {+ width: number;+ height: number;+ _calls: Array<{ method: string; args: unknown[] }> = [];++ constructor(w: number, h: number) {+ this.width = w;+ this.height = h;+ instances.push(this as unknown as ReturnType<typeof mockOffscreenCanvas>);+ }++ getContext() {+ const calls = this._calls;+ return {+ drawImage: (...args: unknown[]) => calls.push({ method: 'drawImage', args }),+ fillRect: (...args: unknown[]) => calls.push({ method: 'fillRect', args }),+ strokeRect: (...args: unknown[]) => calls.push({ method: 'strokeRect', args }),+ set strokeStyle(v: string) { calls.push({ method: 'set:strokeStyle', args: [v] }); },+ set lineWidth(v: number) { calls.push({ method: 'set:lineWidth', args: [v] }); },+ set fillStyle(v: string) { calls.push({ method: 'set:fillStyle', args: [v] }); },+ };+ }++ async convertToBlob({ type }: { type: string }) {+ return new Blob(['fake-image'], { type });+ }+ };++ (globalThis as unknown as Record<string, unknown>).createImageBitmap = vi.fn(async (_blob: Blob) => ({+ width: 1280,+ height: 720,+ close: () => {},+ }));++ (globalThis as unknown as Record<string, unknown>).fetch = vi.fn(async (_url: string) => ({+ blob: async () => new Blob(['fake-jpeg'], { type: 'image/jpeg' }),+ }));+ });++ it('scales CSS rect coordinates by devicePixelRatio (dpr=2: rect x=10 -> canvas x=20)', async () => {+ const { drawHighlight } = await import('../src/background/screenshot');+ const meta = makeElementMeta({ rect: { x: 10, y: 20, width: 100, height: 50 }, devicePixelRatio: 2 });+ await drawHighlight('data:image/jpeg;base64,fake', meta);++ const canvas = OffscreenCanvasInstances[0];+ expect(canvas).toBeDefined();+ const fillRectCall = canvas._calls.find(c => c.method === 'fillRect');+ expect(fillRectCall).toBeDefined();+ expect(fillRectCall!.args).toEqual([20, 40, 200, 100]);+ });++ it('uses blue-600 stroke color (#2563EB) and semi-transparent fill', async () => {+ const { drawHighlight } = await import('../src/background/screenshot');+ const meta = makeElementMeta();+ await drawHighlight('data:image/jpeg;base64,fake', meta);++ const canvas = OffscreenCanvasInstances[0];+ const strokeStyleCall = canvas._calls.find(c => c.method === 'set:strokeStyle');+ const fillStyleCall = canvas._calls.find(c => c.method === 'set:fillStyle');++ expect(strokeStyleCall?.args[0]).toBe('#2563EB');+ expect(fillStyleCall?.args[0]).toContain('rgba');+ expect(fillStyleCall?.args[0]).toContain('37');+ expect(fillStyleCall?.args[0]).toContain('99');+ expect(fillStyleCall?.args[0]).toContain('235');+ expect(fillStyleCall?.args[0]).toContain('0.15');+ });++ it('returns a Blob with type image/jpeg', async () => {+ const { drawHighlight } = await import('../src/background/screenshot');+ const meta = makeElementMeta();+ const result = await drawHighlight('data:image/jpeg;base64,fake', meta);++ expect(result).toBeInstanceOf(Blob);+ expect(result.type).toBe('image/jpeg');+ });+});diff --git a/tests/spa-navigation.test.ts b/tests/spa-navigation.test.tsnew file mode 100644index 0000000..b2acd4a--- /dev/null+++ b/tests/spa-navigation.test.ts@@ -0,0 +1,27 @@+import { describe, it, expect } from 'vitest';+import type { SpaNavigateMessage } from '../src/shared/messages';++describe('SPA navigation', () => {+ it('SPA_NAVIGATE message has the correct shape', () => {+ const message: SpaNavigateMessage = {+ type: 'SPA_NAVIGATE',+ url: 'https://example.com/page2',+ guideId: 'guide-123',+ };+ expect(message.type).toBe('SPA_NAVIGATE');+ expect(typeof message.url).toBe('string');+ expect(typeof message.guideId).toBe('string');+ });++ it('webNavigation listener should only process frameId === 0 (top frame)', () => {+ const topFrameDetails = { frameId: 0, url: 'https://example.com/page2', tabId: 1 };+ const subFrameDetails = { frameId: 1, url: 'https://example.com/iframe', tabId: 1 };++ function shouldProcess(details: { frameId: number }): boolean {+ return details.frameId === 0;+ }++ expect(shouldProcess(topFrameDetails)).toBe(true);+ expect(shouldProcess(subFrameDetails)).toBe(false);+ });+});diff --git a/tests/step-description.test.ts b/tests/step-description.test.tsnew file mode 100644index 0000000..fd916af--- /dev/null+++ b/tests/step-description.test.ts@@ -0,0 +1,87 @@+import { describe, it, expect } from 'vitest';+import { buildFallbackDescription } from '../src/background/step-description';+import type { ElementMeta } from '../src/shared/types';++function makeMeta(overrides: Partial<ElementMeta> = {}): ElementMeta {+ return {+ tag: 'div',+ cssSelector: 'div',+ textContent: null,+ ariaLabel: null,+ placeholder: null,+ altText: null,+ name: null,+ role: null,+ href: null,+ inputType: null,+ dataTestId: null,+ rect: { x: 0, y: 0, width: 100, height: 50 },+ devicePixelRatio: 1,+ ...overrides,+ };+}++describe('buildFallbackDescription', () => {+ it('click action with ariaLabel returns "Click {ariaLabel}"', () => {+ const meta = makeMeta({ ariaLabel: 'Submit button' });+ expect(buildFallbackDescription('click', meta)).toBe('Click Submit button');+ });++ it('click action with no ariaLabel but textContent returns "Click {textContent}" (truncated to 80 chars)', () => {+ const meta = makeMeta({ textContent: 'Sign in' });+ expect(buildFallbackDescription('click', meta)).toBe('Click Sign in');+ });++ it('textContent is truncated to 80 chars', () => {+ const longText = 'A'.repeat(100);+ const meta = makeMeta({ textContent: longText });+ const result = buildFallbackDescription('click', meta);+ expect(result).toBe('Click ' + 'A'.repeat(80));+ });++ it('input action with placeholder returns "Type into {placeholder}"', () => {+ const meta = makeMeta({ placeholder: 'Enter your email' });+ expect(buildFallbackDescription('input', meta)).toBe('Type into Enter your email');+ });++ it('scroll action returns "Scroll the page" regardless of element metadata', () => {+ const meta = makeMeta({ ariaLabel: 'some label', textContent: 'some text' });+ expect(buildFallbackDescription('scroll', meta)).toBe('Scroll the page');+ });++ it('navigate action returns "Navigate to page"', () => {+ const meta = makeMeta({ ariaLabel: 'Navigation link' });+ expect(buildFallbackDescription('navigate', meta)).toBe('Navigate to page');+ });++ it('priority chain: ariaLabel > placeholder > textContent > altText > name > role > tag', () => {+ const meta1 = makeMeta({+ ariaLabel: 'aria', placeholder: 'placeholder', textContent: 'text',+ altText: 'alt', name: 'nameAttr', role: 'button', tag: 'button',+ });+ expect(buildFallbackDescription('click', meta1)).toBe('Click aria');++ const meta2 = makeMeta({+ placeholder: 'placeholder', textContent: 'text',+ altText: 'alt', name: 'nameAttr', role: 'button', tag: 'button',+ });+ expect(buildFallbackDescription('input', meta2)).toBe('Type into placeholder');++ const meta3 = makeMeta({ textContent: 'text', altText: 'alt', name: 'nameAttr', role: 'button', tag: 'button' });+ expect(buildFallbackDescription('click', meta3)).toBe('Click text');++ const meta4 = makeMeta({ altText: 'alt', name: 'nameAttr', role: 'button', tag: 'img' });+ expect(buildFallbackDescription('click', meta4)).toBe('Click alt');++ const meta5 = makeMeta({ name: 'nameAttr', role: 'button', tag: 'input' });+ expect(buildFallbackDescription('click', meta5)).toBe('Click nameAttr');++ const meta6 = makeMeta({ role: 'button', tag: 'div' });+ expect(buildFallbackDescription('click', meta6)).toBe('Click button');+ });++ it('when all metadata is null, falls back to tag name', () => {+ const meta = makeMeta({ tag: 'button' });+ expect(buildFallbackDescription('click', meta)).toBe('Click button');+ });+});diff --git a/wxt.config.ts b/wxt.config.tsindex 2a9189e..5606e0d 100644--- a/wxt.config.ts+++ b/wxt.config.ts@@ -9,7 +9,7 @@ export default defineConfig({manifest: {name: 'Mimik',description: 'Auto-capture browser workflows and generate step-by-step guides with annotated screenshots.',- permissions: ['storage', 'activeTab', 'tabs', 'scripting', 'unlimitedStorage', 'sidePanel'],+ permissions: ['storage', 'activeTab', 'tabs', 'scripting', 'unlimitedStorage', 'sidePanel', 'webNavigation'],host_permissions: ['<all_urls>'],minimum_chrome_version: '118',action: {},