Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat: capture engine with event detection, screenshots, and AI descriptions

Luis Ortiz9b79886fdd64
diff --git a/entrypoints/background.ts b/entrypoints/background.ts
index 2916480..1c27c3f 100644
--- a/entrypoints/background.ts
+++ b/entrypoints/background.ts
@@ -1,33 +1,342 @@
import { defineBackground } from 'wxt/utils/define-background';
import { createActor } from 'xstate';
import { captureMachine } from '../src/background/machine';
+import { captureAnnotated } from '../src/background/screenshot';
+import { buildFallbackDescription } from '../src/background/step-description';
+import { getAIDescription } from '../src/background/ai-description';
+import { db } from '../src/shared/db-schema';
export default defineBackground(() => {
chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true });
+ let actorReadyResolve!: () => void;
+ const actorReady = new Promise<void>(resolve => {
+ actorReadyResolve = resolve;
+ });
+
let actor: ReturnType<typeof createActor<typeof captureMachine>>;
async function startActor() {
- const result = await chrome.storage.session.get('machineSnapshot');
- const snapshot = result.machineSnapshot ?? undefined;
- actor = createActor(captureMachine, { snapshot });
- actor.start();
+ try {
+ const result = await chrome.storage.session.get('machineSnapshot');
+ const snapshot = result.machineSnapshot ?? undefined;
+ actor = createActor(captureMachine, { snapshot });
+ actor.start();
+ } catch (err) {
+ console.warn('[Mimik] Failed to restore snapshot, starting fresh', err);
+ await chrome.storage.session.remove('machineSnapshot');
+ actor = createActor(captureMachine);
+ actor.start();
+ }
actor.subscribe(() => {
const persisted = actor.getPersistedSnapshot();
chrome.storage.session.set({ machineSnapshot: persisted });
});
+
+ actorReadyResolve();
}
- startActor();
+ startActor().catch((err) => {
+ console.error('[Mimik] startActor failed', err);
+ actor = createActor(captureMachine);
+ actor.start();
+ actorReadyResolve();
+ });
- chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => {
+ let captureQueue: Promise<void> = Promise.resolve();
+
+ function enqueueCaptureStep(fn: () => Promise<void>): void {
+ captureQueue = captureQueue.then(fn).catch(err =>
+ console.error('[Mimik] Capture error', err)
+ );
+ }
+
+ function isInjectableTab(tab: chrome.tabs.Tab): boolean {
+ const url = tab.url || tab.pendingUrl || '';
+ if (!url || url.startsWith('chrome://') || url.startsWith('chrome-extension://') ||
+ url.startsWith('chrome.google.com/webstore') || url.startsWith('about:')) return false;
+ return /^https?:/.test(url);
+ }
+
+ async function injectContentScript(tabId: number): Promise<void> {
+ try {
+ await chrome.tabs.sendMessage(tabId, { type: 'PING' });
+ } catch {
+ try {
+ await chrome.scripting.executeScript({
+ target: { tabId, allFrames: true },
+ files: ['content-scripts/content.js'],
+ });
+ } catch (err) {
+ }
+ }
+ }
+
+ async function injectAllTabs(): Promise<void> {
+ try {
+ const windows = await chrome.windows.getAll({ populate: true });
+ for (const win of windows) {
+ for (const tab of win.tabs || []) {
+ if (tab.id && isInjectableTab(tab)) {
+ injectContentScript(tab.id);
+ }
+ }
+ }
+ } catch (err) {
+ console.warn('[Mimik] injectAllTabs failed', err);
+ }
+ }
+
+ async function broadcastStartCapture(guideId: string): Promise<void> {
+ try {
+ const tabs = await chrome.tabs.query({});
+ for (const tab of tabs) {
+ if (tab.id && isInjectableTab(tab)) {
+ chrome.tabs.sendMessage(tab.id, { type: 'START_CAPTURE', guideId }).catch(() => {});
+ }
+ }
+ } catch (err) {
+ console.warn('[Mimik] broadcastStartCapture failed', err);
+ }
+ }
+
+ async function broadcastStopCapture(): Promise<void> {
+ try {
+ const tabs = await chrome.tabs.query({});
+ for (const tab of tabs) {
+ if (tab.id) {
+ chrome.tabs.sendMessage(tab.id, { type: 'STOP_CAPTURE' }).catch(() => {});
+ }
+ }
+ } catch (err) {
+ console.warn('[Mimik] broadcastStopCapture failed', err);
+ }
+ }
+
+ injectAllTabs();
+
+ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
if (msg.type === 'PING') {
sendResponse({ alive: true });
+ return true;
}
+
if (msg.type === 'GET_STATE') {
- sendResponse({ state: actor?.getSnapshot().value ?? 'unknown' });
+ (async () => {
+ try {
+ await actorReady;
+ const snap = actor.getSnapshot();
+ sendResponse({
+ state: snap.value ?? 'unknown',
+ stepCount: snap.context.stepCount,
+ currentGuideId: snap.context.currentGuideId,
+ });
+ } catch (err) {
+ console.error('[Mimik] GET_STATE failed', err);
+ sendResponse({ state: 'unknown' });
+ }
+ })();
+ return true;
+ }
+
+ if (msg.type === 'START_RECORDING') {
+ (async () => {
+ try {
+ await actorReady;
+ actor.send({ type: 'START_RECORDING', url: msg.url });
+
+ const guideId = actor.getSnapshot().context.currentGuideId!;
+
+ await db.guides.add({
+ id: guideId,
+ title: 'Untitled Guide',
+ createdAt: Date.now(),
+ updatedAt: Date.now(),
+ stepIds: [],
+ });
+
+ await broadcastStartCapture(guideId);
+
+ sendResponse({ guideId });
+ } catch (err) {
+ console.error('[Mimik] START_RECORDING failed', err);
+ sendResponse({ error: String(err) });
+ }
+ })();
+ return true;
+ }
+
+ if (msg.type === 'STOP_RECORDING') {
+ (async () => {
+ try {
+ await actorReady;
+
+ await broadcastStopCapture();
+
+ actor.send({ type: 'STOP_RECORDING' });
+ sendResponse({ success: true });
+ } catch (err) {
+ console.error('[Mimik] STOP_RECORDING failed', err);
+ sendResponse({ error: String(err) });
+ }
+ })();
+ return true;
+ }
+
+ if (msg.type === 'USER_ACTION') {
+ (async () => {
+ await actorReady;
+
+ const snap = actor.getSnapshot();
+ if (snap.value !== 'recording') {
+ sendResponse({ ignored: true });
+ return;
+ }
+
+ actor.send({ type: 'USER_ACTION' });
+
+ const stepCount = actor.getSnapshot().context.stepCount;
+ const guideId = snap.context.currentGuideId!;
+ const stepId = crypto.randomUUID();
+ const tabId = sender.tab?.id;
+
+ if (!tabId) {
+ sendResponse({ error: 'no tabId' });
+ return;
+ }
+
+ sendResponse({ stepId });
+
+ enqueueCaptureStep(async () => {
+ const fallbackDescription = buildFallbackDescription(msg.action, msg.elementMeta);
+ const currentUrl = snap.context.currentUrl;
+
+ let screenshotId: string | undefined;
+ let screenshotBlob: Blob | undefined;
+ try {
+ const screenshot = await captureAnnotated(tabId, stepId, msg.elementMeta);
+ screenshotId = screenshot.id;
+ screenshotBlob = screenshot.blob;
+ } catch (err) {
+ console.warn('[Mimik] Screenshot capture failed, storing step without screenshot', err);
+ }
+
+ await db.steps.add({
+ id: stepId,
+ guideId,
+ index: stepCount,
+ description: fallbackDescription,
+ action: msg.action,
+ url: currentUrl,
+ timestamp: Date.now(),
+ screenshotId,
+ });
+
+ const guide = await db.guides.get(guideId);
+ if (guide) {
+ await db.guides.update(guideId, {
+ stepIds: [...guide.stepIds, stepId],
+ updatedAt: Date.now(),
+ });
+ }
+
+ if (screenshotBlob) {
+ (async () => {
+ try {
+ const settings = await chrome.storage.local.get(['aiApiKey', 'aiProvider']);
+ if (settings.aiApiKey) {
+ const provider = (settings.aiProvider as 'openai' | 'anthropic') || 'openai';
+ const aiDescription = await getAIDescription(
+ screenshotBlob,
+ msg.action,
+ msg.elementMeta,
+ provider,
+ settings.aiApiKey
+ );
+ if (aiDescription) {
+ await db.steps.update(stepId, { description: aiDescription });
+ }
+ }
+ } catch (err) {
+ console.error('[Mimik] AI description update failed', err);
+ }
+ })();
+ }
+ });
+ })();
+ return true;
+ }
+
+ if (msg.type === 'RRWEB_CHUNK') {
+ (async () => {
+ await db.rrwebEvents.add({
+ id: crypto.randomUUID(),
+ guideId: msg.guideId,
+ events: msg.events,
+ timestamp: msg.timestamp,
+ });
+ sendResponse({ stored: true });
+ })();
+ return true;
+ }
+
+ return false;
+ });
+
+ chrome.webNavigation.onCompleted.addListener(async (details) => {
+ if (details.frameId !== 0) return;
+ await actorReady;
+ const state = actor.getSnapshot();
+ if (state.value === 'recording') {
+ actor.send({ type: 'SPA_NAVIGATE', url: details.url });
+ }
+ });
+
+ chrome.webNavigation.onHistoryStateUpdated.addListener(async (details) => {
+ if (details.frameId !== 0) return;
+ await actorReady;
+ const state = actor.getSnapshot();
+ if (state.value === 'recording') {
+ actor.send({ type: 'SPA_NAVIGATE', url: details.url });
+ }
+ });
+
+ chrome.tabs.onActivated.addListener(async (activeInfo) => {
+ await actorReady;
+ const state = actor.getSnapshot();
+ if (state.value !== 'recording') return;
+
+ const guideId = state.context.currentGuideId;
+ if (!guideId) return;
+
+ try {
+ await chrome.tabs.sendMessage(activeInfo.tabId, { type: 'PING' });
+ } catch {
+ try {
+ const tab = await chrome.tabs.get(activeInfo.tabId);
+ if (isInjectableTab(tab)) {
+ await injectContentScript(activeInfo.tabId);
+ }
+ } catch {
+ }
+ }
+ });
+
+ chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, tab) => {
+ if (changeInfo.status !== 'complete') return;
+ await actorReady;
+ const state = actor.getSnapshot();
+ if (state.value !== 'recording') return;
+ if (!isInjectableTab(tab)) return;
+
+ try {
+ await chrome.tabs.sendMessage(tabId, { type: 'PING' });
+ } catch {
+ try {
+ await injectContentScript(tabId);
+ } catch {
+ }
}
- return true;
});
});
diff --git a/entrypoints/content.ts b/entrypoints/content.ts
index 82953f7..d4b840a 100644
--- a/entrypoints/content.ts
+++ b/entrypoints/content.ts
@@ -1,8 +1,98 @@
import { defineContentScript } from 'wxt/utils/define-content-script';
+import { startCapture } from '../src/content/events';
+import { startRrwebRecording } from '../src/content/rrweb-recorder';
+import { updateUrl } from '../src/content/spa-nav';
+
+const CLEANUP_EVENT = `mimik_remove_content_script_${chrome.runtime.id}`;
export default defineContentScript({
matches: ['<all_urls>'],
+ allFrames: true,
+ matchAboutBlank: true,
+ runAt: 'document_idle',
+
main() {
+ document.dispatchEvent(new CustomEvent(CLEANUP_EVENT));
+
+ let stopCapture: (() => void) | null = null;
+ let stopRrweb: (() => void) | null = null;
+ let destroyed = false;
+
+ function beginCapture(guideId: string) {
+ if (destroyed) return;
+ stopCapture?.();
+ stopRrweb?.();
+ stopCapture = startCapture(guideId);
+ stopRrweb = startRrwebRecording(guideId);
+ }
+
+ function endCapture() {
+ stopCapture?.();
+ stopRrweb?.();
+ stopCapture = null;
+ stopRrweb = null;
+ }
+
+ function cleanup() {
+ destroyed = true;
+ endCapture();
+ chrome.runtime.onMessage.removeListener(messageHandler);
+ document.removeEventListener(CLEANUP_EVENT, cleanup);
+ }
+
+ document.addEventListener(CLEANUP_EVENT, cleanup);
+
+ window.addEventListener('beforeunload', () => {
+ endCapture();
+ });
+
+ function messageHandler(msg: any, _sender: any, sendResponse: (r: any) => void) {
+ if (destroyed) return false;
+
+ if (msg.type === 'PING') {
+ sendResponse({ alive: true });
+ return true;
+ }
+
+ if (msg.type === 'GET_ROUTE') {
+ sendResponse({ alive: true, capturing: !!stopCapture });
+ return true;
+ }
+
+ if (msg.type === 'START_CAPTURE' && msg.guideId) {
+ beginCapture(msg.guideId);
+ sendResponse({ started: true });
+ return true;
+ }
+
+ if (msg.type === 'STOP_CAPTURE') {
+ endCapture();
+ sendResponse({ stopped: true });
+ return true;
+ }
+
+ if (msg.type === 'SPA_NAVIGATE' && msg.url) {
+ updateUrl(msg.url);
+ sendResponse({ updated: true });
+ return true;
+ }
+
+ return false;
+ }
+
+ chrome.runtime.onMessage.addListener(messageHandler);
+
+ try {
+ chrome.runtime.sendMessage({ type: 'GET_STATE' }, (response) => {
+ if (chrome.runtime.lastError) return; // Extension context invalid
+ if (destroyed) return;
+ if (response?.state === 'recording' && response.currentGuideId) {
+ beginCapture(response.currentGuideId);
+ }
+ });
+ } catch {
+ }
+
console.log('[Mimik] Content script loaded');
},
});
diff --git a/package-lock.json b/package-lock.json
index e5010a1..d01bc22 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,14 +9,18 @@
"version": "1.0.0",
"license": "ISC",
"dependencies": {
+ "@anthropic-ai/sdk": "^0.80.0",
"@tailwindcss/vite": "^4.2.2",
"@wxt-dev/module-react": "^1.2.2",
"@xstate/react": "^6.1.0",
"clsx": "^2.1.1",
+ "css-selector-generator": "^3.8.1",
"dexie": "^4.3.0",
"lucide-react": "^0.577.0",
+ "openai": "^6.32.0",
"react": "^19.2.4",
"react-dom": "^19.2.4",
+ "rrweb": "^1.1.3",
"tailwind-merge": "^3.5.0",
"tailwindcss": "^4.2.2",
"wxt": "^0.20.20",
@@ -142,6 +146,26 @@
"url": "https://github.com/sponsors/jonschlinkert"
}
},
+ "node_modules/@anthropic-ai/sdk": {
+ "version": "0.80.0",
+ "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.80.0.tgz",
+ "integrity": "sha512-WeXLn7zNVk3yjeshn+xZHvld6AoFUOR3Sep6pSoHho5YbSi6HwcirqgPA5ccFuW8QTVJAAU7N8uQQC6Wa9TG+g==",
+ "license": "MIT",
+ "dependencies": {
+ "json-schema-to-ts": "^3.1.1"
+ },
+ "bin": {
+ "anthropic-ai-sdk": "bin/cli"
+ },
+ "peerDependencies": {
+ "zod": "^3.25.0 || ^4.0.0"
+ },
+ "peerDependenciesMeta": {
+ "zod": {
+ "optional": true
+ }
+ }
+ },
"node_modules/@asamuzakjp/css-color": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.0.1.tgz",
@@ -1641,6 +1665,12 @@
"assertion-error": "^2.0.1"
}
},
+ "node_modules/@types/css-font-loading-module": {
+ "version": "0.0.7",
+ "resolved": "https://registry.npmjs.org/@types/css-font-loading-module/-/css-font-loading-module-0.0.7.tgz",
+ "integrity": "sha512-nl09VhutdjINdWyXxHWN/w9zlNCfr60JUqJbd24YXUuCwgeL0TpFSdElCwb6cxfB6ybE19Gjj4g0jsgkXxKv1Q==",
+ "license": "MIT"
+ },
"node_modules/@types/deep-eql": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz",
@@ -1929,6 +1959,12 @@
"url": "https://github.com/sponsors/wxt-dev"
}
},
+ "node_modules/@xstate/fsm": {
+ "version": "1.6.5",
+ "resolved": "https://registry.npmjs.org/@xstate/fsm/-/fsm-1.6.5.tgz",
+ "integrity": "sha512-b5o1I6aLNeYlU/3CPlj/Z91ybk1gUsKT+5NAJI+2W4UjvS5KLG28K9v5UvNoFVjHV8PajVZ00RH3vnjyQO7ZAw==",
+ "license": "MIT"
+ },
"node_modules/@xstate/react": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/@xstate/react/-/react-6.1.0.tgz",
@@ -2148,6 +2184,15 @@
"node": "18 || 20 || >=22"
}
},
+ "node_modules/base64-arraybuffer": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz",
+ "integrity": "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 0.6.0"
+ }
+ },
"node_modules/bidi-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
@@ -2741,6 +2786,12 @@
"url": "https://github.com/sponsors/fb55"
}
},
+ "node_modules/css-selector-generator": {
+ "version": "3.8.1",
+ "resolved": "https://registry.npmjs.org/css-selector-generator/-/css-selector-generator-3.8.1.tgz",
+ "integrity": "sha512-3gT7VgYfhH6uLNdho9Z4lpaoUcFc6pciJwgfBbBrmuvNnTnugJ9+WJRFJkW+cGh2uHCxaIyBAk2pfhiqRP047A==",
+ "license": "MIT"
+ },
"node_modules/css-tree": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz",
@@ -3269,6 +3320,12 @@
}
}
},
+ "node_modules/fflate": {
+ "version": "0.4.8",
+ "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz",
+ "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==",
+ "license": "MIT"
+ },
"node_modules/filesize": {
"version": "11.0.13",
"resolved": "https://registry.npmjs.org/filesize/-/filesize-11.0.13.tgz",
@@ -3863,6 +3920,19 @@
"node": "^14.17.0 || ^16.13.0 || >=18.0.0"
}
},
+ "node_modules/json-schema-to-ts": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz",
+ "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/runtime": "^7.18.3",
+ "ts-algebra": "^2.0.0"
+ },
+ "engines": {
+ "node": ">=16"
+ }
+ },
"node_modules/json5": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
@@ -4576,6 +4646,12 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/mitt": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/mitt/-/mitt-1.2.0.tgz",
+ "integrity": "sha512-r6lj77KlwqLhIUku9UWYes7KJtsczvolZkzp8hbaDPPaE24OmWl5s539Mytlj22siEQKosZ26qCBgda2PKwoJw==",
+ "license": "MIT"
+ },
"node_modules/mlly": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz",
@@ -4882,6 +4958,27 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/openai": {
+ "version": "6.32.0",
+ "resolved": "https://registry.npmjs.org/openai/-/openai-6.32.0.tgz",
+ "integrity": "sha512-j3k+BjydAf8yQlcOI7WUQMQTbbF5GEIMAE2iZYCOzwwB3S2pCheaWYp+XZRNAch4jWVc52PMDGRRjutao3lLCg==",
+ "license": "Apache-2.0",
+ "bin": {
+ "openai": "bin/cli"
+ },
+ "peerDependencies": {
+ "ws": "^8.18.0",
+ "zod": "^3.25 || ^4.0"
+ },
+ "peerDependenciesMeta": {
+ "ws": {
+ "optional": true
+ },
+ "zod": {
+ "optional": true
+ }
+ }
+ },
"node_modules/os-shim": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/os-shim/-/os-shim-0.1.3.tgz",
@@ -5483,6 +5580,26 @@
"integrity": "sha512-UkVDEFk1w3mveXeKgaTuYfKWtPbvgck1dT8TUG3bnccrH0XtLTuAyfCoks4Q/M5ZGToSVJTIQYCzy2g/atAOeg==",
"license": "MIT"
},
+ "node_modules/rrweb": {
+ "version": "1.1.3",
+ "resolved": "https://registry.npmjs.org/rrweb/-/rrweb-1.1.3.tgz",
+ "integrity": "sha512-F2qp8LteJLyycsv+lCVJqtVpery63L3U+/ogqMA0da8R7Jx57o6gT+HpjrzdeeGMIBZR7kKNaKyJwDupTTu5KA==",
+ "license": "MIT",
+ "dependencies": {
+ "@types/css-font-loading-module": "0.0.7",
+ "@xstate/fsm": "^1.4.0",
+ "base64-arraybuffer": "^1.0.1",
+ "fflate": "^0.4.4",
+ "mitt": "^1.1.3",
+ "rrweb-snapshot": "^1.1.14"
+ }
+ },
+ "node_modules/rrweb-snapshot": {
+ "version": "1.1.14",
+ "resolved": "https://registry.npmjs.org/rrweb-snapshot/-/rrweb-snapshot-1.1.14.tgz",
+ "integrity": "sha512-eP5pirNjP5+GewQfcOQY4uBiDnpqxNRc65yKPW0eSoU1XamDfc4M8oqpXGMyUyvLyxFDB0q0+DChuxxiU2FXBQ==",
+ "license": "MIT"
+ },
"node_modules/run-applescript": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz",
@@ -5997,6 +6114,12 @@
"node": ">=20"
}
},
+ "node_modules/ts-algebra": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz",
+ "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",
+ "license": "MIT"
+ },
"node_modules/tslib": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
@@ -6576,7 +6699,7 @@
"version": "8.19.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz",
"integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==",
- "dev": true,
+ "devOptional": true,
"license": "MIT",
"engines": {
"node": ">=10.0.0"
diff --git a/package.json b/package.json
index 5721fa1..1c868b6 100644
--- a/package.json
+++ b/package.json
@@ -10,14 +10,18 @@
"author": "",
"license": "ISC",
"dependencies": {
+ "@anthropic-ai/sdk": "^0.80.0",
"@tailwindcss/vite": "^4.2.2",
"@wxt-dev/module-react": "^1.2.2",
"@xstate/react": "^6.1.0",
"clsx": "^2.1.1",
+ "css-selector-generator": "^3.8.1",
"dexie": "^4.3.0",
"lucide-react": "^0.577.0",
+ "openai": "^6.32.0",
"react": "^19.2.4",
"react-dom": "^19.2.4",
+ "rrweb": "^1.1.3",
"tailwind-merge": "^3.5.0",
"tailwindcss": "^4.2.2",
"wxt": "^0.20.20",
diff --git a/src/background/ai-description.ts b/src/background/ai-description.ts
new file mode 100644
index 0000000..61e2f5a
--- /dev/null
+++ b/src/background/ai-description.ts
@@ -0,0 +1,56 @@
+import OpenAI from 'openai';
+import type { ElementMeta } from '../shared/types';
+
+export async function getAIDescription(
+ blob: Blob,
+ action: string,
+ meta: ElementMeta,
+ provider: 'openai' | 'anthropic',
+ apiKey: string
+): Promise<string | null> {
+ const arrayBuffer = await blob.arrayBuffer();
+ const base64 = btoa(String.fromCharCode(...new Uint8Array(arrayBuffer)));
+
+ const targetText = meta.textContent || meta.ariaLabel || meta.placeholder || meta.tag;
+ const prompt = `This is a browser screenshot. The highlighted element (blue border) is a ${meta.tag} with text "${targetText}". The user performed a "${action}" action on it. Write a single concise sentence describing this step (e.g., "Click the Submit button"). No preamble.`;
+
+ try {
+ if (provider === 'openai') {
+ const client = new OpenAI({ apiKey, dangerouslyAllowBrowser: true });
+ const response = await client.chat.completions.create({
+ model: 'gpt-4o-mini',
+ max_tokens: 100,
+ messages: [{
+ role: 'user',
+ content: [
+ { type: 'image_url', image_url: { url: `data:image/jpeg;base64,${base64}` } },
+ { type: 'text', text: prompt },
+ ],
+ }],
+ });
+ return response.choices[0]?.message?.content?.trim() || null;
+ } else {
+ const { default: Anthropic } = await import('@anthropic-ai/sdk');
+ const client = new Anthropic({ apiKey });
+ const response = await client.messages.create({
+ model: 'claude-3-5-haiku-20241022',
+ max_tokens: 100,
+ messages: [{
+ role: 'user',
+ content: [
+ {
+ type: 'image',
+ source: { type: 'base64', media_type: 'image/jpeg', data: base64 },
+ },
+ { type: 'text', text: prompt },
+ ],
+ }],
+ });
+ const block = response.content[0];
+ return block.type === 'text' ? block.text.trim() : null;
+ }
+ } catch (err) {
+ console.error('[Mimik] AI description failed, using fallback', err);
+ return null;
+ }
+}
diff --git a/src/background/machine.ts b/src/background/machine.ts
index 4dd7353..2e4c29d 100644
--- a/src/background/machine.ts
+++ b/src/background/machine.ts
@@ -6,6 +6,7 @@ export const captureMachine = createMachine({
context: {
currentGuideId: null as string | null,
stepCount: 0,
+ currentUrl: '' as string,
},
states: {
idle: {
@@ -15,6 +16,7 @@ export const captureMachine = createMachine({
actions: assign({
currentGuideId: () => crypto.randomUUID(),
stepCount: 0,
+ currentUrl: ({ event }) => (event as { type: 'START_RECORDING'; url?: string }).url ?? '',
}),
},
},
@@ -26,6 +28,17 @@ export const captureMachine = createMachine({
actions: assign({
currentGuideId: null,
stepCount: 0,
+ currentUrl: '',
+ }),
+ },
+ USER_ACTION: {
+ actions: assign({
+ stepCount: ({ context }) => context.stepCount + 1,
+ }),
+ },
+ SPA_NAVIGATE: {
+ actions: assign({
+ currentUrl: ({ event }) => (event as { type: 'SPA_NAVIGATE'; url: string }).url,
}),
},
},
diff --git a/src/background/screenshot.ts b/src/background/screenshot.ts
index a5ea850..b6fc520 100644
--- a/src/background/screenshot.ts
+++ b/src/background/screenshot.ts
@@ -1,27 +1,64 @@
import { db } from '../shared/db-schema';
-import type { Screenshot } from '../shared/types';
+import type { Screenshot, ElementMeta } from '../shared/types';
export async function captureAndStore(tabId: number, stepId: string): Promise<Screenshot> {
+ const dataUrl = await chrome.tabs.captureVisibleTab(null, { format: 'jpeg', quality: 85 });
+ const blob = await fetch(dataUrl).then(r => r.blob());
+ const screenshot: Screenshot = {
+ id: crypto.randomUUID(), stepId, blob, mimeType: 'image/jpeg', width: 0, height: 0,
+ };
+ await db.screenshots.add(screenshot);
+ return screenshot;
+}
+
+export function dataUrlToBlob(dataUrl: string): Promise<Blob> {
+ return fetch(dataUrl).then(r => r.blob());
+}
+
+export async function captureAnnotated(
+ tabId: number,
+ stepId: string,
+ elementMeta: ElementMeta
+): Promise<Screenshot> {
const dataUrl = await chrome.tabs.captureVisibleTab(null, {
format: 'jpeg',
quality: 85,
});
- const blob = await fetch(dataUrl).then(r => r.blob());
+ const annotatedBlob = await drawHighlight(dataUrl, elementMeta);
+ const img = await createImageBitmap(annotatedBlob);
const screenshot: Screenshot = {
id: crypto.randomUUID(),
stepId,
- blob,
+ blob: annotatedBlob,
mimeType: 'image/jpeg',
- width: 0,
- height: 0,
+ width: img.width,
+ height: img.height,
};
await db.screenshots.add(screenshot);
return screenshot;
}
-export function dataUrlToBlob(dataUrl: string): Promise<Blob> {
- return fetch(dataUrl).then(r => r.blob());
+export async function drawHighlight(dataUrl: string, meta: ElementMeta): Promise<Blob> {
+ const img = await createImageBitmap(await fetch(dataUrl).then(r => r.blob()));
+
+ const canvas = new OffscreenCanvas(img.width, img.height);
+ const ctx = canvas.getContext('2d')!;
+ ctx.drawImage(img, 0, 0);
+
+ const dpr = meta.devicePixelRatio || 1;
+ const x = meta.rect.x * dpr;
+ const y = meta.rect.y * dpr;
+ const w = meta.rect.width * dpr;
+ const h = meta.rect.height * dpr;
+
+ ctx.strokeStyle = '#2563EB';
+ ctx.lineWidth = 3 * dpr;
+ ctx.fillStyle = 'rgba(37, 99, 235, 0.15)';
+ ctx.fillRect(x, y, w, h);
+ ctx.strokeRect(x, y, w, h);
+
+ return canvas.convertToBlob({ type: 'image/jpeg', quality: 0.85 });
}
diff --git a/src/background/step-description.ts b/src/background/step-description.ts
new file mode 100644
index 0000000..51bb59b
--- /dev/null
+++ b/src/background/step-description.ts
@@ -0,0 +1,33 @@
+import type { ElementMeta } from '../shared/types';
+
+export function buildFallbackDescription(action: string, meta: ElementMeta): string {
+ const target = meta.ariaLabel
+ || meta.placeholder
+ || meta.textContent?.slice(0, 80)
+ || meta.altText
+ || meta.name
+ || meta.role
+ || meta.tag;
+
+ if (action.startsWith('keydown:')) {
+ const key = action.split(':')[1];
+ return `Press ${key} on ${target}`;
+ }
+
+ switch (action) {
+ case 'click':
+ if (meta.tag === 'input' && meta.inputType === 'checkbox') return `Toggle checkbox ${target}`;
+ if (meta.tag === 'input' && meta.inputType === 'radio') return `Select radio ${target}`;
+ if (meta.href) return `Click link "${target}"`;
+ return `Click ${target}`;
+ case 'input':
+ if (meta.inputType) return `Type into ${meta.inputType} field ${target}`;
+ return `Type into ${target}`;
+ case 'scroll':
+ return 'Scroll the page';
+ case 'navigate':
+ return 'Navigate to page';
+ default:
+ return `${action} ${target}`;
+ }
+}
diff --git a/src/content/element-meta.ts b/src/content/element-meta.ts
new file mode 100644
index 0000000..c4ca16e
--- /dev/null
+++ b/src/content/element-meta.ts
@@ -0,0 +1,27 @@
+import { getCssSelector } from 'css-selector-generator';
+import type { ElementMeta } from '../shared/types';
+
+export function extractElementMeta(el: HTMLElement): ElementMeta {
+ const rect = el.getBoundingClientRect();
+ let cssSelector: string;
+ try {
+ cssSelector = getCssSelector(el);
+ } catch {
+ cssSelector = el.tagName?.toLowerCase() ?? 'unknown';
+ }
+ return {
+ tag: el.tagName?.toLowerCase() ?? 'unknown',
+ cssSelector,
+ textContent: el.textContent?.trim().slice(0, 200) || null,
+ ariaLabel: el.getAttribute('aria-label'),
+ placeholder: el.getAttribute('placeholder'),
+ altText: el instanceof HTMLImageElement ? el.alt : null,
+ name: el.getAttribute('name'),
+ role: el.getAttribute('role') || (el.tagName?.toLowerCase() ?? null),
+ href: el instanceof HTMLAnchorElement ? el.href : null,
+ inputType: el instanceof HTMLInputElement ? el.type : null,
+ dataTestId: el.getAttribute('data-testid') || el.getAttribute('data-test-id') || el.getAttribute('data-qa') || null,
+ rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
+ devicePixelRatio: window.devicePixelRatio,
+ };
+}
diff --git a/src/content/events.ts b/src/content/events.ts
new file mode 100644
index 0000000..f3d8177
--- /dev/null
+++ b/src/content/events.ts
@@ -0,0 +1,104 @@
+import { extractElementMeta } from './element-meta';
+
+const FOCUSABLE_SELECTOR = 'a[href], button, input, select, textarea, [role="button"], [role="link"], [role="tab"], [role="menuitem"], [role="checkbox"], [role="radio"], [tabindex]';
+
+const DEDUP_MS = 200;
+let lastClickTarget: Element | null = null;
+let lastClickTime = 0;
+
+function findFocusableAncestor(el: Element): HTMLElement {
+ const focusable = el.closest(FOCUSABLE_SELECTOR);
+ if (focusable instanceof HTMLElement) return focusable;
+
+ if (el instanceof HTMLElement) return el;
+ if (el.parentElement instanceof HTMLElement) return el.parentElement;
+
+ return document.body;
+}
+
+export function startCapture(guideId: string): () => void {
+ let scrollTimer: ReturnType<typeof setTimeout> | null = null;
+
+ function sendAction(action: string, meta: ReturnType<typeof extractElementMeta>) {
+ try {
+ chrome.runtime.sendMessage({
+ type: 'USER_ACTION',
+ guideId,
+ action,
+ elementMeta: meta,
+ });
+ } catch (err) {
+ console.warn('[Mimik] Failed to send action', err);
+ }
+ }
+
+ function handleClick(e: MouseEvent) {
+ const rawTarget = e.target;
+ if (!rawTarget || !(rawTarget instanceof Element)) return;
+
+ const target = findFocusableAncestor(rawTarget);
+ if (target.closest('[data-mimik-ignore]')) return;
+
+ const now = Date.now();
+ if (target === lastClickTarget && now - lastClickTime < DEDUP_MS) return;
+ lastClickTarget = target;
+ lastClickTime = now;
+
+ try {
+ const meta = extractElementMeta(target);
+ sendAction('click', meta);
+ } catch (err) {
+ console.warn('[Mimik] Failed to capture click', err);
+ }
+ }
+
+ function handleKeydown(e: KeyboardEvent) {
+ if (!['Enter', 'Escape', 'Tab'].includes(e.key)) return;
+ const target = e.target instanceof HTMLElement ? e.target : document.activeElement;
+ if (!target || !(target instanceof HTMLElement)) return;
+
+ try {
+ const meta = extractElementMeta(target);
+ sendAction(`keydown:${e.key}`, meta);
+ } catch (err) {
+ console.warn('[Mimik] Failed to capture keydown', err);
+ }
+ }
+
+ function handleInput(e: Event) {
+ if (!(e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement || e.target instanceof HTMLSelectElement)) return;
+ try {
+ const meta = extractElementMeta(e.target as HTMLElement);
+ sendAction('input', meta);
+ } catch (err) {
+ console.warn('[Mimik] Failed to capture input', err);
+ }
+ }
+
+ function handleScroll() {
+ if (scrollTimer) clearTimeout(scrollTimer);
+ scrollTimer = setTimeout(() => {
+ sendAction('scroll', {
+ tag: 'window', cssSelector: 'window', textContent: null,
+ ariaLabel: null, placeholder: null, altText: null,
+ name: null, role: 'scrollbar', href: null,
+ inputType: null, dataTestId: null,
+ rect: { x: 0, y: 0, width: window.innerWidth, height: window.innerHeight },
+ devicePixelRatio: window.devicePixelRatio,
+ });
+ }, 500);
+ }
+
+ window.addEventListener('click', handleClick, { capture: true });
+ window.addEventListener('keydown', handleKeydown, { capture: true });
+ window.addEventListener('input', handleInput, { capture: true });
+ window.addEventListener('scroll', handleScroll, { capture: true, passive: true });
+
+ return () => {
+ window.removeEventListener('click', handleClick, { capture: true });
+ window.removeEventListener('keydown', handleKeydown, { capture: true });
+ window.removeEventListener('input', handleInput, { capture: true });
+ window.removeEventListener('scroll', handleScroll, { capture: true });
+ if (scrollTimer) clearTimeout(scrollTimer);
+ };
+}
diff --git a/src/content/rrweb-recorder.ts b/src/content/rrweb-recorder.ts
new file mode 100644
index 0000000..236f826
--- /dev/null
+++ b/src/content/rrweb-recorder.ts
@@ -0,0 +1,41 @@
+// @ts-expect-error rrweb 1.1.3 type resolution
+import { record } from 'rrweb';
+
+const MAX_BUFFER_SIZE = 10_000;
+
+export function startRrwebRecording(guideId: string): () => void {
+ let eventBuffer: unknown[] = [];
+ let flushTimer: ReturnType<typeof setInterval>;
+
+ function flushChunk() {
+ if (eventBuffer.length === 0) return;
+ const chunk = eventBuffer.splice(0);
+ chrome.runtime.sendMessage({
+ type: 'RRWEB_CHUNK',
+ guideId,
+ events: chunk,
+ timestamp: Date.now(),
+ });
+ }
+
+ const stopRecord = record({
+ emit(event: unknown, isCheckout?: boolean) {
+ eventBuffer.push(event);
+ if (eventBuffer.length > MAX_BUFFER_SIZE) {
+ eventBuffer = eventBuffer.slice(-MAX_BUFFER_SIZE);
+ }
+ if (isCheckout) {
+ flushChunk();
+ }
+ },
+ checkoutEveryNms: 30_000,
+ });
+
+ flushTimer = setInterval(flushChunk, 30_000);
+
+ return () => {
+ flushChunk();
+ clearInterval(flushTimer);
+ stopRecord?.();
+ };
+}
diff --git a/src/content/spa-nav.ts b/src/content/spa-nav.ts
new file mode 100644
index 0000000..8c4fa5e
--- /dev/null
+++ b/src/content/spa-nav.ts
@@ -0,0 +1,9 @@
+let currentUrl = window.location.href;
+
+export function getCurrentUrl(): string {
+ return currentUrl;
+}
+
+export function updateUrl(url: string): void {
+ currentUrl = url;
+}
diff --git a/src/shared/db-schema.ts b/src/shared/db-schema.ts
index 425d0d0..c5fc11d 100644
--- a/src/shared/db-schema.ts
+++ b/src/shared/db-schema.ts
@@ -1,10 +1,11 @@
import Dexie, { type EntityTable } from 'dexie';
-import type { Guide, Step, Screenshot } from './types';
+import type { Guide, Step, Screenshot, RrwebEventChunk } from './types';
export class MimikDB extends Dexie {
guides!: EntityTable<Guide, 'id'>;
steps!: EntityTable<Step, 'id'>;
screenshots!: EntityTable<Screenshot, 'id'>;
+ rrwebEvents!: EntityTable<RrwebEventChunk, 'id'>;
constructor() {
super('mimik');
@@ -13,6 +14,12 @@ export class MimikDB extends Dexie {
steps: 'id, guideId, index',
screenshots: 'id, stepId',
});
+ this.version(2).stores({
+ guides: 'id, createdAt, updatedAt',
+ steps: 'id, guideId, index',
+ screenshots: 'id, stepId',
+ rrwebEvents: 'id, guideId, timestamp',
+ });
}
}
diff --git a/src/shared/messages.ts b/src/shared/messages.ts
index 340d01a..e0fd119 100644
--- a/src/shared/messages.ts
+++ b/src/shared/messages.ts
@@ -1,3 +1,5 @@
+import type { ElementMeta } from './types';
+
export type MessageType = 'PING' | 'CAPTURE_SCREENSHOT' | 'GET_STATE';
export interface PingMessage { type: 'PING'; }
@@ -9,4 +11,25 @@ export interface CaptureScreenshotResponse { screenshotId: string; }
export interface GetStateMessage { type: 'GET_STATE'; }
export interface GetStateResponse { state: string; }
-export type ExtensionMessage = PingMessage | CaptureScreenshotMessage | GetStateMessage;
+export interface StartRecordingMessage { type: 'START_RECORDING'; url: string; }
+export interface StopRecordingMessage { type: 'STOP_RECORDING'; }
+export interface UserActionMessage {
+ type: 'USER_ACTION';
+ guideId: string;
+ action: 'click' | 'input' | 'scroll';
+ elementMeta: ElementMeta;
+}
+export interface RrwebChunkMessage {
+ type: 'RRWEB_CHUNK';
+ guideId: string;
+ events: unknown[];
+ timestamp: number;
+}
+export interface SpaNavigateMessage {
+ type: 'SPA_NAVIGATE';
+ url: string;
+ guideId: string;
+}
+
+export type ExtensionMessage = PingMessage | CaptureScreenshotMessage | GetStateMessage
+ | StartRecordingMessage | StopRecordingMessage | UserActionMessage | RrwebChunkMessage | SpaNavigateMessage;
diff --git a/src/shared/types.ts b/src/shared/types.ts
index eda0651..cab6a8d 100644
--- a/src/shared/types.ts
+++ b/src/shared/types.ts
@@ -11,10 +11,10 @@ export interface Step {
guideId: string;
index: number;
description: string;
- action: 'click' | 'input' | 'scroll' | 'navigate';
+ action: string;
url: string;
timestamp: number;
- screenshotId: string;
+ screenshotId?: string;
}
export interface Screenshot {
@@ -30,3 +30,14 @@ export interface Settings {
aiApiKey: string;
aiProvider: 'openai' | 'anthropic';
}
+
+export interface ElementMeta {
+ tag: string; cssSelector: string; textContent: string | null;
+ ariaLabel: string | null; placeholder: string | null; altText: string | null;
+ name: string | null; role: string | null; href: string | null;
+ inputType: string | null; dataTestId: string | null;
+ rect: { x: number; y: number; width: number; height: number };
+ devicePixelRatio: number;
+}
+
+export interface RrwebEventChunk { id: string; guideId: string; events: unknown[]; timestamp: number; }
diff --git a/src/sidepanel/App.tsx b/src/sidepanel/App.tsx
index 68cc1c0..d017333 100644
--- a/src/sidepanel/App.tsx
+++ b/src/sidepanel/App.tsx
@@ -1,12 +1,53 @@
-import { useState, useEffect } from 'react';
+import { useState, useEffect, useCallback } from 'react';
export default function App() {
const [isAlive, setIsAlive] = useState(false);
+ const [isRecording, setIsRecording] = useState(false);
+ const [stepCount, setStepCount] = useState(0);
useEffect(() => {
chrome.runtime.sendMessage({ type: 'PING' }, (response) => {
if (response?.alive) setIsAlive(true);
});
+
+ const interval = setInterval(() => {
+ chrome.runtime.sendMessage({ type: 'GET_STATE' }, (response) => {
+ if (response?.state === 'recording') {
+ setIsRecording(true);
+ if (response.stepCount !== undefined) {
+ setStepCount(response.stepCount);
+ }
+ } else {
+ setIsRecording(false);
+ }
+ });
+ }, 1000);
+
+ return () => clearInterval(interval);
+ }, []);
+
+ const handleStartRecording = useCallback(async () => {
+ const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
+ const url = tab?.url || '';
+
+ chrome.runtime.sendMessage(
+ { type: 'START_RECORDING', url },
+ (response) => {
+ if (response?.guideId) {
+ setIsRecording(true);
+ setStepCount(0);
+ }
+ }
+ );
+ }, []);
+
+ const handleStopRecording = useCallback(() => {
+ chrome.runtime.sendMessage({ type: 'STOP_RECORDING' }, (response) => {
+ if (response?.success) {
+ setIsRecording(false);
+ setStepCount(0);
+ }
+ });
}, []);
return (
@@ -17,15 +58,31 @@ export default function App() {
</p>
<div className="mt-4 p-3 rounded-lg bg-gray-50 border border-gray-200">
<p className="text-xs text-gray-500">
- Service Worker: {isAlive ? 'βœ“ Connected' : '... Connecting'}
+ Service Worker: {isAlive ? 'Connected' : '... Connecting'}
</p>
+ {isRecording && (
+ <p className="text-xs text-red-600 mt-1 font-medium">
+ Recording... {stepCount} step{stepCount !== 1 ? 's' : ''} captured
+ </p>
+ )}
</div>
- <button
- className="mt-4 w-full py-2 px-4 bg-blue-600 text-white rounded-lg font-medium hover:bg-blue-700 transition-colors"
- disabled
- >
- Start Recording (Phase 2)
- </button>
+
+ {!isRecording ? (
+ <button
+ onClick={handleStartRecording}
+ className="mt-4 w-full py-2 px-4 bg-red-600 text-white rounded-lg font-medium hover:bg-red-700 transition-colors"
+ disabled={!isAlive}
+ >
+ Start Recording
+ </button>
+ ) : (
+ <button
+ onClick={handleStopRecording}
+ className="mt-4 w-full py-2 px-4 bg-gray-600 text-white rounded-lg font-medium hover:bg-gray-700 transition-colors"
+ >
+ Stop Recording
+ </button>
+ )}
</div>
);
}
diff --git a/tests/ai-description.test.ts b/tests/ai-description.test.ts
new file mode 100644
index 0000000..fbd7fc5
--- /dev/null
+++ b/tests/ai-description.test.ts
@@ -0,0 +1,108 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { getAIDescription } from '../src/background/ai-description';
+import type { ElementMeta } from '../src/shared/types';
+
+const mockOpenAICreate = vi.fn().mockResolvedValue({
+ choices: [{ message: { content: 'Click the Submit button' } }],
+});
+
+const mockAnthropicCreate = vi.fn().mockResolvedValue({
+ content: [{ type: 'text', text: 'Click the Submit button' }],
+});
+
+vi.mock('openai', () => ({
+ default: vi.fn().mockImplementation(function () {
+ return {
+ chat: {
+ completions: {
+ create: mockOpenAICreate,
+ },
+ },
+ };
+ }),
+}));
+
+vi.mock('@anthropic-ai/sdk', () => ({
+ default: vi.fn().mockImplementation(function () {
+ return {
+ messages: {
+ create: mockAnthropicCreate,
+ },
+ };
+ }),
+}));
+
+function makeMeta(): ElementMeta {
+ return {
+ tag: 'button',
+ cssSelector: '#submit-btn',
+ textContent: 'Submit',
+ ariaLabel: null,
+ placeholder: null,
+ altText: null,
+ name: null,
+ role: 'button',
+ href: null,
+ inputType: null,
+ dataTestId: null,
+ rect: { x: 100, y: 200, width: 80, height: 32 },
+ devicePixelRatio: 1,
+ };
+}
+
+function makeBlob(): Blob {
+ return new Blob(['test'], { type: 'image/jpeg' });
+}
+
+describe('getAIDescription', () => {
+ const blob = makeBlob();
+ const meta = makeMeta();
+
+ beforeEach(() => {
+ mockOpenAICreate.mockResolvedValue({
+ choices: [{ message: { content: 'Click the Submit button' } }],
+ });
+ mockAnthropicCreate.mockResolvedValue({
+ content: [{ type: 'text', text: 'Click the Submit button' }],
+ });
+ });
+
+ it('OpenAI provider returns description string from mocked completions.create', async () => {
+ const result = await getAIDescription(blob, 'click', meta, 'openai', 'test-key');
+ expect(result).toBe('Click the Submit button');
+ });
+
+ it('Anthropic provider returns description string from mocked messages.create', async () => {
+ const result = await getAIDescription(blob, 'click', meta, 'anthropic', 'test-key');
+ expect(result).toBe('Click the Submit button');
+ });
+
+ it('getAIDescription returns null when OpenAI API throws error', async () => {
+ mockOpenAICreate.mockRejectedValueOnce(new Error('API Error'));
+ const result = await getAIDescription(blob, 'click', meta, 'openai', 'bad-key');
+ expect(result).toBeNull();
+ });
+
+ it('getAIDescription returns null when Anthropic API throws error', async () => {
+ mockAnthropicCreate.mockRejectedValueOnce(new Error('API Error'));
+ const result = await getAIDescription(blob, 'click', meta, 'anthropic', 'bad-key');
+ expect(result).toBeNull();
+ });
+
+ it('Blob is converted to base64 and included in prompt', async () => {
+ let capturedParams: unknown = null;
+ mockOpenAICreate.mockImplementationOnce((params: unknown) => {
+ capturedParams = params;
+ return Promise.resolve({
+ choices: [{ message: { content: 'Click the Submit button' } }],
+ });
+ });
+ await getAIDescription(blob, 'click', meta, 'openai', 'test-key');
+ const params = capturedParams as { messages: Array<{ content: Array<{ type: string; image_url?: { url: string } }> }> };
+ expect(params).not.toBeNull();
+ const content = params.messages[0].content;
+ const imageContent = content.find((c) => c.type === 'image_url');
+ expect(imageContent).toBeDefined();
+ expect(imageContent?.image_url?.url).toMatch(/^data:image\/jpeg;base64,/);
+ });
+});
diff --git a/tests/capture-machine.test.ts b/tests/capture-machine.test.ts
new file mode 100644
index 0000000..395e63b
--- /dev/null
+++ b/tests/capture-machine.test.ts
@@ -0,0 +1,90 @@
+import { describe, it, expect } from 'vitest';
+import { createActor } from 'xstate';
+import { captureMachine } from '../src/background/machine';
+
+describe('captureMachine Phase 2', () => {
+ it('idle -> recording on START_RECORDING; context.currentGuideId is a UUID string', () => {
+ const actor = createActor(captureMachine);
+ actor.start();
+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });
+ const snapshot = actor.getSnapshot();
+ expect(snapshot.value).toBe('recording');
+ expect(typeof snapshot.context.currentGuideId).toBe('string');
+ expect(snapshot.context.currentGuideId).toMatch(
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+ );
+ actor.stop();
+ });
+
+ it('recording -> idle on STOP_RECORDING; context resets to null/0/empty', () => {
+ const actor = createActor(captureMachine);
+ actor.start();
+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });
+ actor.send({ type: 'STOP_RECORDING' });
+ const snapshot = actor.getSnapshot();
+ expect(snapshot.value).toBe('idle');
+ expect(snapshot.context.currentGuideId).toBeNull();
+ expect(snapshot.context.stepCount).toBe(0);
+ expect(snapshot.context.currentUrl).toBe('');
+ actor.stop();
+ });
+
+ it('USER_ACTION in recording state increments stepCount by 1', () => {
+ const actor = createActor(captureMachine);
+ actor.start();
+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });
+ expect(actor.getSnapshot().context.stepCount).toBe(0);
+ actor.send({ type: 'USER_ACTION' });
+ expect(actor.getSnapshot().context.stepCount).toBe(1);
+ actor.send({ type: 'USER_ACTION' });
+ expect(actor.getSnapshot().context.stepCount).toBe(2);
+ actor.stop();
+ });
+
+ it('USER_ACTION in idle state does nothing (no transition)', () => {
+ const actor = createActor(captureMachine);
+ actor.start();
+ expect(actor.getSnapshot().value).toBe('idle');
+ actor.send({ type: 'USER_ACTION' });
+ expect(actor.getSnapshot().value).toBe('idle');
+ expect(actor.getSnapshot().context.stepCount).toBe(0);
+ actor.stop();
+ });
+
+ it('SPA_NAVIGATE in recording state updates currentUrl', () => {
+ const actor = createActor(captureMachine);
+ actor.start();
+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });
+ actor.send({ type: 'SPA_NAVIGATE', url: 'https://example.com/page2' });
+ expect(actor.getSnapshot().context.currentUrl).toBe('https://example.com/page2');
+ expect(actor.getSnapshot().value).toBe('recording');
+ actor.stop();
+ });
+
+ it('START_RECORDING sets currentUrl from event payload', () => {
+ const actor = createActor(captureMachine);
+ actor.start();
+ actor.send({ type: 'START_RECORDING', url: 'https://example.com/start' });
+ expect(actor.getSnapshot().context.currentUrl).toBe('https://example.com/start');
+ actor.stop();
+ });
+
+ it('getPersistedSnapshot() roundtrip preserves currentUrl and stepCount', () => {
+ const actor = createActor(captureMachine);
+ actor.start();
+ actor.send({ type: 'START_RECORDING', url: 'https://example.com' });
+ actor.send({ type: 'USER_ACTION' });
+ actor.send({ type: 'USER_ACTION' });
+ actor.send({ type: 'SPA_NAVIGATE', url: 'https://example.com/dashboard' });
+
+ const persisted = actor.getPersistedSnapshot();
+ actor.stop();
+
+ const restored = createActor(captureMachine, { snapshot: persisted });
+ restored.start();
+ const snapshot = restored.getSnapshot();
+ expect(snapshot.context.currentUrl).toBe('https://example.com/dashboard');
+ expect(snapshot.context.stepCount).toBe(2);
+ restored.stop();
+ });
+});
diff --git a/tests/db-migration.test.ts b/tests/db-migration.test.ts
new file mode 100644
index 0000000..7aded97
--- /dev/null
+++ b/tests/db-migration.test.ts
@@ -0,0 +1,62 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import 'fake-indexeddb/auto';
+import { MimikDB } from '../src/shared/db-schema';
+import type { RrwebEventChunk, Guide, Step, Screenshot } from '../src/shared/types';
+
+describe('Dexie v2 migration', () => {
+ let db: MimikDB;
+
+ beforeEach(async () => {
+ db = new MimikDB();
+ await db.open();
+ });
+
+ afterEach(async () => {
+ await db.delete();
+ });
+
+ it('creates rrwebEvents table (db.rrwebEvents.add succeeds)', async () => {
+ const chunk: RrwebEventChunk = {
+ id: crypto.randomUUID(),
+ guideId: 'guide-1',
+ events: [{ type: 1, data: {} }],
+ timestamp: Date.now(),
+ };
+ await expect(db.rrwebEvents.add(chunk)).resolves.toBeDefined();
+ });
+
+ it('preserves existing guides table after v2 migration', async () => {
+ const guide: Guide = {
+ id: crypto.randomUUID(),
+ title: 'Migration Test Guide',
+ createdAt: Date.now(),
+ updatedAt: Date.now(),
+ stepIds: [],
+ };
+ await db.guides.add(guide);
+ const retrieved = await db.guides.get(guide.id);
+ expect(retrieved).toBeDefined();
+ expect(retrieved!.title).toBe('Migration Test Guide');
+ });
+
+ it('can store and retrieve RrwebEventChunk with events array intact', async () => {
+ const events = [{ type: 2, data: { x: 100, y: 200 } }, { type: 3, data: { text: 'hello' } }];
+ const chunk: RrwebEventChunk = {
+ id: crypto.randomUUID(),
+ guideId: 'guide-2',
+ events,
+ timestamp: 1234567890,
+ };
+ await db.rrwebEvents.add(chunk);
+ const retrieved = await db.rrwebEvents.get(chunk.id);
+ expect(retrieved).toBeDefined();
+ expect(retrieved!.guideId).toBe('guide-2');
+ expect(retrieved!.timestamp).toBe(1234567890);
+ expect(retrieved!.events).toHaveLength(2);
+ expect(retrieved!.events[0]).toEqual({ type: 2, data: { x: 100, y: 200 } });
+ });
+
+ it('has version 2 with four tables', () => {
+ expect(db.tables.map(t => t.name).sort()).toEqual(['guides', 'rrwebEvents', 'screenshots', 'steps']);
+ });
+});
diff --git a/tests/db-schema.test.ts b/tests/db-schema.test.ts
index b49cd04..b16a771 100644
--- a/tests/db-schema.test.ts
+++ b/tests/db-schema.test.ts
@@ -48,7 +48,7 @@ describe('MimikDB Schema', () => {
expect(typeof retrieved!.blob).not.toBe('string');
});
- it('has version 1 with three tables', () => {
- expect(db.tables.map(t => t.name).sort()).toEqual(['guides', 'screenshots', 'steps']);
+ it('has version 2 with four tables including rrwebEvents', () => {
+ expect(db.tables.map(t => t.name).sort()).toEqual(['guides', 'rrwebEvents', 'screenshots', 'steps']);
});
});
diff --git a/tests/rrweb-storage.test.ts b/tests/rrweb-storage.test.ts
new file mode 100644
index 0000000..f623391
--- /dev/null
+++ b/tests/rrweb-storage.test.ts
@@ -0,0 +1,61 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import 'fake-indexeddb/auto';
+import { MimikDB } from '../src/shared/db-schema';
+import type { RrwebEventChunk } from '../src/shared/types';
+
+describe('rrweb event chunk storage', () => {
+ let db: MimikDB;
+
+ beforeEach(async () => {
+ db = new MimikDB();
+ await db.open();
+ });
+
+ afterEach(async () => {
+ await db.delete();
+ });
+
+ it('can add an RrwebEventChunk to db.rrwebEvents and retrieve it by id', async () => {
+ const chunk: RrwebEventChunk = {
+ id: crypto.randomUUID(),
+ guideId: 'guide-123',
+ events: [{ type: 2, data: { node: {} } }, { type: 3, data: {} }],
+ timestamp: Date.now(),
+ };
+
+ await db.rrwebEvents.add(chunk);
+ const retrieved = await db.rrwebEvents.get(chunk.id);
+
+ expect(retrieved).toBeDefined();
+ expect(retrieved!.guideId).toBe('guide-123');
+ expect(retrieved!.events).toHaveLength(2);
+ });
+
+ it('can retrieve multiple chunks for the same guideId via where({guideId}).toArray()', async () => {
+ const guideId = 'guide-abc';
+ const chunk1: RrwebEventChunk = {
+ id: crypto.randomUUID(),
+ guideId,
+ events: [{ type: 1 }],
+ timestamp: Date.now(),
+ };
+ const chunk2: RrwebEventChunk = {
+ id: crypto.randomUUID(),
+ guideId,
+ events: [{ type: 2 }, { type: 3 }],
+ timestamp: Date.now() + 1000,
+ };
+ const otherChunk: RrwebEventChunk = {
+ id: crypto.randomUUID(),
+ guideId: 'other-guide',
+ events: [{ type: 99 }],
+ timestamp: Date.now(),
+ };
+
+ await db.rrwebEvents.bulkAdd([chunk1, chunk2, otherChunk]);
+
+ const results = await db.rrwebEvents.where({ guideId }).toArray();
+ expect(results).toHaveLength(2);
+ expect(results.map(r => r.id).sort()).toEqual([chunk1.id, chunk2.id].sort());
+ });
+});
diff --git a/tests/screenshot-annotation.test.ts b/tests/screenshot-annotation.test.ts
new file mode 100644
index 0000000..5903136
--- /dev/null
+++ b/tests/screenshot-annotation.test.ts
@@ -0,0 +1,130 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import type { ElementMeta } from '../src/shared/types';
+
+function makeElementMeta(overrides: Partial<ElementMeta> = {}): ElementMeta {
+ return {
+ tag: 'button',
+ cssSelector: 'button',
+ textContent: 'Click me',
+ ariaLabel: null,
+ placeholder: null,
+ altText: null,
+ name: null,
+ role: 'button',
+ href: null,
+ inputType: null,
+ dataTestId: null,
+ rect: { x: 10, y: 20, width: 100, height: 50 },
+ devicePixelRatio: 2,
+ ...overrides,
+ };
+}
+
+function mockOffscreenCanvas(width: number, height: number) {
+ const calls: Array<{ method: string; args: unknown[] }> = [];
+ const ctx = {
+ drawImage: (...args: unknown[]) => calls.push({ method: 'drawImage', args }),
+ fillRect: (...args: unknown[]) => calls.push({ method: 'fillRect', args }),
+ strokeRect: (...args: unknown[]) => calls.push({ method: 'strokeRect', args }),
+ set strokeStyle(v: string) { calls.push({ method: 'set:strokeStyle', args: [v] }); },
+ set lineWidth(v: number) { calls.push({ method: 'set:lineWidth', args: [v] }); },
+ set fillStyle(v: string) { calls.push({ method: 'set:fillStyle', args: [v] }); },
+ };
+ return {
+ width,
+ height,
+ getContext: () => ctx,
+ convertToBlob: async ({ type }: { type: string }) => new Blob(['fake-image'], { type }),
+ _calls: calls,
+ };
+}
+
+describe('drawHighlight β€” OffscreenCanvas annotation', () => {
+ let OffscreenCanvasInstances: ReturnType<typeof mockOffscreenCanvas>[] = [];
+
+ afterEach(() => {
+ vi.resetModules();
+ });
+
+ beforeEach(() => {
+ vi.resetModules();
+ OffscreenCanvasInstances = [];
+
+ const instances = OffscreenCanvasInstances;
+ (globalThis as unknown as Record<string, unknown>).OffscreenCanvas = class MockOffscreenCanvas {
+ width: number;
+ height: number;
+ _calls: Array<{ method: string; args: unknown[] }> = [];
+
+ constructor(w: number, h: number) {
+ this.width = w;
+ this.height = h;
+ instances.push(this as unknown as ReturnType<typeof mockOffscreenCanvas>);
+ }
+
+ getContext() {
+ const calls = this._calls;
+ return {
+ drawImage: (...args: unknown[]) => calls.push({ method: 'drawImage', args }),
+ fillRect: (...args: unknown[]) => calls.push({ method: 'fillRect', args }),
+ strokeRect: (...args: unknown[]) => calls.push({ method: 'strokeRect', args }),
+ set strokeStyle(v: string) { calls.push({ method: 'set:strokeStyle', args: [v] }); },
+ set lineWidth(v: number) { calls.push({ method: 'set:lineWidth', args: [v] }); },
+ set fillStyle(v: string) { calls.push({ method: 'set:fillStyle', args: [v] }); },
+ };
+ }
+
+ async convertToBlob({ type }: { type: string }) {
+ return new Blob(['fake-image'], { type });
+ }
+ };
+
+ (globalThis as unknown as Record<string, unknown>).createImageBitmap = vi.fn(async (_blob: Blob) => ({
+ width: 1280,
+ height: 720,
+ close: () => {},
+ }));
+
+ (globalThis as unknown as Record<string, unknown>).fetch = vi.fn(async (_url: string) => ({
+ blob: async () => new Blob(['fake-jpeg'], { type: 'image/jpeg' }),
+ }));
+ });
+
+ it('scales CSS rect coordinates by devicePixelRatio (dpr=2: rect x=10 -> canvas x=20)', async () => {
+ const { drawHighlight } = await import('../src/background/screenshot');
+ const meta = makeElementMeta({ rect: { x: 10, y: 20, width: 100, height: 50 }, devicePixelRatio: 2 });
+ await drawHighlight('data:image/jpeg;base64,fake', meta);
+
+ const canvas = OffscreenCanvasInstances[0];
+ expect(canvas).toBeDefined();
+ const fillRectCall = canvas._calls.find(c => c.method === 'fillRect');
+ expect(fillRectCall).toBeDefined();
+ expect(fillRectCall!.args).toEqual([20, 40, 200, 100]);
+ });
+
+ it('uses blue-600 stroke color (#2563EB) and semi-transparent fill', async () => {
+ const { drawHighlight } = await import('../src/background/screenshot');
+ const meta = makeElementMeta();
+ await drawHighlight('data:image/jpeg;base64,fake', meta);
+
+ const canvas = OffscreenCanvasInstances[0];
+ const strokeStyleCall = canvas._calls.find(c => c.method === 'set:strokeStyle');
+ const fillStyleCall = canvas._calls.find(c => c.method === 'set:fillStyle');
+
+ expect(strokeStyleCall?.args[0]).toBe('#2563EB');
+ expect(fillStyleCall?.args[0]).toContain('rgba');
+ expect(fillStyleCall?.args[0]).toContain('37');
+ expect(fillStyleCall?.args[0]).toContain('99');
+ expect(fillStyleCall?.args[0]).toContain('235');
+ expect(fillStyleCall?.args[0]).toContain('0.15');
+ });
+
+ it('returns a Blob with type image/jpeg', async () => {
+ const { drawHighlight } = await import('../src/background/screenshot');
+ const meta = makeElementMeta();
+ const result = await drawHighlight('data:image/jpeg;base64,fake', meta);
+
+ expect(result).toBeInstanceOf(Blob);
+ expect(result.type).toBe('image/jpeg');
+ });
+});
diff --git a/tests/spa-navigation.test.ts b/tests/spa-navigation.test.ts
new file mode 100644
index 0000000..b2acd4a
--- /dev/null
+++ b/tests/spa-navigation.test.ts
@@ -0,0 +1,27 @@
+import { describe, it, expect } from 'vitest';
+import type { SpaNavigateMessage } from '../src/shared/messages';
+
+describe('SPA navigation', () => {
+ it('SPA_NAVIGATE message has the correct shape', () => {
+ const message: SpaNavigateMessage = {
+ type: 'SPA_NAVIGATE',
+ url: 'https://example.com/page2',
+ guideId: 'guide-123',
+ };
+ expect(message.type).toBe('SPA_NAVIGATE');
+ expect(typeof message.url).toBe('string');
+ expect(typeof message.guideId).toBe('string');
+ });
+
+ it('webNavigation listener should only process frameId === 0 (top frame)', () => {
+ const topFrameDetails = { frameId: 0, url: 'https://example.com/page2', tabId: 1 };
+ const subFrameDetails = { frameId: 1, url: 'https://example.com/iframe', tabId: 1 };
+
+ function shouldProcess(details: { frameId: number }): boolean {
+ return details.frameId === 0;
+ }
+
+ expect(shouldProcess(topFrameDetails)).toBe(true);
+ expect(shouldProcess(subFrameDetails)).toBe(false);
+ });
+});
diff --git a/tests/step-description.test.ts b/tests/step-description.test.ts
new file mode 100644
index 0000000..fd916af
--- /dev/null
+++ b/tests/step-description.test.ts
@@ -0,0 +1,87 @@
+import { describe, it, expect } from 'vitest';
+import { buildFallbackDescription } from '../src/background/step-description';
+import type { ElementMeta } from '../src/shared/types';
+
+function makeMeta(overrides: Partial<ElementMeta> = {}): ElementMeta {
+ return {
+ tag: 'div',
+ cssSelector: 'div',
+ textContent: null,
+ ariaLabel: null,
+ placeholder: null,
+ altText: null,
+ name: null,
+ role: null,
+ href: null,
+ inputType: null,
+ dataTestId: null,
+ rect: { x: 0, y: 0, width: 100, height: 50 },
+ devicePixelRatio: 1,
+ ...overrides,
+ };
+}
+
+describe('buildFallbackDescription', () => {
+ it('click action with ariaLabel returns "Click {ariaLabel}"', () => {
+ const meta = makeMeta({ ariaLabel: 'Submit button' });
+ expect(buildFallbackDescription('click', meta)).toBe('Click Submit button');
+ });
+
+ it('click action with no ariaLabel but textContent returns "Click {textContent}" (truncated to 80 chars)', () => {
+ const meta = makeMeta({ textContent: 'Sign in' });
+ expect(buildFallbackDescription('click', meta)).toBe('Click Sign in');
+ });
+
+ it('textContent is truncated to 80 chars', () => {
+ const longText = 'A'.repeat(100);
+ const meta = makeMeta({ textContent: longText });
+ const result = buildFallbackDescription('click', meta);
+ expect(result).toBe('Click ' + 'A'.repeat(80));
+ });
+
+ it('input action with placeholder returns "Type into {placeholder}"', () => {
+ const meta = makeMeta({ placeholder: 'Enter your email' });
+ expect(buildFallbackDescription('input', meta)).toBe('Type into Enter your email');
+ });
+
+ it('scroll action returns "Scroll the page" regardless of element metadata', () => {
+ const meta = makeMeta({ ariaLabel: 'some label', textContent: 'some text' });
+ expect(buildFallbackDescription('scroll', meta)).toBe('Scroll the page');
+ });
+
+ it('navigate action returns "Navigate to page"', () => {
+ const meta = makeMeta({ ariaLabel: 'Navigation link' });
+ expect(buildFallbackDescription('navigate', meta)).toBe('Navigate to page');
+ });
+
+ it('priority chain: ariaLabel > placeholder > textContent > altText > name > role > tag', () => {
+ const meta1 = makeMeta({
+ ariaLabel: 'aria', placeholder: 'placeholder', textContent: 'text',
+ altText: 'alt', name: 'nameAttr', role: 'button', tag: 'button',
+ });
+ expect(buildFallbackDescription('click', meta1)).toBe('Click aria');
+
+ const meta2 = makeMeta({
+ placeholder: 'placeholder', textContent: 'text',
+ altText: 'alt', name: 'nameAttr', role: 'button', tag: 'button',
+ });
+ expect(buildFallbackDescription('input', meta2)).toBe('Type into placeholder');
+
+ const meta3 = makeMeta({ textContent: 'text', altText: 'alt', name: 'nameAttr', role: 'button', tag: 'button' });
+ expect(buildFallbackDescription('click', meta3)).toBe('Click text');
+
+ const meta4 = makeMeta({ altText: 'alt', name: 'nameAttr', role: 'button', tag: 'img' });
+ expect(buildFallbackDescription('click', meta4)).toBe('Click alt');
+
+ const meta5 = makeMeta({ name: 'nameAttr', role: 'button', tag: 'input' });
+ expect(buildFallbackDescription('click', meta5)).toBe('Click nameAttr');
+
+ const meta6 = makeMeta({ role: 'button', tag: 'div' });
+ expect(buildFallbackDescription('click', meta6)).toBe('Click button');
+ });
+
+ it('when all metadata is null, falls back to tag name', () => {
+ const meta = makeMeta({ tag: 'button' });
+ expect(buildFallbackDescription('click', meta)).toBe('Click button');
+ });
+});
diff --git a/wxt.config.ts b/wxt.config.ts
index 2a9189e..5606e0d 100644
--- a/wxt.config.ts
+++ b/wxt.config.ts
@@ -9,7 +9,7 @@ export default defineConfig({
manifest: {
name: 'Mimik',
description: 'Auto-capture browser workflows and generate step-by-step guides with annotated screenshots.',
- permissions: ['storage', 'activeTab', 'tabs', 'scripting', 'unlimitedStorage', 'sidePanel'],
+ permissions: ['storage', 'activeTab', 'tabs', 'scripting', 'unlimitedStorage', 'sidePanel', 'webNavigation'],
host_permissions: ['<all_urls>'],
minimum_chrome_version: '118',
action: {},