Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

refactor: simplify background step pipeline and message handlers

Luis Ortizd4f4ce026431
diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.ts
index 9f0e85d..af35582 100644
--- a/src/entrypoints/background/index.ts
+++ b/src/entrypoints/background/index.ts
@@ -3,8 +3,9 @@ import { logger } from "@/lib/logger";
import { setSidePanelBehavior, getActiveTab, localStorage } from "@/lib/browser-api";
import { onMessage } from "@/lib/messaging";
import { setupPortListener, broadcastStateToPanel } from "@/lib/port";
-import { createGuide, saveRrwebChunk, getStepsForGuide, updateGuideTitle } from "@/core/guides/service";
-import { generateGuideTitle } from "@/core/capture/ai-description";
+import { createGuide, saveRrwebChunk, getStepsForGuide, updateGuideTitle, updateStepDescription } from "@/core/guides/service";
+import { generateGuideTitle } from "@/core/capture/ai/title";
+import { handleCaptureStep, handleUpdateInputStep, handleFinalizeInputStep } from "./step-pipeline";
import {
initActor,
initActorFallback,
@@ -13,7 +14,6 @@ import {
waitUntilReady,
} from "./actor";
import { broadcastStartCapture, broadcastStopCapture, showNotificationOnTab } from "./tab-manager";
-import { handleUserAction } from "./step-pipeline";
import { registerNavigationListeners } from "./navigation";
async function generateTitleInBackground(guideId: string) {
@@ -22,12 +22,15 @@ async function generateTitleInBackground(guideId: string) {
if (!settings.aiApiKey) return;
const steps = await getStepsForGuide(guideId);
- const descriptions = steps.map(s => s.description).filter(Boolean);
- if (descriptions.length === 0) return;
+ const allSteps = steps.filter(s => s.description).map(s => ({ description: s.description, url: s.url }));
+ if (allSteps.length === 0) return;
+ const stepsWithUrl = allSteps.length > 15
+ ? [...allSteps.slice(0, 10), ...allSteps.slice(-5)]
+ : allSteps;
const provider = (settings.aiProvider as string) || 'openai';
const model = (settings.aiModel as string) || 'gpt-4o-mini';
- const title = await generateGuideTitle(descriptions, provider, model, settings.aiApiKey as string);
+ const title = await generateGuideTitle(stepsWithUrl, provider, model, settings.aiApiKey as string);
if (title) {
await updateGuideTitle(guideId, title);
logger.info('Generated guide title:', title);
@@ -47,9 +50,7 @@ export default defineBackground(() => {
setupPortListener((port) => {
logger.debug("Panel connected via port");
waitUntilReady().then(() => {
- try {
- port.postMessage(getStateUpdate());
- } catch {}
+ try { port.postMessage(getStateUpdate()); } catch {}
});
});
@@ -59,13 +60,10 @@ export default defineBackground(() => {
onMessage("getState", async () => {
await waitUntilReady();
- const update = getStateUpdate();
- logger.debug("getState β†’", update.state);
- return update;
+ return getStateUpdate();
});
onMessage("startRecording", async ({ data }) => {
- logger.info("startRecording β†’", data.url);
await waitUntilReady();
const actor = getActor();
actor.send({ type: "START_RECORDING", url: data.url });
@@ -74,12 +72,9 @@ export default defineBackground(() => {
await createGuide(guideId);
const activeTab = await getActiveTab();
- if (activeTab?.id) {
- await showNotificationOnTab(activeTab.id);
- }
+ if (activeTab?.id) await showNotificationOnTab(activeTab.id);
await broadcastStartCapture(guideId);
- logger.info("Recording started β†’ guideId:", guideId);
return { guideId };
});
@@ -87,25 +82,32 @@ export default defineBackground(() => {
await waitUntilReady();
const actor = getActor();
const guideId = actor.getSnapshot().context.currentGuideId;
- logger.info("stopRecording β†’ guideId:", guideId);
await broadcastStopCapture();
actor.send({ type: "STOP_RECORDING" });
- if (guideId) {
- generateTitleInBackground(guideId);
- }
+ if (guideId) generateTitleInBackground(guideId);
return { success: true, guideId: guideId ?? undefined };
});
- onMessage("userAction", async (message) => {
+ onMessage("captureStep", async ({ data }) => {
+ await waitUntilReady();
+ return handleCaptureStep(data);
+ });
+
+ onMessage("updateInputStep", async ({ data }) => {
+ await waitUntilReady();
+ await handleUpdateInputStep(data.stepId, data.description);
+ return { updated: true };
+ });
+
+ onMessage("finalizeInputStep", async ({ data }) => {
await waitUntilReady();
- logger.debug("userAction β†’", message.data.action);
- return handleUserAction(message.data, message.sender);
+ await handleFinalizeInputStep(data.stepId, data.elementMeta, data.domContext);
+ return { updated: true };
});
onMessage("rrwebChunk", async ({ data }) => {
- logger.debug("rrwebChunk β†’", data.events.length, "events");
await saveRrwebChunk({
id: crypto.randomUUID(),
guideId: data.guideId,
diff --git a/src/entrypoints/background/step-pipeline.ts b/src/entrypoints/background/step-pipeline.ts
index 3486da6..79a55e9 100644
--- a/src/entrypoints/background/step-pipeline.ts
+++ b/src/entrypoints/background/step-pipeline.ts
@@ -1,107 +1,97 @@
import { logger } from '@/lib/logger';
-import { localStorage, sendMessageToTab } from '@/lib/browser-api';
-import { TabMessage } from '@/lib/tab-messages';
-import { captureAnnotated } from '@/core/capture/screenshot';
+import { localStorage, captureVisibleTab } from '@/lib/browser-api';
import { buildFallbackDescription } from '@/core/capture/step-description';
-import { getAIDescription } from '@/core/capture/ai-description';
-import { createStep, addStepToGuide, updateStepDescription } from '@/core/guides/service';
+import { getAIDescription } from '@/core/capture/ai/description';
+import { createStep, addStepToGuide, updateStepDescription, saveScreenshot } from '@/core/guides/service';
+import { db } from '@/core/guides/db';
import { getActor } from './actor';
-import type { Browser } from '#imports';
import { CaptureState } from '@/core/capture/machine';
-import type { UserActionData, UserActionResponse } from '@/lib/messaging';
+import type { CaptureStepData, CaptureStepResponse } from '@/lib/messaging';
+import type { ElementMeta, Screenshot } from '@/core/guides/types';
+import type { DOMContext } from '@/core/capture/dom/context';
-let captureQueue: Promise<void> = Promise.resolve();
-
-function enqueue(fn: () => Promise<void>): void {
- captureQueue = captureQueue
- .then(fn)
- .catch((err) => logger.error(' Capture error', err));
+async function takeScreenshot(stepId: string, meta: ElementMeta): Promise<string | undefined> {
+ try {
+ const dataUrl = await captureVisibleTab('jpeg', 90);
+ const blob = await fetch(dataUrl).then(r => r.blob());
+ const img = await createImageBitmap(blob);
+ const screenshot: Screenshot = {
+ id: crypto.randomUUID(),
+ stepId,
+ blob,
+ mimeType: 'image/jpeg',
+ width: img.width,
+ height: img.height,
+ bounds: { x: meta.rect.x, y: meta.rect.y, width: meta.rect.width, height: meta.rect.height },
+ pixelRatio: meta.devicePixelRatio,
+ };
+ img.close();
+ await saveScreenshot(screenshot);
+ return screenshot.id;
+ } catch (err) {
+ logger.warn('Screenshot capture failed', err);
+ return undefined;
+ }
}
-export function handleUserAction(
- data: UserActionData,
- sender: Browser.runtime.MessageSender,
-): UserActionResponse {
- const actor = getActor();
- const snap = actor.getSnapshot();
+async function tryAIDescription(stepId: string, domContext: DOMContext) {
+ const settings = await localStorage.get(['aiApiKey', 'aiProvider', 'aiModel']);
+ if (!settings.aiApiKey) return;
- if (snap.value !== CaptureState.RECORDING) {
- return { ignored: true };
- }
+ const provider = (settings.aiProvider as string) || 'openai';
+ const model = (settings.aiModel as string) || 'gpt-4o-mini';
+ const description = await getAIDescription(domContext, provider, model, settings.aiApiKey as string);
+ if (description) await updateStepDescription(stepId, description);
+}
+
+export async function handleCaptureStep(data: CaptureStepData): Promise<CaptureStepResponse> {
+ const snap = getActor().getSnapshot();
+ if (snap.value !== CaptureState.RECORDING) return { ignored: true };
const stepIndex = snap.context.stepCount;
- actor.send({ type: 'USER_ACTION' });
+ getActor().send({ type: 'USER_ACTION' });
+
const guideId = snap.context.currentGuideId!;
const stepId = crypto.randomUUID();
- const tabId = sender.tab?.id;
-
- if (!tabId) {
- return { error: 'no tabId' };
- }
-
- enqueue(async () => {
- const fallbackDescription = buildFallbackDescription(data.action, data.elementMeta);
- const currentUrl = snap.context.currentUrl;
- let screenshotId: string | undefined;
- let screenshotBlob: Blob | undefined;
- try {
- logger.debug('Hiding overlay before screenshot...');
- const hideResp = await sendMessageToTab(tabId, { type: TabMessage.HIDE_OVERLAY }).catch((e) => {
- logger.warn('HIDE_OVERLAY failed:', e);
- return null;
- });
- logger.debug('HIDE_OVERLAY response:', hideResp);
+ const screenshotId = await takeScreenshot(stepId, data.elementMeta);
- await new Promise(r => setTimeout(r, 150));
- logger.debug('Taking screenshot...');
+ await createStep({
+ id: stepId,
+ guideId,
+ index: stepIndex,
+ description: buildFallbackDescription(data.action, data.elementMeta),
+ action: data.action,
+ url: snap.context.currentUrl,
+ timestamp: Date.now(),
+ screenshotId,
+ });
+ await addStepToGuide(guideId, stepId);
- const screenshot = await captureAnnotated(tabId, stepId, data.elementMeta);
- logger.debug('Screenshot taken, showing overlay...');
- await sendMessageToTab(tabId, { type: TabMessage.SHOW_OVERLAY }).catch(() => {});
- screenshotId = screenshot.id;
- screenshotBlob = screenshot.blob;
- } catch (err) {
- await sendMessageToTab(tabId, { type: TabMessage.SHOW_OVERLAY }).catch(() => {});
- logger.warn('Screenshot capture failed, storing step without screenshot', err);
- }
+ if (data.action !== 'input' && data.domContext) {
+ try { await tryAIDescription(stepId, data.domContext); }
+ catch (err) { logger.error('AI description failed', err); }
+ }
- await createStep({
- id: stepId,
- guideId,
- index: stepIndex,
- description: fallbackDescription,
- action: data.action,
- url: currentUrl,
- timestamp: Date.now(),
- screenshotId,
- });
+ return { stepId };
+}
- await addStepToGuide(guideId, stepId);
+export async function handleUpdateInputStep(stepId: string, description: string) {
+ await updateStepDescription(stepId, description);
+}
- if (screenshotBlob) {
- try {
- const settings = await localStorage.get(['aiApiKey', 'aiProvider', 'aiModel']);
- if (settings.aiApiKey) {
- const provider = (settings.aiProvider as string) || 'openai';
- const model = (settings.aiModel as string) || 'gpt-4o-mini';
- const aiDescription = await getAIDescription(
- screenshotBlob,
- data.action,
- data.elementMeta,
- provider,
- model,
- settings.aiApiKey as string,
- );
- if (aiDescription) {
- await updateStepDescription(stepId, aiDescription);
- }
- }
- } catch (err) {
- logger.error('AI description update failed', err);
- }
- }
- });
+export async function handleFinalizeInputStep(
+ stepId: string,
+ elementMeta: ElementMeta,
+ domContext: DOMContext | undefined,
+) {
+ const screenshotId = await takeScreenshot(stepId, elementMeta);
+ if (screenshotId) {
+ await db.steps.update(stepId, { screenshotId });
+ }
- return { stepId };
+ if (domContext) {
+ try { await tryAIDescription(stepId, domContext); }
+ catch (err) { logger.error('AI description failed on finalize', err); }
+ }
}