Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix(voice): start narration mid-recording and keep ai out of narrated steps

Luis Ortizee6c365d59cc
diff --git a/src/core/guides/__tests__/ai-pending.test.ts b/src/core/guides/__tests__/ai-pending.test.ts
index 8462ce8..aa14ed8 100644
--- a/src/core/guides/__tests__/ai-pending.test.ts
+++ b/src/core/guides/__tests__/ai-pending.test.ts
@@ -24,7 +24,13 @@ const { broadcasts } = vi.hoisted(() => {
});
import { db } from '../db';
-import { applyNarrationToSteps, clearStepAiPending, type GuideChangeEvent, updateStepDescription } from '../service';
+import {
+ applyAiDescription,
+ applyNarrationToSteps,
+ clearStepAiPending,
+ type GuideChangeEvent,
+ updateStepDescription,
+} from '../service';
import type { Guide, Step } from '../types';
const FALLBACK = 'Clicked Save';
@@ -193,3 +199,27 @@ describe('clearStepAiPending', () => {
expect(sibling?.aiPending).toBe(true);
});
});
+
+describe('applyAiDescription', () => {
+ it('fills a step that narration did not cover', async () => {
+ await db.steps.add(makeStep({ id: 's9', aiPending: false }));
+
+ await applyAiDescription('s9', AI_TEXT);
+
+ const step = await db.steps.get('s9');
+ expect(step?.description).toBe(AI_TEXT);
+ expect(step?.descriptionSource).toBe('ai');
+ });
+
+ it('leaves a narrated step alone', async () => {
+ await db.steps.add(makeStep({ id: 's10', description: 'What I said out loud', descriptionSource: 'narration' }));
+
+ await applyAiDescription('s10', AI_TEXT);
+
+ expect((await db.steps.get('s10'))?.description).toBe('What I said out loud');
+ });
+
+ it('does nothing for a step that no longer exists', async () => {
+ await expect(applyAiDescription('gone', AI_TEXT)).resolves.toBeUndefined();
+ });
+});
diff --git a/src/core/guides/service.ts b/src/core/guides/service.ts
index b7c88a5..725b966 100644
--- a/src/core/guides/service.ts
+++ b/src/core/guides/service.ts
@@ -209,6 +209,16 @@ export async function applyNarrationToSteps(updates: readonly NarrationUpdate[])
notifyGuidesChanged({ type: 'mutated' });
}
+export async function applyAiDescription(stepId: string, description: string): Promise<void> {
+ const wrote = await db.transaction('rw', db.steps, async () => {
+ const step = await db.steps.get(stepId);
+ if (!step || step.descriptionSource === 'narration') return false;
+ await db.steps.update(stepId, { description, descriptionSource: 'ai', aiPending: false });
+ return true;
+ });
+ if (wrote) notifyGuidesChanged({ type: 'mutated' });
+}
+
export async function clearStepAiPending(stepId: string, description?: string): Promise<void> {
const wrote = await db.transaction('rw', db.steps, async () => {
const step = await db.steps.get(stepId);
diff --git a/src/entrypoints/background/__tests__/deferred-descriptions.test.ts b/src/entrypoints/background/__tests__/deferred-descriptions.test.ts
new file mode 100644
index 0000000..793e5bd
--- /dev/null
+++ b/src/entrypoints/background/__tests__/deferred-descriptions.test.ts
@@ -0,0 +1,82 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+import type { DOMContext } from '@/core/capture/dom/context';
+import {
+ clearDeferredDescriptions,
+ deferDescription,
+ shouldQueueAiDescription,
+ takeDeferredDescriptions,
+} from '../deferred-descriptions';
+
+const ctx = (name: string) => ({ page: { title: name, path: '/' } }) as DOMContext;
+
+const base = { action: 'click', hasDomContext: true, hasAiKey: true, narrationCapturing: false };
+
+describe('shouldQueueAiDescription', () => {
+ it('queues a description for a normal click', () => {
+ expect(shouldQueueAiDescription(base)).toBe(true);
+ });
+
+ it('does not queue while narration is capturing', () => {
+ expect(shouldQueueAiDescription({ ...base, narrationCapturing: true })).toBe(false);
+ });
+
+ it('does not queue without an api key', () => {
+ expect(shouldQueueAiDescription({ ...base, hasAiKey: false })).toBe(false);
+ });
+
+ it('does not queue without dom context', () => {
+ expect(shouldQueueAiDescription({ ...base, hasDomContext: false })).toBe(false);
+ });
+
+ it('does not queue for input actions', () => {
+ expect(shouldQueueAiDescription({ ...base, action: 'input' })).toBe(false);
+ });
+});
+
+describe('takeDeferredDescriptions', () => {
+ beforeEach(() => {
+ clearDeferredDescriptions('g1');
+ clearDeferredDescriptions('g2');
+ });
+
+ it('returns the steps narration did not cover', () => {
+ deferDescription('g1', 's1', ctx('one'));
+ deferDescription('g1', 's2', ctx('two'));
+
+ const pending = takeDeferredDescriptions('g1', ['s1']);
+
+ expect(pending.map((p) => p.stepId)).toEqual(['s2']);
+ });
+
+ it('returns every step when nothing was narrated', () => {
+ deferDescription('g1', 's1', ctx('one'));
+ deferDescription('g1', 's2', ctx('two'));
+
+ expect(takeDeferredDescriptions('g1', []).map((p) => p.stepId)).toEqual(['s1', 's2']);
+ });
+
+ it('carries the dom context captured at the time', () => {
+ deferDescription('g1', 's1', ctx('billing'));
+
+ expect(takeDeferredDescriptions('g1', [])[0].domContext.page.title).toBe('billing');
+ });
+
+ it('drains, so a second call returns nothing', () => {
+ deferDescription('g1', 's1', ctx('one'));
+ takeDeferredDescriptions('g1', []);
+
+ expect(takeDeferredDescriptions('g1', [])).toEqual([]);
+ });
+
+ it('keeps guides separate', () => {
+ deferDescription('g1', 's1', ctx('one'));
+ deferDescription('g2', 's2', ctx('two'));
+
+ expect(takeDeferredDescriptions('g1', []).map((p) => p.stepId)).toEqual(['s1']);
+ expect(takeDeferredDescriptions('g2', []).map((p) => p.stepId)).toEqual(['s2']);
+ });
+
+ it('returns nothing for a guide that deferred nothing', () => {
+ expect(takeDeferredDescriptions('g1', [])).toEqual([]);
+ });
+});
diff --git a/src/entrypoints/background/__tests__/start-narration.test.ts b/src/entrypoints/background/__tests__/start-narration.test.ts
new file mode 100644
index 0000000..bc630bb
--- /dev/null
+++ b/src/entrypoints/background/__tests__/start-narration.test.ts
@@ -0,0 +1,25 @@
+import { describe, expect, it } from 'vitest';
+import { CaptureState } from '@/core/capture/machine';
+import { canStartNarrationNow } from '../voice';
+
+describe('canStartNarrationNow', () => {
+ it('starts narration when a recording is running and narration is not', () => {
+ expect(canStartNarrationNow(CaptureState.RECORDING, 'idle')).toBe(true);
+ });
+
+ it('does not start when nothing is being recorded', () => {
+ expect(canStartNarrationNow(CaptureState.IDLE, 'idle')).toBe(false);
+ });
+
+ it('does not start a second capture when narration is already running', () => {
+ expect(canStartNarrationNow(CaptureState.RECORDING, 'recording')).toBe(false);
+ });
+
+ it('does not start while a previous take is still transcribing', () => {
+ expect(canStartNarrationNow(CaptureState.RECORDING, 'transcribing')).toBe(false);
+ });
+
+ it('starts again after a failed take', () => {
+ expect(canStartNarrationNow(CaptureState.RECORDING, 'error')).toBe(true);
+ });
+});
diff --git a/src/entrypoints/background/ai-description.ts b/src/entrypoints/background/ai-description.ts
new file mode 100644
index 0000000..85fca81
--- /dev/null
+++ b/src/entrypoints/background/ai-description.ts
@@ -0,0 +1,13 @@
+import { getAIDescription } from '@/core/capture/ai/description';
+import type { DOMContext } from '@/core/capture/dom/context';
+import { localStorage } from '@/lib/browser-api';
+
+export async function generateAiDescription(domContext: DOMContext): Promise<string | undefined> {
+ const settings = await localStorage.get(['aiApiKey', 'aiProvider', 'aiModel']);
+ if (!settings.aiApiKey) return undefined;
+
+ const provider = (settings.aiProvider as string) || 'openai';
+ const model = (settings.aiModel as string) || 'gpt-4o-mini';
+ const description = await getAIDescription(domContext, provider, model, settings.aiApiKey as string);
+ return description || undefined;
+}
diff --git a/src/entrypoints/background/deferred-descriptions.ts b/src/entrypoints/background/deferred-descriptions.ts
new file mode 100644
index 0000000..af98748
--- /dev/null
+++ b/src/entrypoints/background/deferred-descriptions.ts
@@ -0,0 +1,45 @@
+import type { DOMContext } from '@/core/capture/dom/context';
+
+export interface DeferredDescription {
+ stepId: string;
+ domContext: DOMContext;
+}
+
+export interface AiDescriptionInput {
+ action: string;
+ hasDomContext: boolean;
+ hasAiKey: boolean;
+ narrationCapturing: boolean;
+}
+
+const deferred = new Map<string, Map<string, DOMContext>>();
+
+export function shouldQueueAiDescription({
+ action,
+ hasDomContext,
+ hasAiKey,
+ narrationCapturing,
+}: AiDescriptionInput): boolean {
+ return action !== 'input' && hasDomContext && hasAiKey && !narrationCapturing;
+}
+
+export function deferDescription(guideId: string, stepId: string, domContext: DOMContext): void {
+ const forGuide = deferred.get(guideId) ?? new Map<string, DOMContext>();
+ forGuide.set(stepId, domContext);
+ deferred.set(guideId, forGuide);
+}
+
+export function takeDeferredDescriptions(guideId: string, narratedStepIds: readonly string[]): DeferredDescription[] {
+ const forGuide = deferred.get(guideId);
+ if (!forGuide) return [];
+ deferred.delete(guideId);
+
+ const narrated = new Set(narratedStepIds);
+ return [...forGuide]
+ .filter(([stepId]) => !narrated.has(stepId))
+ .map(([stepId, domContext]) => ({ stepId, domContext }));
+}
+
+export function clearDeferredDescriptions(guideId: string): void {
+ deferred.delete(guideId);
+}
diff --git a/src/entrypoints/background/describe-unnarrated.ts b/src/entrypoints/background/describe-unnarrated.ts
new file mode 100644
index 0000000..adc5e28
--- /dev/null
+++ b/src/entrypoints/background/describe-unnarrated.ts
@@ -0,0 +1,13 @@
+import { applyAiDescription } from '@/core/guides/service';
+import { generateAiDescription } from './ai-description';
+import { takeDeferredDescriptions } from './deferred-descriptions';
+import { queueDescription } from './description-queue';
+
+export function describeUnnarratedSteps(guideId: string, narratedStepIds: readonly string[]): void {
+ for (const { stepId, domContext } of takeDeferredDescriptions(guideId, narratedStepIds)) {
+ queueDescription(guideId, async () => {
+ const description = await generateAiDescription(domContext);
+ if (description) await applyAiDescription(stepId, description);
+ });
+ }
+}
diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.ts
index 4932840..e4d5e1c 100644
--- a/src/entrypoints/background/index.ts
+++ b/src/entrypoints/background/index.ts
@@ -28,7 +28,13 @@ import { generateDescriptionOnDemand, generateGuideMetaOnStop, settlePendingDesc
import { registerNavigationListeners } from './navigation';
import { handleCaptureStep, handleFinalizeInputStep, handleUpdateInputStep } from './step-pipeline';
import { broadcastStartCapture, broadcastStopCapture, showNotificationOnTab } from './tab-manager';
-import { getVoiceUpdate, registerVoiceListeners, startVoiceNarration, stopVoiceNarration } from './voice';
+import {
+ canStartNarrationNow,
+ getVoiceUpdate,
+ registerVoiceListeners,
+ startVoiceNarration,
+ stopVoiceNarration,
+} from './voice';
async function resolveManual(step: Step): Promise<boolean> {
if (!step.screenshotId) return stepRequiresManual(step, null);
@@ -142,6 +148,17 @@ export default defineBackground(() => {
return { success: true, guideId: guideId ?? undefined, inserted: false };
});
+ onMessage('startNarration', async () => {
+ await waitUntilReady();
+ const actor = getActor();
+ if (!canStartNarrationNow(String(actor.getSnapshot().value), getVoiceUpdate().phase)) {
+ return { started: false };
+ }
+ const activeTab = await getActiveTab();
+ await startVoiceNarration(activeTab?.id);
+ return { started: getVoiceUpdate().phase === 'recording' };
+ });
+
onMessage('enterBlurMode', async () => {
await waitUntilReady();
await broadcastStopCapture();
diff --git a/src/entrypoints/background/step-pipeline.ts b/src/entrypoints/background/step-pipeline.ts
index c9a37a1..cc1c381 100644
--- a/src/entrypoints/background/step-pipeline.ts
+++ b/src/entrypoints/background/step-pipeline.ts
@@ -1,4 +1,3 @@
-import { getAIDescription } from '@/core/capture/ai/description';
import type { DOMContext } from '@/core/capture/dom/context';
import { CaptureState } from '@/core/capture/machine';
import { buildFallbackDescription } from '@/core/capture/step-description';
@@ -16,7 +15,10 @@ import { captureVisibleTab, localStorage } from '@/lib/browser-api';
import { logger } from '@/lib/logger';
import type { CaptureStepData, CaptureStepResponse } from '@/lib/messaging';
import { getActor } from './actor';
+import { generateAiDescription } from './ai-description';
+import { deferDescription, shouldQueueAiDescription } from './deferred-descriptions';
import { queueDescription } from './description-queue';
+import { getVoiceUpdate } from './voice';
async function takeScreenshot(stepId: string, meta: ElementMeta): Promise<string | undefined> {
try {
@@ -55,14 +57,9 @@ async function takeScreenshot(stepId: string, meta: ElementMeta): Promise<string
}
async function tryAIDescription(stepId: string, domContext: DOMContext) {
- const settings = await localStorage.get(['aiApiKey', 'aiProvider', 'aiModel']);
- if (!settings.aiApiKey) return;
-
- const provider = (settings.aiProvider as string) || 'openai';
- const model = (settings.aiModel as string) || 'gpt-4o-mini';
+ if (!(await localStorage.get(['aiApiKey'])).aiApiKey) return;
try {
- const description = await getAIDescription(domContext, provider, model, settings.aiApiKey as string);
- await clearStepAiPending(stepId, description || undefined);
+ await clearStepAiPending(stepId, await generateAiDescription(domContext));
} catch (err) {
await clearStepAiPending(stepId);
throw err;
@@ -81,7 +78,14 @@ export async function handleCaptureStep(data: CaptureStepData): Promise<CaptureS
const screenshotId = await takeScreenshot(stepId, data.elementMeta);
- const willUseAI = data.action !== 'input' && !!data.domContext && !!(await localStorage.get(['aiApiKey'])).aiApiKey;
+ const narrationCapturing = getVoiceUpdate().phase === 'recording';
+ const hasAiKey = !!(await localStorage.get(['aiApiKey'])).aiApiKey;
+ const willUseAI = shouldQueueAiDescription({
+ action: data.action,
+ hasDomContext: !!data.domContext,
+ hasAiKey,
+ narrationCapturing,
+ });
await createStep({
id: stepId,
@@ -99,7 +103,8 @@ export async function handleCaptureStep(data: CaptureStepData): Promise<CaptureS
const domContext = data.domContext;
if (data.action !== 'input' && domContext) {
- queueDescription(guideId, () => tryAIDescription(stepId, domContext));
+ if (willUseAI) queueDescription(guideId, () => tryAIDescription(stepId, domContext));
+ else if (narrationCapturing && hasAiKey) deferDescription(guideId, stepId, domContext);
}
return { stepId };
diff --git a/src/entrypoints/background/voice.ts b/src/entrypoints/background/voice.ts
index f705994..89bd79c 100644
--- a/src/entrypoints/background/voice.ts
+++ b/src/entrypoints/background/voice.ts
@@ -1,3 +1,4 @@
+import { CaptureState } from '@/core/capture/machine';
import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@/core/capture/voice/api-key';
import { narrationUpdates } from '@/core/capture/voice/narration-updates';
import { applyNarrationToSteps, findExistingStepIds, getStepsForGuide } from '@/core/guides/service';
@@ -15,6 +16,7 @@ import {
stopVoiceCapture,
supportsVoice,
} from '@/lib/offscreen';
+import type { VoicePhase } from '@/lib/port';
import { broadcastVoiceToPanel, type PanelVoiceUpdate } from '@/lib/port';
import {
isVoiceMessageFor,
@@ -29,6 +31,7 @@ import {
} from '@/lib/voice-messages';
import { narrateRecording, readTranscriptionSettings, type VoiceRecording } from '@/lib/voice-narration';
import { handedOffPcm, voiceStopAction } from '@/lib/voice-recovery';
+import { describeUnnarratedSteps } from './describe-unnarrated';
const START_TIMEOUT_MS = 8000;
@@ -102,6 +105,10 @@ async function beginVoiceCapture(microphoneId: string | undefined, tabId: number
await closeVoiceHost();
}
+export function canStartNarrationNow(captureState: string, voicePhase: VoicePhase): boolean {
+ return captureState === CaptureState.RECORDING && voicePhase !== 'recording' && voicePhase !== 'transcribing';
+}
+
export async function startVoiceNarration(tabId?: number): Promise<void> {
if (!supportsVoice()) {
logger.warn('voice: no microphone host available in this browser, recording without narration');
@@ -188,7 +195,7 @@ export async function stopVoiceNarration(guideId: string): Promise<void> {
}
}
-async function applyNarration(_guideId: string, result: VoiceResultEvent['result']): Promise<void> {
+async function applyNarration(guideId: string, result: VoiceResultEvent['result']): Promise<void> {
try {
transcribingGuideId = null;
const narrated = result.descriptions.map((entry) => entry.stepId);
@@ -197,6 +204,10 @@ async function applyNarration(_guideId: string, result: VoiceResultEvent['result
await applyNarrationToSteps(updates);
logger.info('voice: narration applied', { narrated: updates.length, of: narrated.length, stats: result.stats });
report({ phase: 'idle', narrated: updates.length });
+ describeUnnarratedSteps(
+ guideId,
+ updates.map((update) => update.stepId),
+ );
} catch (error) {
logger.error('voice: narration could not be applied', error);
report({ phase: 'error', reason: 'unknown', error: String(error) });
diff --git a/src/lib/messaging.ts b/src/lib/messaging.ts
index 694c5ec..6600c5b 100644
--- a/src/lib/messaging.ts
+++ b/src/lib/messaging.ts
@@ -121,6 +121,10 @@ export interface EnterBlurModeResponse {
entered: boolean;
}
+export interface StartNarrationResponse {
+ started: boolean;
+}
+
export interface ExitBlurModeResponse {
exited: boolean;
}
@@ -138,6 +142,7 @@ interface MimikProtocol {
guideMeGoTo(data: GuideMe_GoToData): GuideMe_GoToResponse;
enterBlurMode(): EnterBlurModeResponse;
exitBlurMode(): ExitBlurModeResponse;
+ startNarration(): StartNarrationResponse;
generateGuideDescription(data: GenerateGuideDescriptionData): GenerateGuideDescriptionResponse;
validateApiKey(data: ValidateApiKeyData): ValidateApiKeyResponse;
rewriteSelection(data: RewriteSelectionData): RewriteSelectionResponse;
diff --git a/src/ui/sidepanel/MicToggle.tsx b/src/ui/sidepanel/MicToggle.tsx
index 31cb8d1..50de1c9 100644
--- a/src/ui/sidepanel/MicToggle.tsx
+++ b/src/ui/sidepanel/MicToggle.tsx
@@ -3,6 +3,7 @@ import { useCallback, useEffect, useState } from 'react';
import { browser, i18n } from '#imports';
import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@/core/capture/voice/api-key';
import { getActiveTab, localStorage } from '@/lib/browser-api';
+import { sendMessage } from '@/lib/messaging';
import { abortVoiceCapture, openMicPermissionPage } from '@/lib/offscreen';
import { Tooltip, TooltipContent, TooltipTrigger } from '@/ui/components/ui/tooltip';
@@ -59,7 +60,10 @@ export default function MicToggle({ enabled, live, onChange }: MicToggleProps) {
return;
}
- if (await microphoneGranted()) return;
+ if (await microphoneGranted()) {
+ await sendMessage('startNarration', undefined).catch(() => undefined);
+ return;
+ }
const tab = await getActiveTab();
await openMicPermissionPage(tab?.id).catch(() => undefined);
}, [enabled, live, locked, onChange]);