π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
fix: wait for narration before naming or merging, share narration handling in core and let the background turn the mic off
Luis Ortizf388a309cd7c
diff --git a/apps/desktop/src/renderer/narration.ts b/apps/desktop/src/renderer/narration.tsindex 6347124..b3df548 100644--- a/apps/desktop/src/renderer/narration.ts+++ b/apps/desktop/src/renderer/narration.ts@@ -1,28 +1,21 @@-import { queueDescription } from '@mimik/core/capture/ai/description-queue';-import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@mimik/core/capture/voice/api-key';+import { applyNarrationResult } from '@mimik/core/capture/voice/apply-narration-result';+import { deferDescription } from '@mimik/core/capture/voice/deferred-descriptions';+import { describeStepNow, describeUnnarratedSteps } from '@mimik/core/capture/voice/describe-unnarrated';import { MicRecorder } from '@mimik/core/capture/voice/mic-recorder';import { narrateRecording, type VoiceRecording } from '@mimik/core/capture/voice/narrate-recording';-import { narrationUpdates } from '@mimik/core/capture/voice/narration-updates';+import { NARRATION_SETTLE_MS } from '@mimik/core/capture/voice/narration-settle-ms';import { partialRecording } from '@mimik/core/capture/voice/partial-recording';import { readTranscriptionSettings } from '@mimik/core/capture/voice/read-transcription-settings';+import { readVoiceSettings } from '@mimik/core/capture/voice/read-voice-settings';import { startFailureReason } from '@mimik/core/capture/voice/start-failure-reason';import type { StepMark } from '@mimik/core/capture/voice/step-windows';import { usableRecording } from '@mimik/core/capture/voice/usable-recording';import type { VoiceUpdate } from '@mimik/core/capture/voice/voice-update';+import type { Describe } from '@mimik/core/capture/write-step';import { assetUrl, localStorage } from '@mimik/core/env';-import {- applyNarrationToSteps,- clearStepAiPending,- findExistingStepIds,- getStepsForGuide,- saveTranscript,-} from '@mimik/core/guides/service';+import { clearStepAiPending, getStepsForGuide } from '@mimik/core/guides/service';import { logger } from '@mimik/core/logger';-const SETTLE_MS = 30_000;--type Describe = () => Promise<void>;-export class DesktopNarration {private live: Promise<MicRecorder | null> | null = null;private mic: MicRecorder | null = null;@@ -33,7 +26,6 @@ export class DesktopNarration {private current: VoiceUpdate = { phase: 'idle' };private readonly listeners = new Set<(update: VoiceUpdate) => void>();private readonly pending = new Set<Promise<void>>();- private readonly deferredDescriptions = new Map<string, { guideId: string; describe: Describe | null }>();get update(): VoiceUpdate {return this.current;@@ -51,19 +43,19 @@ export class DesktopNarration {}flushForStep(guideId: string, mark: StepMark, describe: Describe | null): void {- this.deferredDescriptions.set(mark.stepId, { guideId, describe });+ deferDescription(guideId, mark.stepId, describe ?? (() => this.undescribed(mark.stepId)));const mic = this.mic;this.track(async () => {const full = mic && usableRecording(mic.snapshot());const closesAt = full ? (mark.timestamp - full.audioEpochMs) / 1000 : 0;const slice = full && closesAt > this.flushedUpTo ? partialRecording(full, this.flushedUpTo, closesAt) : null;if (!slice) {- this.describeStepNow(mark.stepId);+ describeStepNow(guideId, mark.stepId);return;}this.flushedUpTo = closesAt;const narrated = await this.transcribe(guideId, slice, [mark]);- if (!narrated.includes(mark.stepId)) this.describeStepNow(mark.stepId);+ if (!narrated.includes(mark.stepId)) describeStepNow(guideId, mark.stepId);});}@@ -92,7 +84,7 @@ export class DesktopNarration {steps.map((step) => ({ stepId: step.id, timestamp: step.timestamp })),);}- this.describeDeferred(guideId);+ describeUnnarratedSteps(guideId, []);this.settled();});}@@ -108,13 +100,13 @@ export class DesktopNarration {mic?.release();window.mimik.capture.narration(null);});- if (guideId) this.describeDeferred(guideId);+ if (guideId) describeUnnarratedSteps(guideId, []);if (this.current.phase === 'recording') this.report({ phase: 'idle' });}async settle(): Promise<void> {this.stop();- await Promise.race([Promise.all(this.pending), new Promise((resolve) => setTimeout(resolve, SETTLE_MS))]);+ await Promise.race([Promise.all(this.pending), new Promise((resolve) => setTimeout(resolve, NARRATION_SETTLE_MS))]);}private report(update: VoiceUpdate): void {@@ -133,10 +125,9 @@ export class DesktopNarration {if (this.current.phase === 'transcribing') this.report({ phase: 'idle', narrated: this.narrated });}- private describeDeferred(guideId: string): void {- for (const [stepId, entry] of [...this.deferredDescriptions]) {- if (entry.guideId === guideId) this.describeStepNow(stepId);- }+ private async undescribed(stepId: string): Promise<void> {+ await clearStepAiPending(stepId);+ window.mimik.capture.described(stepId, null, null);}private track(work: () => Promise<void>): void {@@ -146,21 +137,10 @@ export class DesktopNarration {this.pending.add(running);}- private describeStepNow(stepId: string): void {- const entry = this.deferredDescriptions.get(stepId);- if (!entry) return;- this.deferredDescriptions.delete(stepId);- if (entry.describe) {- queueDescription(entry.guideId, entry.describe);- return;- }- void clearStepAiPending(stepId).then(() => window.mimik.capture.described(stepId, null, null));- }-private async open(): Promise<MicRecorder | null> {- const stored = await localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled', 'voiceMicrophoneId']);- if (stored.voiceEnabled !== true) return null;- if (!hasVoiceApiKey(stored)) {+ const voice = await readVoiceSettings();+ if (!voice.enabled) return null;+ if (!voice.hasApiKey) {this.report({ phase: 'error', reason: 'missing-api-key' });return null;}@@ -173,7 +153,7 @@ export class DesktopNarration {},});try {- await mic.start(stored.voiceMicrophoneId || undefined);+ await mic.start(voice.microphoneId);if (!this.live) {mic.release();return null;@@ -195,14 +175,9 @@ export class DesktopNarration {private async transcribe(guideId: string, audio: VoiceRecording, marks: StepMark[]): Promise<string[]> {const settings = await readTranscriptionSettings();if (!settings.apiKey) return [];- const result = await narrateRecording(audio, marks, settings);- await saveTranscript(guideId, result.transcript);- const surviving = await findExistingStepIds(result.descriptions.map((entry) => entry.stepId));- const updates = narrationUpdates(result, surviving);- await applyNarrationToSteps(updates, result.transcript.epochMs);+ const updates = await applyNarrationResult(guideId, await narrateRecording(audio, marks, settings));this.narrated += updates.length;for (const { stepId, description } of updates) {- this.deferredDescriptions.delete(stepId);window.mimik.capture.described(stepId, description, null, 'narration');}return updates.map((update) => update.stepId);diff --git a/packages/core/src/capture/ai/__tests__/merge-recording.test.ts b/packages/core/src/capture/ai/__tests__/merge-recording.test.tsnew file mode 100644index 0000000..c11a237--- /dev/null+++ b/packages/core/src/capture/ai/__tests__/merge-recording.test.ts@@ -0,0 +1,32 @@+import { describe, expect, it, vi } from 'vitest';++const order: string[] = [];++vi.mock('@/core/guides/service', () => ({+ createSnapshot: vi.fn(async () => order.push('snapshot')),+ mergeGuideInto: vi.fn(async () => order.push('merge')),+}));+vi.mock('../settle-descriptions', () => ({ settleDescriptions: vi.fn(async () => order.push('descriptions')) }));++import { mergeGuideInto } from '@/core/guides/service';+import { mergeRecording } from '../merge-recording';++describe('mergeRecording', () => {+ it('waits for the narration and the descriptions before it snapshots and merges', async () => {+ let finishNarration = () => {};+ const narration = new Promise<void>((resolve) => {+ finishNarration = resolve;+ });+ const merging = mergeRecording('staging', { insertTargetGuideId: 'target', insertAtIndex: 2 }, () =>+ narration.then(() => void order.push('narration')),+ );++ await Promise.resolve();+ expect(order).toEqual([]);++ finishNarration();+ await merging;+ expect(order).toEqual(['narration', 'descriptions', 'snapshot', 'merge']);+ expect(mergeGuideInto).toHaveBeenCalledWith('staging', 'target', 2);+ });+});diff --git a/packages/core/src/capture/ai/merge-recording.ts b/packages/core/src/capture/ai/merge-recording.tsnew file mode 100644index 0000000..a17fc8c--- /dev/null+++ b/packages/core/src/capture/ai/merge-recording.ts@@ -0,0 +1,14 @@+import { createSnapshot, mergeGuideInto } from '@/core/guides/service';+import type { CaptureInsert } from '../capture-insert';+import { settleDescriptions } from './settle-descriptions';++export async function mergeRecording(+ guideId: string,+ { insertTargetGuideId, insertAtIndex }: Pick<CaptureInsert, 'insertTargetGuideId' | 'insertAtIndex'>,+ settleNarration: () => Promise<void>,+): Promise<void> {+ await settleNarration();+ await settleDescriptions(guideId);+ await createSnapshot(insertTargetGuideId);+ await mergeGuideInto(guideId, insertTargetGuideId, insertAtIndex);+}diff --git a/packages/core/src/capture/voice/__tests__/apply-narration-result.test.ts b/packages/core/src/capture/voice/__tests__/apply-narration-result.test.tsnew file mode 100644index 0000000..3209e87--- /dev/null+++ b/packages/core/src/capture/voice/__tests__/apply-narration-result.test.ts@@ -0,0 +1,36 @@+import { beforeEach, describe, expect, it, vi } from 'vitest';+import type { NarrationResult } from '../types';++vi.mock('@/core/guides/service', () => ({+ saveTranscript: vi.fn(async () => {+ throw new Error('quota');+ }),+ findExistingStepIds: vi.fn(async (ids: string[]) => ids.filter((id) => id !== 'deleted')),+ applyNarrationToSteps: vi.fn(async () => undefined),+}));++import { applyNarrationResult } from '../apply-narration-result';+import { clearDeferredDescriptions, deferDescription, takeDeferredDescriptions } from '../deferred-descriptions';++const result = {+ descriptions: [+ { stepId: 's1', text: 'Open billing' },+ { stepId: 'deleted', text: 'Gone' },+ ],+ transcript: { epochMs: 0, lines: [] },+ stats: {},+} as unknown as NarrationResult;++beforeEach(() => clearDeferredDescriptions('g'));++describe('applyNarrationResult', () => {+ it('narrates the surviving steps and drops their held AI descriptions, even when the transcript fails to save', async () => {+ deferDescription('g', 's1', async () => undefined);+ deferDescription('g', 's2', async () => undefined);++ const updates = await applyNarrationResult('g', result);++ expect(updates.map((update) => update.stepId)).toEqual(['s1']);+ expect(takeDeferredDescriptions('g', []).map((held) => held.stepId)).toEqual(['s2']);+ });+});diff --git a/src/entrypoints/background/__tests__/deferred-descriptions.test.ts b/packages/core/src/capture/voice/__tests__/deferred-descriptions.test.tssimilarity index 100%rename from src/entrypoints/background/__tests__/deferred-descriptions.test.tsrename to packages/core/src/capture/voice/__tests__/deferred-descriptions.test.tsdiff --git a/packages/core/src/capture/voice/apply-narration-result.ts b/packages/core/src/capture/voice/apply-narration-result.tsnew file mode 100644index 0000000..89e9aab--- /dev/null+++ b/packages/core/src/capture/voice/apply-narration-result.ts@@ -0,0 +1,19 @@+import { applyNarrationToSteps, findExistingStepIds, saveTranscript } from '@/core/guides/service';+import { logger } from '@/core/logger';+import { discardDeferred } from './deferred-descriptions';+import { type NarrationUpdate, narrationUpdates } from './narration-updates';+import type { NarrationResult } from './types';++export async function applyNarrationResult(guideId: string, result: NarrationResult): Promise<NarrationUpdate[]> {+ await saveTranscript(guideId, result.transcript).catch((error: unknown) =>+ logger.warn('voice: the transcript could not be stored', error),+ );+ const surviving = await findExistingStepIds(result.descriptions.map((entry) => entry.stepId));+ const updates = narrationUpdates(result, surviving);+ await applyNarrationToSteps(updates, result.transcript.epochMs);+ discardDeferred(+ guideId,+ updates.map((update) => update.stepId),+ );+ return updates;+}diff --git a/src/entrypoints/background/deferred-descriptions.ts b/packages/core/src/capture/voice/deferred-descriptions.tssimilarity index 95%rename from src/entrypoints/background/deferred-descriptions.tsrename to packages/core/src/capture/voice/deferred-descriptions.tsindex 776465f..f76b152 100644--- a/src/entrypoints/background/deferred-descriptions.ts+++ b/packages/core/src/capture/voice/deferred-descriptions.ts@@ -1,4 +1,4 @@-import type { Describe } from '@/core/capture/write-step';+import type { Describe } from '../write-step';export interface DeferredDescription {stepId: string;diff --git a/src/entrypoints/background/describe-unnarrated.ts b/packages/core/src/capture/voice/describe-unnarrated.tssimilarity index 85%rename from src/entrypoints/background/describe-unnarrated.tsrename to packages/core/src/capture/voice/describe-unnarrated.tsindex a17bb4e..9144aa1 100644--- a/src/entrypoints/background/describe-unnarrated.ts+++ b/packages/core/src/capture/voice/describe-unnarrated.ts@@ -1,6 +1,6 @@-import { logger } from '@mimik/core/logger';-import { queueDescription } from '@/core/capture/ai/description-queue';import { clearStepAiPending } from '@/core/guides/service';+import { logger } from '@/core/logger';+import { queueDescription } from '../ai/description-queue';import { takeDeferredDescription, takeDeferredDescriptions } from './deferred-descriptions';export function describeStepNow(guideId: string, stepId: string): void {diff --git a/packages/core/src/capture/voice/narration-settle-ms.ts b/packages/core/src/capture/voice/narration-settle-ms.tsnew file mode 100644index 0000000..8b75a7d--- /dev/null+++ b/packages/core/src/capture/voice/narration-settle-ms.ts@@ -0,0 +1 @@+export const NARRATION_SETTLE_MS = 30_000;diff --git a/packages/core/src/capture/voice/read-voice-settings.ts b/packages/core/src/capture/voice/read-voice-settings.tsnew file mode 100644index 0000000..bff1c13--- /dev/null+++ b/packages/core/src/capture/voice/read-voice-settings.ts@@ -0,0 +1,18 @@+import { localStorage } from '@/core/env';+import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from './api-key';++export interface VoiceSettings {+ enabled: boolean;+ hasApiKey: boolean;+ microphoneId?: string;+}++export async function readVoiceSettings(): Promise<VoiceSettings> {+ const stored = await localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled', 'voiceMicrophoneId']);+ const microphoneId = typeof stored.voiceMicrophoneId === 'string' ? stored.voiceMicrophoneId.trim() : '';+ return {+ enabled: stored.voiceEnabled === true,+ hasApiKey: hasVoiceApiKey(stored),+ microphoneId: microphoneId || undefined,+ };+}diff --git a/src/entrypoints/background/__tests__/narration-settle.test.ts b/src/entrypoints/background/__tests__/narration-settle.test.tsindex 4ce2eb8..12092d3 100644--- a/src/entrypoints/background/__tests__/narration-settle.test.ts+++ b/src/entrypoints/background/__tests__/narration-settle.test.ts@@ -26,11 +26,15 @@ vi.mock('@/lib/offscreen/close-voice-host-if-idle', () => ({closeVoiceHostIfIdle: (...args: unknown[]) => closeVoiceHostIfIdle(...args),}));vi.mock('@/lib/offscreen/close-voice-host', () => ({ closeVoiceHost: vi.fn() }));+vi.mock('@/lib/offscreen/abort-voice-capture', () => ({+ abortVoiceCapture: vi.fn().mockResolvedValue({ aborted: true }),+}));vi.mock('@/lib/offscreen/flush-voice-capture', () => ({ flushVoiceCapture: vi.fn() }));vi.mock('@/lib/offscreen/open-mic-permission-page', () => ({ openMicPermissionPage: vi.fn() }));vi.mock('@/lib/offscreen/register-voice-panel-relay', () => ({ registerVoicePanelRelay: vi.fn() }));-vi.mock('@/lib/browser-api/local-storage', () => ({+vi.mock('@/core/env', async (importOriginal) => ({+ ...(await importOriginal<typeof import('@/core/env')>()),localStorage: { get: (...args: unknown[]) => storageGet(...args), set: vi.fn() },}));vi.mock('@/lib/browser-api/on-message', () => ({ onMessage: vi.fn() }));@@ -51,14 +55,19 @@ vi.mock('@/core/guides/service', () => ({getStepsForGuide: (...args: unknown[]) => getStepsForGuide(...args),}));-vi.mock('../deferred-descriptions', () => ({ discardDeferred: vi.fn() }));-vi.mock('../describe-unnarrated', () => ({ describeStepNow: vi.fn(), describeUnnarratedSteps: vi.fn() }));+vi.mock('@/core/capture/voice/deferred-descriptions', () => ({ discardDeferred: vi.fn() }));+vi.mock('@/core/capture/voice/describe-unnarrated', () => ({+ describeStepNow: vi.fn(),+ describeUnnarratedSteps: vi.fn(),+}));+import { describeUnnarratedSteps } from '@/core/capture/voice/describe-unnarrated';import {applyNarration,getVoiceUpdate,startVoiceNarration,stopVoiceNarration,+ turnOffNarration,whenNarrationSettled,} from '../voice';@@ -188,3 +197,17 @@ describe('the steps a stop hands to the transcriber', () => {);});});++describe('turning the mic off mid-recording', () => {+ it('releases the steps held for narration and leaves nothing for Stop to report', async () => {+ await beginRecording();+ await turnOffNarration('g1');++ expect(getVoiceUpdate().phase).toBe('idle');+ expect(describeUnnarratedSteps).toHaveBeenCalledWith('g1', []);++ await stopVoiceNarration('g1');+ expect(stopVoiceCapture).not.toHaveBeenCalled();+ expect(getVoiceUpdate().phase).toBe('idle');+ });+});diff --git a/src/entrypoints/background/__tests__/narration-transcript.test.ts b/src/entrypoints/background/__tests__/narration-transcript.test.tsindex c963d1a..40f90e3 100644--- a/src/entrypoints/background/__tests__/narration-transcript.test.ts+++ b/src/entrypoints/background/__tests__/narration-transcript.test.ts@@ -11,8 +11,11 @@ vi.mock('@/core/guides/service', () => ({getStepsForGuide: vi.fn(),}));-vi.mock('../deferred-descriptions', () => ({ discardDeferred: vi.fn() }));-vi.mock('../describe-unnarrated', () => ({ describeStepNow: vi.fn(), describeUnnarratedSteps: vi.fn() }));+vi.mock('@/core/capture/voice/deferred-descriptions', () => ({ discardDeferred: vi.fn() }));+vi.mock('@/core/capture/voice/describe-unnarrated', () => ({+ describeStepNow: vi.fn(),+ describeUnnarratedSteps: vi.fn(),+}));import { applyNarration, takeNarrated } from '../voice';diff --git a/src/entrypoints/background/__tests__/paused-steps.test.ts b/src/entrypoints/background/__tests__/paused-steps.test.tsindex 8391959..1582180 100644--- a/src/entrypoints/background/__tests__/paused-steps.test.ts+++ b/src/entrypoints/background/__tests__/paused-steps.test.ts@@ -40,7 +40,10 @@ vi.mock('@mimik/core/logger', () => ({ logger: { info: vi.fn(), debug: vi.fn(),vi.mock('../voice', () => ({ getVoiceUpdate: () => ({ phase: 'idle' }), flushNarrationForStep: vi.fn() }));vi.mock('../ai-description', () => ({ generateAiDescription: vi.fn() }));vi.mock('@/core/capture/ai/description-queue', () => ({ queueDescription: vi.fn() }));-vi.mock('../deferred-descriptions', () => ({ deferDescription: vi.fn(), shouldQueueAiDescription: () => false }));+vi.mock('@/core/capture/voice/deferred-descriptions', () => ({+ deferDescription: vi.fn(),+ shouldQueueAiDescription: () => false,+}));vi.mock('@/core/capture/ai/keys', () => ({ AI_KEY_SETTINGS: [], resolveAiKey: () => ({ apiKey: undefined }) }));const meta = { rect: { x: 0, y: 0, width: 10, height: 10 }, devicePixelRatio: 1 } as ElementMeta;diff --git a/src/entrypoints/background/guide-meta.ts b/src/entrypoints/background/guide-meta.tsindex 97a8d9c..434423f 100644--- a/src/entrypoints/background/guide-meta.ts+++ b/src/entrypoints/background/guide-meta.ts@@ -1,6 +1,5 @@import { i18n } from '#imports';import { finishGuide } from '@/core/capture/ai/finish-guide';-import { settleDescriptions } from '@/core/capture/ai/settle-descriptions';import { getGuideDomain } from '@/core/guides/service';import { whenNarrationSettled } from './voice';@@ -9,11 +8,6 @@ async function domainTitle(guideId: string): Promise<string> {return domain ? i18n.t('background.guideOnDomain', [domain]) : i18n.t('background.newGuide');}-export async function settlePendingDescriptions(guideId: string) {- await whenNarrationSettled();- await settleDescriptions(guideId);-}-export function generateGuideMetaOnStop(guideId: string) {return finishGuide(guideId, { fallbackTitle: domainTitle, settleNarration: whenNarrationSettled });}diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.tsindex b79f10e..fec233b 100644--- a/src/entrypoints/background/index.ts+++ b/src/entrypoints/background/index.ts@@ -3,6 +3,7 @@ import '@/lib/ui-env';import { logger } from '@mimik/core/logger';import { browser, defineBackground } from '#imports';import { generateDescriptionOnDemand } from '@/core/capture/ai/guide-description';+import { mergeRecording } from '@/core/capture/ai/merge-recording';import { rewriteSelection } from '@/core/capture/ai/rewrite';import { validateApiKey } from '@/core/capture/ai/validate';import { stepRequiresManual } from '@/core/guideme/manual';@@ -10,10 +11,8 @@ import { advanceSession, cancelSession, completeSession, getSession, startSessioimport { actionSteps } from '@/core/guides/blocks';import {createGuide,- createSnapshot,getScreenshotsForSteps,getStepsForGuide,- mergeGuideInto,permanentlyDeleteGuide,softDeleteGuide,} from '@/core/guides/service';@@ -30,7 +29,7 @@ import { setupPortListener } from '@/lib/port/setup-port-listener';import { TabMessage } from '@/lib/tab-messages';import { recordUpdate } from '@/lib/update-notice/record-update';import { getActor, getStateUpdate, initActor, initActorFallback, waitUntilReady } from './actor';-import { generateGuideMetaOnStop, settlePendingDescriptions } from './guide-meta';+import { generateGuideMetaOnStop } from './guide-meta';import { registerNavigationListeners } from './navigation';import { pauseCapture, resumeFromPause, whenPauseSettled } from './pause';import { handleCaptureStep, handleFinalizeInputStep, handleUpdateInputStep } from './step-pipeline';@@ -49,6 +48,8 @@ import {registerVoiceListeners,startVoiceNarration,stopVoiceNarration,+ turnOffNarration,+ whenNarrationSettled,} from './voice';async function resolveManual(step: Step): Promise<boolean> {@@ -163,16 +164,17 @@ export default defineBackground(() => {await broadcastClearBlur();actor.send({ type: 'STOP_RECORDING' });- if (guideId) void stopVoiceNarration(guideId);+ const narrationStopped = guideId ? stopVoiceNarration(guideId) : Promise.resolve();if (guideId && insertTargetGuideId !== null && insertAtIndex !== null) {- await settlePendingDescriptions(guideId);- await createSnapshot(insertTargetGuideId);- await mergeGuideInto(guideId, insertTargetGuideId, insertAtIndex);+ await narrationStopped;+ await mergeRecording(guideId, { insertTargetGuideId, insertAtIndex }, whenNarrationSettled);return { success: true, guideId: insertTargetGuideId, inserted: true };}- if (guideId) generateGuideMetaOnStop(guideId).catch(() => {});+ if (guideId) {+ narrationStopped.then(() => generateGuideMetaOnStop(guideId)).catch(() => {});+ }return { success: true, guideId: guideId ?? undefined, inserted: false };});@@ -194,6 +196,12 @@ export default defineBackground(() => {onMessage('startNarration', async () => ({ started: await startNarrationIfPossible() }));+ onMessage('stopNarration', async () => {+ await waitUntilReady();+ await turnOffNarration(getActor().getSnapshot().context.currentGuideId);+ return { stopped: true };+ });+onMessage('enterBlurMode', async () => {await waitUntilReady();const activeTab = await getActiveTab();diff --git a/src/entrypoints/background/step-pipeline.ts b/src/entrypoints/background/step-pipeline.tsindex 8bb8b65..5959fbe 100644--- a/src/entrypoints/background/step-pipeline.ts+++ b/src/entrypoints/background/step-pipeline.ts@@ -3,6 +3,7 @@ import { queueDescription } from '@/core/capture/ai/description-queue';import type { DOMContext } from '@/core/capture/dom/context';import { isLive } from '@/core/capture/is-live';import { CaptureState } from '@/core/capture/machine';+import { deferDescription } from '@/core/capture/voice/deferred-descriptions';import { type StepNarration, writeStep } from '@/core/capture/write-step';import {getStep,@@ -17,7 +18,6 @@ import { captureVisibleTab } from '@/lib/browser-api/capture-visible-tab';import { localStorage } from '@/lib/browser-api/local-storage';import type { CaptureStepData, CaptureStepResponse } from '@/lib/messaging';import { getActor } from './actor';-import { deferDescription } from './deferred-descriptions';import { describeDomStep } from './describe-dom-step';import { flushNarrationForStep, getVoiceUpdate } from './voice';diff --git a/src/entrypoints/background/voice.ts b/src/entrypoints/background/voice.tsindex 5131843..d466b9e 100644--- a/src/entrypoints/background/voice.ts+++ b/src/entrypoints/background/voice.ts@@ -1,12 +1,13 @@import { logger } from '@mimik/core/logger';import { CaptureState } from '@/core/capture/machine';-import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@/core/capture/voice/api-key';+import { applyNarrationResult } from '@/core/capture/voice/apply-narration-result';+import { describeStepNow, describeUnnarratedSteps } from '@/core/capture/voice/describe-unnarrated';import { narrateRecording, type VoiceRecording } from '@/core/capture/voice/narrate-recording';-import { narrationUpdates } from '@/core/capture/voice/narration-updates';+import { NARRATION_SETTLE_MS } from '@/core/capture/voice/narration-settle-ms';import { readTranscriptionSettings } from '@/core/capture/voice/read-transcription-settings';+import { readVoiceSettings } from '@/core/capture/voice/read-voice-settings';import type { VoicePhase } from '@/core/capture/voice/voice-update';-import { applyNarrationToSteps, findExistingStepIds, getStepsForGuide, saveTranscript } from '@/core/guides/service';-import { localStorage } from '@/lib/browser-api/local-storage';+import { getStepsForGuide } from '@/core/guides/service';import { onMessage as onRuntimeMessage } from '@/lib/browser-api/on-message';import { abortVoiceCapture } from '@/lib/offscreen/abort-voice-capture';import { closeVoiceHost } from '@/lib/offscreen/close-voice-host';@@ -37,8 +38,6 @@ import {type VoiceStepMark,} from '@/lib/voice/voice-message';import { voiceStopAction } from '@/lib/voice/voice-stop-action';-import { discardDeferred } from './deferred-descriptions';-import { describeStepNow, describeUnnarratedSteps } from './describe-unnarrated';const START_TIMEOUT_MS = 8000;@@ -83,29 +82,17 @@ function withTimeout(work: Promise<void>, ms: number): Promise<void> {});}-async function readVoiceSettings(): Promise<{ enabled: boolean; hasApiKey: boolean; microphoneId?: string }> {- const stored = await localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled', 'voiceMicrophoneId']);- const microphoneId = typeof stored.voiceMicrophoneId === 'string' ? stored.voiceMicrophoneId.trim() : '';- return {- enabled: stored.voiceEnabled === true,- hasApiKey: hasVoiceApiKey(stored),- microphoneId: microphoneId || undefined,- };-}-let orphanAudio: VoiceRecording | null = null;let transcribingGuideId: string | null = null;let settleNarration: (() => void) | null = null;let narrationSettled: Promise<void> | null = null;const outstandingTranscriptions = new Map<string, number>();-const NARRATION_SETTLE_TIMEOUT_MS = 30000;-export function whenNarrationSettled(): Promise<void> {const pending = narrationSettled;if (!pending) return Promise.resolve();return new Promise<void>((resolve) => {- const timer = setTimeout(resolve, NARRATION_SETTLE_TIMEOUT_MS);+ const timer = setTimeout(resolve, NARRATION_SETTLE_MS);void pending.then(() => {clearTimeout(timer);resolve();@@ -271,8 +258,14 @@ export async function abortVoiceNarration(): Promise<void> {await closeVoiceHostIfIdle();}+export async function turnOffNarration(guideId: string | null): Promise<void> {+ await abortVoiceNarration();+ if (guideId) describeUnnarratedSteps(guideId, takeNarrated(guideId));+}+export async function stopVoiceNarration(guideId: string): Promise<void> {if (!supportsVoice()) return;+ if (phase.phase === 'idle' && orphanAudio === null) return;try {const action = voiceStopAction({hostAlive: await hasVoiceHost(),@@ -319,19 +312,14 @@ export async function applyNarration(final: boolean,): Promise<void> {try {- await saveTranscript(guideId, result.transcript).catch((error: unknown) =>- logger.warn('voice: the transcript could not be stored', error),+ const updates = await applyNarrationResult(guideId, result);+ recordNarrated(+ guideId,+ updates.map((update) => update.stepId),);- const narrated = result.descriptions.map((entry) => entry.stepId);- const surviving = await findExistingStepIds(narrated);- const updates = narrationUpdates(result, surviving);- await applyNarrationToSteps(updates, result.transcript.epochMs);- const narratedIds = updates.map((update) => update.stepId);- discardDeferred(guideId, narratedIds);- recordNarrated(guideId, narratedIds);logger.info('voice: narration applied', {narrated: updates.length,- of: narrated.length,+ of: result.descriptions.length,final,stats: result.stats,});diff --git a/src/lib/messaging.ts b/src/lib/messaging.tsindex 695fa18..cbe14ad 100644--- a/src/lib/messaging.ts+++ b/src/lib/messaging.ts@@ -1,6 +1,6 @@import { defineExtensionMessaging } from '@webext-core/messaging';import type { GenerateGuideDescriptionResponse } from '@/core/capture/ai/guide-description';-import type { RewriteError, RewriteSelectionResponse } from '@/core/capture/ai/rewrite';+import type { RewriteSelectionResponse } from '@/core/capture/ai/rewrite';import type { CaptureStateValue, PauseReason } from '@/core/capture/machine';import type {CaptureStepData,@@ -19,105 +19,109 @@ export interface GetStateResponse {pauseReason: PauseReason | null;}-export interface StartRecordingData {+interface StartRecordingData {url: string;insertTargetGuideId?: string;insertAtIndex?: number;}-export interface StartRecordingResponse {+interface StartRecordingResponse {guideId: string;}-export interface StopRecordingResponse {+interface StopRecordingResponse {success: boolean;guideId?: string;inserted?: boolean;}-export interface DiscardRecordingResponse {+interface DiscardRecordingResponse {discarded: boolean;}-export interface StartGuideMeData {+interface StartGuideMeData {guideId: string;}-export interface StartGuideMeResponse {+interface StartGuideMeResponse {started: boolean;error?: string;}-export interface GuideMeStepCompletedData {+interface GuideMeStepCompletedData {stepIndex: number;}-export interface GuideMeStepCompletedResponse {+interface GuideMeStepCompletedResponse {advanced: boolean;completed?: boolean;}-export interface GuideMe_CancelResponse {+interface GuideMe_CancelResponse {cancelled: boolean;}-export interface GuideMe_GoToData {+interface GuideMe_GoToData {stepIndex: number;}-export interface GuideMe_GoToResponse {+interface GuideMe_GoToResponse {moved: boolean;}-export interface GenerateGuideDescriptionData {+interface GenerateGuideDescriptionData {guideId: string;}-export interface RewriteSelectionData {+interface RewriteSelectionData {text: string;instruction: string;}-export interface ValidateApiKeyData {+interface ValidateApiKeyData {provider: string;apiKey: string;baseUrl?: string;model?: string;}-export interface ValidateApiKeyResponse {+interface ValidateApiKeyResponse {valid: boolean;reason?: 'rejected' | 'network' | 'model-required' | 'model-invalid';models?: string[];warning?: 'cannot-spend';}-export interface ListVoicesData {+interface ListVoicesData {provider: VoiceoverProviderKey;apiKey: string;}-export interface ListVoicesResponse {+interface ListVoicesResponse {voices: { id: string; name: string }[];}-export interface EnterBlurModeResponse {+interface EnterBlurModeResponse {entered: boolean;}-export interface StartNarrationResponse {+interface StartNarrationResponse {started: boolean;}-export interface ExitBlurModeResponse {+interface StopNarrationResponse {+ stopped: boolean;+}++interface ExitBlurModeResponse {exited: boolean;}-export interface PauseCaptureResponse {+interface PauseCaptureResponse {paused: boolean;}-export interface ResumeCaptureResponse {+interface ResumeCaptureResponse {resumed: boolean;}@@ -138,21 +142,13 @@ interface MimikProtocol {pauseCapture(): PauseCaptureResponse;resumeCapture(): ResumeCaptureResponse;startNarration(): StartNarrationResponse;+ stopNarration(): StopNarrationResponse;generateGuideDescription(data: GenerateGuideDescriptionData): GenerateGuideDescriptionResponse;validateApiKey(data: ValidateApiKeyData): ValidateApiKeyResponse;listVoices(data: ListVoicesData): ListVoicesResponse;rewriteSelection(data: RewriteSelectionData): RewriteSelectionResponse;}-export type {- CaptureStepData,- CaptureStepResponse,- FinalizeInputStepData,- FinalizeInputStepResponse,- RewriteError,- RewriteSelectionResponse,- UpdateInputStepData,- UpdateInputStepResponse,-};+export type { CaptureStepData, CaptureStepResponse };export const { sendMessage, onMessage } = defineExtensionMessaging<MimikProtocol>();diff --git a/src/ui/sidepanel/MicToggle.tsx b/src/ui/sidepanel/MicToggle.tsxindex c920a9e..13821ca 100644--- a/src/ui/sidepanel/MicToggle.tsx+++ b/src/ui/sidepanel/MicToggle.tsx@@ -7,7 +7,6 @@ import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@/core/capture/voice/api-keyimport { getActiveTab } from '@/lib/browser-api/get-active-tab';import { localStorage } from '@/lib/browser-api/local-storage';import { sendMessage } from '@/lib/messaging';-import { abortVoiceCapture } from '@/lib/offscreen/abort-voice-capture';import { openMicPermissionPage } from '@/lib/offscreen/open-mic-permission-page';import { microphoneGranted } from './lib/microphone-granted';@@ -50,7 +49,11 @@ export function MicToggle({ enabled, live, paused = false, onChange }: MicToggleawait localStorage.set({ voiceEnabled: next });if (!next) {- if (live) await abortVoiceCapture().catch(() => undefined);+ if (live) {+ await sendMessage('stopNarration', undefined).catch((error) => {+ logger.error('voice: the background did not take the request to stop narration', error);+ });+ }return;}