Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix: wait for narration before naming or merging, share narration handling in core and let the background turn the mic off

Luis Ortizf388a309cd7c
diff --git a/apps/desktop/src/renderer/narration.ts b/apps/desktop/src/renderer/narration.ts
index 6347124..b3df548 100644
--- a/apps/desktop/src/renderer/narration.ts
+++ b/apps/desktop/src/renderer/narration.ts
@@ -1,28 +1,21 @@
-import { queueDescription } from '@mimik/core/capture/ai/description-queue';
-import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@mimik/core/capture/voice/api-key';
+import { applyNarrationResult } from '@mimik/core/capture/voice/apply-narration-result';
+import { deferDescription } from '@mimik/core/capture/voice/deferred-descriptions';
+import { describeStepNow, describeUnnarratedSteps } from '@mimik/core/capture/voice/describe-unnarrated';
import { MicRecorder } from '@mimik/core/capture/voice/mic-recorder';
import { narrateRecording, type VoiceRecording } from '@mimik/core/capture/voice/narrate-recording';
-import { narrationUpdates } from '@mimik/core/capture/voice/narration-updates';
+import { NARRATION_SETTLE_MS } from '@mimik/core/capture/voice/narration-settle-ms';
import { partialRecording } from '@mimik/core/capture/voice/partial-recording';
import { readTranscriptionSettings } from '@mimik/core/capture/voice/read-transcription-settings';
+import { readVoiceSettings } from '@mimik/core/capture/voice/read-voice-settings';
import { startFailureReason } from '@mimik/core/capture/voice/start-failure-reason';
import type { StepMark } from '@mimik/core/capture/voice/step-windows';
import { usableRecording } from '@mimik/core/capture/voice/usable-recording';
import type { VoiceUpdate } from '@mimik/core/capture/voice/voice-update';
+import type { Describe } from '@mimik/core/capture/write-step';
import { assetUrl, localStorage } from '@mimik/core/env';
-import {
- applyNarrationToSteps,
- clearStepAiPending,
- findExistingStepIds,
- getStepsForGuide,
- saveTranscript,
-} from '@mimik/core/guides/service';
+import { clearStepAiPending, getStepsForGuide } from '@mimik/core/guides/service';
import { logger } from '@mimik/core/logger';
-const SETTLE_MS = 30_000;
-
-type Describe = () => Promise<void>;
-
export class DesktopNarration {
private live: Promise<MicRecorder | null> | null = null;
private mic: MicRecorder | null = null;
@@ -33,7 +26,6 @@ export class DesktopNarration {
private current: VoiceUpdate = { phase: 'idle' };
private readonly listeners = new Set<(update: VoiceUpdate) => void>();
private readonly pending = new Set<Promise<void>>();
- private readonly deferredDescriptions = new Map<string, { guideId: string; describe: Describe | null }>();
get update(): VoiceUpdate {
return this.current;
@@ -51,19 +43,19 @@ export class DesktopNarration {
}
flushForStep(guideId: string, mark: StepMark, describe: Describe | null): void {
- this.deferredDescriptions.set(mark.stepId, { guideId, describe });
+ deferDescription(guideId, mark.stepId, describe ?? (() => this.undescribed(mark.stepId)));
const mic = this.mic;
this.track(async () => {
const full = mic && usableRecording(mic.snapshot());
const closesAt = full ? (mark.timestamp - full.audioEpochMs) / 1000 : 0;
const slice = full && closesAt > this.flushedUpTo ? partialRecording(full, this.flushedUpTo, closesAt) : null;
if (!slice) {
- this.describeStepNow(mark.stepId);
+ describeStepNow(guideId, mark.stepId);
return;
}
this.flushedUpTo = closesAt;
const narrated = await this.transcribe(guideId, slice, [mark]);
- if (!narrated.includes(mark.stepId)) this.describeStepNow(mark.stepId);
+ if (!narrated.includes(mark.stepId)) describeStepNow(guideId, mark.stepId);
});
}
@@ -92,7 +84,7 @@ export class DesktopNarration {
steps.map((step) => ({ stepId: step.id, timestamp: step.timestamp })),
);
}
- this.describeDeferred(guideId);
+ describeUnnarratedSteps(guideId, []);
this.settled();
});
}
@@ -108,13 +100,13 @@ export class DesktopNarration {
mic?.release();
window.mimik.capture.narration(null);
});
- if (guideId) this.describeDeferred(guideId);
+ if (guideId) describeUnnarratedSteps(guideId, []);
if (this.current.phase === 'recording') this.report({ phase: 'idle' });
}
async settle(): Promise<void> {
this.stop();
- await Promise.race([Promise.all(this.pending), new Promise((resolve) => setTimeout(resolve, SETTLE_MS))]);
+ await Promise.race([Promise.all(this.pending), new Promise((resolve) => setTimeout(resolve, NARRATION_SETTLE_MS))]);
}
private report(update: VoiceUpdate): void {
@@ -133,10 +125,9 @@ export class DesktopNarration {
if (this.current.phase === 'transcribing') this.report({ phase: 'idle', narrated: this.narrated });
}
- private describeDeferred(guideId: string): void {
- for (const [stepId, entry] of [...this.deferredDescriptions]) {
- if (entry.guideId === guideId) this.describeStepNow(stepId);
- }
+ private async undescribed(stepId: string): Promise<void> {
+ await clearStepAiPending(stepId);
+ window.mimik.capture.described(stepId, null, null);
}
private track(work: () => Promise<void>): void {
@@ -146,21 +137,10 @@ export class DesktopNarration {
this.pending.add(running);
}
- private describeStepNow(stepId: string): void {
- const entry = this.deferredDescriptions.get(stepId);
- if (!entry) return;
- this.deferredDescriptions.delete(stepId);
- if (entry.describe) {
- queueDescription(entry.guideId, entry.describe);
- return;
- }
- void clearStepAiPending(stepId).then(() => window.mimik.capture.described(stepId, null, null));
- }
-
private async open(): Promise<MicRecorder | null> {
- const stored = await localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled', 'voiceMicrophoneId']);
- if (stored.voiceEnabled !== true) return null;
- if (!hasVoiceApiKey(stored)) {
+ const voice = await readVoiceSettings();
+ if (!voice.enabled) return null;
+ if (!voice.hasApiKey) {
this.report({ phase: 'error', reason: 'missing-api-key' });
return null;
}
@@ -173,7 +153,7 @@ export class DesktopNarration {
},
});
try {
- await mic.start(stored.voiceMicrophoneId || undefined);
+ await mic.start(voice.microphoneId);
if (!this.live) {
mic.release();
return null;
@@ -195,14 +175,9 @@ export class DesktopNarration {
private async transcribe(guideId: string, audio: VoiceRecording, marks: StepMark[]): Promise<string[]> {
const settings = await readTranscriptionSettings();
if (!settings.apiKey) return [];
- const result = await narrateRecording(audio, marks, settings);
- await saveTranscript(guideId, result.transcript);
- const surviving = await findExistingStepIds(result.descriptions.map((entry) => entry.stepId));
- const updates = narrationUpdates(result, surviving);
- await applyNarrationToSteps(updates, result.transcript.epochMs);
+ const updates = await applyNarrationResult(guideId, await narrateRecording(audio, marks, settings));
this.narrated += updates.length;
for (const { stepId, description } of updates) {
- this.deferredDescriptions.delete(stepId);
window.mimik.capture.described(stepId, description, null, 'narration');
}
return updates.map((update) => update.stepId);
diff --git a/packages/core/src/capture/ai/__tests__/merge-recording.test.ts b/packages/core/src/capture/ai/__tests__/merge-recording.test.ts
new file mode 100644
index 0000000..c11a237
--- /dev/null
+++ b/packages/core/src/capture/ai/__tests__/merge-recording.test.ts
@@ -0,0 +1,32 @@
+import { describe, expect, it, vi } from 'vitest';
+
+const order: string[] = [];
+
+vi.mock('@/core/guides/service', () => ({
+ createSnapshot: vi.fn(async () => order.push('snapshot')),
+ mergeGuideInto: vi.fn(async () => order.push('merge')),
+}));
+vi.mock('../settle-descriptions', () => ({ settleDescriptions: vi.fn(async () => order.push('descriptions')) }));
+
+import { mergeGuideInto } from '@/core/guides/service';
+import { mergeRecording } from '../merge-recording';
+
+describe('mergeRecording', () => {
+ it('waits for the narration and the descriptions before it snapshots and merges', async () => {
+ let finishNarration = () => {};
+ const narration = new Promise<void>((resolve) => {
+ finishNarration = resolve;
+ });
+ const merging = mergeRecording('staging', { insertTargetGuideId: 'target', insertAtIndex: 2 }, () =>
+ narration.then(() => void order.push('narration')),
+ );
+
+ await Promise.resolve();
+ expect(order).toEqual([]);
+
+ finishNarration();
+ await merging;
+ expect(order).toEqual(['narration', 'descriptions', 'snapshot', 'merge']);
+ expect(mergeGuideInto).toHaveBeenCalledWith('staging', 'target', 2);
+ });
+});
diff --git a/packages/core/src/capture/ai/merge-recording.ts b/packages/core/src/capture/ai/merge-recording.ts
new file mode 100644
index 0000000..a17fc8c
--- /dev/null
+++ b/packages/core/src/capture/ai/merge-recording.ts
@@ -0,0 +1,14 @@
+import { createSnapshot, mergeGuideInto } from '@/core/guides/service';
+import type { CaptureInsert } from '../capture-insert';
+import { settleDescriptions } from './settle-descriptions';
+
+export async function mergeRecording(
+ guideId: string,
+ { insertTargetGuideId, insertAtIndex }: Pick<CaptureInsert, 'insertTargetGuideId' | 'insertAtIndex'>,
+ settleNarration: () => Promise<void>,
+): Promise<void> {
+ await settleNarration();
+ await settleDescriptions(guideId);
+ await createSnapshot(insertTargetGuideId);
+ await mergeGuideInto(guideId, insertTargetGuideId, insertAtIndex);
+}
diff --git a/packages/core/src/capture/voice/__tests__/apply-narration-result.test.ts b/packages/core/src/capture/voice/__tests__/apply-narration-result.test.ts
new file mode 100644
index 0000000..3209e87
--- /dev/null
+++ b/packages/core/src/capture/voice/__tests__/apply-narration-result.test.ts
@@ -0,0 +1,36 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import type { NarrationResult } from '../types';
+
+vi.mock('@/core/guides/service', () => ({
+ saveTranscript: vi.fn(async () => {
+ throw new Error('quota');
+ }),
+ findExistingStepIds: vi.fn(async (ids: string[]) => ids.filter((id) => id !== 'deleted')),
+ applyNarrationToSteps: vi.fn(async () => undefined),
+}));
+
+import { applyNarrationResult } from '../apply-narration-result';
+import { clearDeferredDescriptions, deferDescription, takeDeferredDescriptions } from '../deferred-descriptions';
+
+const result = {
+ descriptions: [
+ { stepId: 's1', text: 'Open billing' },
+ { stepId: 'deleted', text: 'Gone' },
+ ],
+ transcript: { epochMs: 0, lines: [] },
+ stats: {},
+} as unknown as NarrationResult;
+
+beforeEach(() => clearDeferredDescriptions('g'));
+
+describe('applyNarrationResult', () => {
+ it('narrates the surviving steps and drops their held AI descriptions, even when the transcript fails to save', async () => {
+ deferDescription('g', 's1', async () => undefined);
+ deferDescription('g', 's2', async () => undefined);
+
+ const updates = await applyNarrationResult('g', result);
+
+ expect(updates.map((update) => update.stepId)).toEqual(['s1']);
+ expect(takeDeferredDescriptions('g', []).map((held) => held.stepId)).toEqual(['s2']);
+ });
+});
diff --git a/src/entrypoints/background/__tests__/deferred-descriptions.test.ts b/packages/core/src/capture/voice/__tests__/deferred-descriptions.test.ts
similarity index 100%
rename from src/entrypoints/background/__tests__/deferred-descriptions.test.ts
rename to packages/core/src/capture/voice/__tests__/deferred-descriptions.test.ts
diff --git a/packages/core/src/capture/voice/apply-narration-result.ts b/packages/core/src/capture/voice/apply-narration-result.ts
new file mode 100644
index 0000000..89e9aab
--- /dev/null
+++ b/packages/core/src/capture/voice/apply-narration-result.ts
@@ -0,0 +1,19 @@
+import { applyNarrationToSteps, findExistingStepIds, saveTranscript } from '@/core/guides/service';
+import { logger } from '@/core/logger';
+import { discardDeferred } from './deferred-descriptions';
+import { type NarrationUpdate, narrationUpdates } from './narration-updates';
+import type { NarrationResult } from './types';
+
+export async function applyNarrationResult(guideId: string, result: NarrationResult): Promise<NarrationUpdate[]> {
+ await saveTranscript(guideId, result.transcript).catch((error: unknown) =>
+ logger.warn('voice: the transcript could not be stored', error),
+ );
+ const surviving = await findExistingStepIds(result.descriptions.map((entry) => entry.stepId));
+ const updates = narrationUpdates(result, surviving);
+ await applyNarrationToSteps(updates, result.transcript.epochMs);
+ discardDeferred(
+ guideId,
+ updates.map((update) => update.stepId),
+ );
+ return updates;
+}
diff --git a/src/entrypoints/background/deferred-descriptions.ts b/packages/core/src/capture/voice/deferred-descriptions.ts
similarity index 95%
rename from src/entrypoints/background/deferred-descriptions.ts
rename to packages/core/src/capture/voice/deferred-descriptions.ts
index 776465f..f76b152 100644
--- a/src/entrypoints/background/deferred-descriptions.ts
+++ b/packages/core/src/capture/voice/deferred-descriptions.ts
@@ -1,4 +1,4 @@
-import type { Describe } from '@/core/capture/write-step';
+import type { Describe } from '../write-step';
export interface DeferredDescription {
stepId: string;
diff --git a/src/entrypoints/background/describe-unnarrated.ts b/packages/core/src/capture/voice/describe-unnarrated.ts
similarity index 85%
rename from src/entrypoints/background/describe-unnarrated.ts
rename to packages/core/src/capture/voice/describe-unnarrated.ts
index a17bb4e..9144aa1 100644
--- a/src/entrypoints/background/describe-unnarrated.ts
+++ b/packages/core/src/capture/voice/describe-unnarrated.ts
@@ -1,6 +1,6 @@
-import { logger } from '@mimik/core/logger';
-import { queueDescription } from '@/core/capture/ai/description-queue';
import { clearStepAiPending } from '@/core/guides/service';
+import { logger } from '@/core/logger';
+import { queueDescription } from '../ai/description-queue';
import { takeDeferredDescription, takeDeferredDescriptions } from './deferred-descriptions';
export function describeStepNow(guideId: string, stepId: string): void {
diff --git a/packages/core/src/capture/voice/narration-settle-ms.ts b/packages/core/src/capture/voice/narration-settle-ms.ts
new file mode 100644
index 0000000..8b75a7d
--- /dev/null
+++ b/packages/core/src/capture/voice/narration-settle-ms.ts
@@ -0,0 +1 @@
+export const NARRATION_SETTLE_MS = 30_000;
diff --git a/packages/core/src/capture/voice/read-voice-settings.ts b/packages/core/src/capture/voice/read-voice-settings.ts
new file mode 100644
index 0000000..bff1c13
--- /dev/null
+++ b/packages/core/src/capture/voice/read-voice-settings.ts
@@ -0,0 +1,18 @@
+import { localStorage } from '@/core/env';
+import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from './api-key';
+
+export interface VoiceSettings {
+ enabled: boolean;
+ hasApiKey: boolean;
+ microphoneId?: string;
+}
+
+export async function readVoiceSettings(): Promise<VoiceSettings> {
+ const stored = await localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled', 'voiceMicrophoneId']);
+ const microphoneId = typeof stored.voiceMicrophoneId === 'string' ? stored.voiceMicrophoneId.trim() : '';
+ return {
+ enabled: stored.voiceEnabled === true,
+ hasApiKey: hasVoiceApiKey(stored),
+ microphoneId: microphoneId || undefined,
+ };
+}
diff --git a/src/entrypoints/background/__tests__/narration-settle.test.ts b/src/entrypoints/background/__tests__/narration-settle.test.ts
index 4ce2eb8..12092d3 100644
--- a/src/entrypoints/background/__tests__/narration-settle.test.ts
+++ b/src/entrypoints/background/__tests__/narration-settle.test.ts
@@ -26,11 +26,15 @@ vi.mock('@/lib/offscreen/close-voice-host-if-idle', () => ({
closeVoiceHostIfIdle: (...args: unknown[]) => closeVoiceHostIfIdle(...args),
}));
vi.mock('@/lib/offscreen/close-voice-host', () => ({ closeVoiceHost: vi.fn() }));
+vi.mock('@/lib/offscreen/abort-voice-capture', () => ({
+ abortVoiceCapture: vi.fn().mockResolvedValue({ aborted: true }),
+}));
vi.mock('@/lib/offscreen/flush-voice-capture', () => ({ flushVoiceCapture: vi.fn() }));
vi.mock('@/lib/offscreen/open-mic-permission-page', () => ({ openMicPermissionPage: vi.fn() }));
vi.mock('@/lib/offscreen/register-voice-panel-relay', () => ({ registerVoicePanelRelay: vi.fn() }));
-vi.mock('@/lib/browser-api/local-storage', () => ({
+vi.mock('@/core/env', async (importOriginal) => ({
+ ...(await importOriginal<typeof import('@/core/env')>()),
localStorage: { get: (...args: unknown[]) => storageGet(...args), set: vi.fn() },
}));
vi.mock('@/lib/browser-api/on-message', () => ({ onMessage: vi.fn() }));
@@ -51,14 +55,19 @@ vi.mock('@/core/guides/service', () => ({
getStepsForGuide: (...args: unknown[]) => getStepsForGuide(...args),
}));
-vi.mock('../deferred-descriptions', () => ({ discardDeferred: vi.fn() }));
-vi.mock('../describe-unnarrated', () => ({ describeStepNow: vi.fn(), describeUnnarratedSteps: vi.fn() }));
+vi.mock('@/core/capture/voice/deferred-descriptions', () => ({ discardDeferred: vi.fn() }));
+vi.mock('@/core/capture/voice/describe-unnarrated', () => ({
+ describeStepNow: vi.fn(),
+ describeUnnarratedSteps: vi.fn(),
+}));
+import { describeUnnarratedSteps } from '@/core/capture/voice/describe-unnarrated';
import {
applyNarration,
getVoiceUpdate,
startVoiceNarration,
stopVoiceNarration,
+ turnOffNarration,
whenNarrationSettled,
} from '../voice';
@@ -188,3 +197,17 @@ describe('the steps a stop hands to the transcriber', () => {
);
});
});
+
+describe('turning the mic off mid-recording', () => {
+ it('releases the steps held for narration and leaves nothing for Stop to report', async () => {
+ await beginRecording();
+ await turnOffNarration('g1');
+
+ expect(getVoiceUpdate().phase).toBe('idle');
+ expect(describeUnnarratedSteps).toHaveBeenCalledWith('g1', []);
+
+ await stopVoiceNarration('g1');
+ expect(stopVoiceCapture).not.toHaveBeenCalled();
+ expect(getVoiceUpdate().phase).toBe('idle');
+ });
+});
diff --git a/src/entrypoints/background/__tests__/narration-transcript.test.ts b/src/entrypoints/background/__tests__/narration-transcript.test.ts
index c963d1a..40f90e3 100644
--- a/src/entrypoints/background/__tests__/narration-transcript.test.ts
+++ b/src/entrypoints/background/__tests__/narration-transcript.test.ts
@@ -11,8 +11,11 @@ vi.mock('@/core/guides/service', () => ({
getStepsForGuide: vi.fn(),
}));
-vi.mock('../deferred-descriptions', () => ({ discardDeferred: vi.fn() }));
-vi.mock('../describe-unnarrated', () => ({ describeStepNow: vi.fn(), describeUnnarratedSteps: vi.fn() }));
+vi.mock('@/core/capture/voice/deferred-descriptions', () => ({ discardDeferred: vi.fn() }));
+vi.mock('@/core/capture/voice/describe-unnarrated', () => ({
+ describeStepNow: vi.fn(),
+ describeUnnarratedSteps: vi.fn(),
+}));
import { applyNarration, takeNarrated } from '../voice';
diff --git a/src/entrypoints/background/__tests__/paused-steps.test.ts b/src/entrypoints/background/__tests__/paused-steps.test.ts
index 8391959..1582180 100644
--- a/src/entrypoints/background/__tests__/paused-steps.test.ts
+++ b/src/entrypoints/background/__tests__/paused-steps.test.ts
@@ -40,7 +40,10 @@ vi.mock('@mimik/core/logger', () => ({ logger: { info: vi.fn(), debug: vi.fn(),
vi.mock('../voice', () => ({ getVoiceUpdate: () => ({ phase: 'idle' }), flushNarrationForStep: vi.fn() }));
vi.mock('../ai-description', () => ({ generateAiDescription: vi.fn() }));
vi.mock('@/core/capture/ai/description-queue', () => ({ queueDescription: vi.fn() }));
-vi.mock('../deferred-descriptions', () => ({ deferDescription: vi.fn(), shouldQueueAiDescription: () => false }));
+vi.mock('@/core/capture/voice/deferred-descriptions', () => ({
+ deferDescription: vi.fn(),
+ shouldQueueAiDescription: () => false,
+}));
vi.mock('@/core/capture/ai/keys', () => ({ AI_KEY_SETTINGS: [], resolveAiKey: () => ({ apiKey: undefined }) }));
const meta = { rect: { x: 0, y: 0, width: 10, height: 10 }, devicePixelRatio: 1 } as ElementMeta;
diff --git a/src/entrypoints/background/guide-meta.ts b/src/entrypoints/background/guide-meta.ts
index 97a8d9c..434423f 100644
--- a/src/entrypoints/background/guide-meta.ts
+++ b/src/entrypoints/background/guide-meta.ts
@@ -1,6 +1,5 @@
import { i18n } from '#imports';
import { finishGuide } from '@/core/capture/ai/finish-guide';
-import { settleDescriptions } from '@/core/capture/ai/settle-descriptions';
import { getGuideDomain } from '@/core/guides/service';
import { whenNarrationSettled } from './voice';
@@ -9,11 +8,6 @@ async function domainTitle(guideId: string): Promise<string> {
return domain ? i18n.t('background.guideOnDomain', [domain]) : i18n.t('background.newGuide');
}
-export async function settlePendingDescriptions(guideId: string) {
- await whenNarrationSettled();
- await settleDescriptions(guideId);
-}
-
export function generateGuideMetaOnStop(guideId: string) {
return finishGuide(guideId, { fallbackTitle: domainTitle, settleNarration: whenNarrationSettled });
}
diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.ts
index b79f10e..fec233b 100644
--- a/src/entrypoints/background/index.ts
+++ b/src/entrypoints/background/index.ts
@@ -3,6 +3,7 @@ import '@/lib/ui-env';
import { logger } from '@mimik/core/logger';
import { browser, defineBackground } from '#imports';
import { generateDescriptionOnDemand } from '@/core/capture/ai/guide-description';
+import { mergeRecording } from '@/core/capture/ai/merge-recording';
import { rewriteSelection } from '@/core/capture/ai/rewrite';
import { validateApiKey } from '@/core/capture/ai/validate';
import { stepRequiresManual } from '@/core/guideme/manual';
@@ -10,10 +11,8 @@ import { advanceSession, cancelSession, completeSession, getSession, startSessio
import { actionSteps } from '@/core/guides/blocks';
import {
createGuide,
- createSnapshot,
getScreenshotsForSteps,
getStepsForGuide,
- mergeGuideInto,
permanentlyDeleteGuide,
softDeleteGuide,
} from '@/core/guides/service';
@@ -30,7 +29,7 @@ import { setupPortListener } from '@/lib/port/setup-port-listener';
import { TabMessage } from '@/lib/tab-messages';
import { recordUpdate } from '@/lib/update-notice/record-update';
import { getActor, getStateUpdate, initActor, initActorFallback, waitUntilReady } from './actor';
-import { generateGuideMetaOnStop, settlePendingDescriptions } from './guide-meta';
+import { generateGuideMetaOnStop } from './guide-meta';
import { registerNavigationListeners } from './navigation';
import { pauseCapture, resumeFromPause, whenPauseSettled } from './pause';
import { handleCaptureStep, handleFinalizeInputStep, handleUpdateInputStep } from './step-pipeline';
@@ -49,6 +48,8 @@ import {
registerVoiceListeners,
startVoiceNarration,
stopVoiceNarration,
+ turnOffNarration,
+ whenNarrationSettled,
} from './voice';
async function resolveManual(step: Step): Promise<boolean> {
@@ -163,16 +164,17 @@ export default defineBackground(() => {
await broadcastClearBlur();
actor.send({ type: 'STOP_RECORDING' });
- if (guideId) void stopVoiceNarration(guideId);
+ const narrationStopped = guideId ? stopVoiceNarration(guideId) : Promise.resolve();
if (guideId && insertTargetGuideId !== null && insertAtIndex !== null) {
- await settlePendingDescriptions(guideId);
- await createSnapshot(insertTargetGuideId);
- await mergeGuideInto(guideId, insertTargetGuideId, insertAtIndex);
+ await narrationStopped;
+ await mergeRecording(guideId, { insertTargetGuideId, insertAtIndex }, whenNarrationSettled);
return { success: true, guideId: insertTargetGuideId, inserted: true };
}
- if (guideId) generateGuideMetaOnStop(guideId).catch(() => {});
+ if (guideId) {
+ narrationStopped.then(() => generateGuideMetaOnStop(guideId)).catch(() => {});
+ }
return { success: true, guideId: guideId ?? undefined, inserted: false };
});
@@ -194,6 +196,12 @@ export default defineBackground(() => {
onMessage('startNarration', async () => ({ started: await startNarrationIfPossible() }));
+ onMessage('stopNarration', async () => {
+ await waitUntilReady();
+ await turnOffNarration(getActor().getSnapshot().context.currentGuideId);
+ return { stopped: true };
+ });
+
onMessage('enterBlurMode', async () => {
await waitUntilReady();
const activeTab = await getActiveTab();
diff --git a/src/entrypoints/background/step-pipeline.ts b/src/entrypoints/background/step-pipeline.ts
index 8bb8b65..5959fbe 100644
--- a/src/entrypoints/background/step-pipeline.ts
+++ b/src/entrypoints/background/step-pipeline.ts
@@ -3,6 +3,7 @@ import { queueDescription } from '@/core/capture/ai/description-queue';
import type { DOMContext } from '@/core/capture/dom/context';
import { isLive } from '@/core/capture/is-live';
import { CaptureState } from '@/core/capture/machine';
+import { deferDescription } from '@/core/capture/voice/deferred-descriptions';
import { type StepNarration, writeStep } from '@/core/capture/write-step';
import {
getStep,
@@ -17,7 +18,6 @@ import { captureVisibleTab } from '@/lib/browser-api/capture-visible-tab';
import { localStorage } from '@/lib/browser-api/local-storage';
import type { CaptureStepData, CaptureStepResponse } from '@/lib/messaging';
import { getActor } from './actor';
-import { deferDescription } from './deferred-descriptions';
import { describeDomStep } from './describe-dom-step';
import { flushNarrationForStep, getVoiceUpdate } from './voice';
diff --git a/src/entrypoints/background/voice.ts b/src/entrypoints/background/voice.ts
index 5131843..d466b9e 100644
--- a/src/entrypoints/background/voice.ts
+++ b/src/entrypoints/background/voice.ts
@@ -1,12 +1,13 @@
import { logger } from '@mimik/core/logger';
import { CaptureState } from '@/core/capture/machine';
-import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@/core/capture/voice/api-key';
+import { applyNarrationResult } from '@/core/capture/voice/apply-narration-result';
+import { describeStepNow, describeUnnarratedSteps } from '@/core/capture/voice/describe-unnarrated';
import { narrateRecording, type VoiceRecording } from '@/core/capture/voice/narrate-recording';
-import { narrationUpdates } from '@/core/capture/voice/narration-updates';
+import { NARRATION_SETTLE_MS } from '@/core/capture/voice/narration-settle-ms';
import { readTranscriptionSettings } from '@/core/capture/voice/read-transcription-settings';
+import { readVoiceSettings } from '@/core/capture/voice/read-voice-settings';
import type { VoicePhase } from '@/core/capture/voice/voice-update';
-import { applyNarrationToSteps, findExistingStepIds, getStepsForGuide, saveTranscript } from '@/core/guides/service';
-import { localStorage } from '@/lib/browser-api/local-storage';
+import { getStepsForGuide } from '@/core/guides/service';
import { onMessage as onRuntimeMessage } from '@/lib/browser-api/on-message';
import { abortVoiceCapture } from '@/lib/offscreen/abort-voice-capture';
import { closeVoiceHost } from '@/lib/offscreen/close-voice-host';
@@ -37,8 +38,6 @@ import {
type VoiceStepMark,
} from '@/lib/voice/voice-message';
import { voiceStopAction } from '@/lib/voice/voice-stop-action';
-import { discardDeferred } from './deferred-descriptions';
-import { describeStepNow, describeUnnarratedSteps } from './describe-unnarrated';
const START_TIMEOUT_MS = 8000;
@@ -83,29 +82,17 @@ function withTimeout(work: Promise<void>, ms: number): Promise<void> {
});
}
-async function readVoiceSettings(): Promise<{ enabled: boolean; hasApiKey: boolean; microphoneId?: string }> {
- const stored = await localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled', 'voiceMicrophoneId']);
- const microphoneId = typeof stored.voiceMicrophoneId === 'string' ? stored.voiceMicrophoneId.trim() : '';
- return {
- enabled: stored.voiceEnabled === true,
- hasApiKey: hasVoiceApiKey(stored),
- microphoneId: microphoneId || undefined,
- };
-}
-
let orphanAudio: VoiceRecording | null = null;
let transcribingGuideId: string | null = null;
let settleNarration: (() => void) | null = null;
let narrationSettled: Promise<void> | null = null;
const outstandingTranscriptions = new Map<string, number>();
-const NARRATION_SETTLE_TIMEOUT_MS = 30000;
-
export function whenNarrationSettled(): Promise<void> {
const pending = narrationSettled;
if (!pending) return Promise.resolve();
return new Promise<void>((resolve) => {
- const timer = setTimeout(resolve, NARRATION_SETTLE_TIMEOUT_MS);
+ const timer = setTimeout(resolve, NARRATION_SETTLE_MS);
void pending.then(() => {
clearTimeout(timer);
resolve();
@@ -271,8 +258,14 @@ export async function abortVoiceNarration(): Promise<void> {
await closeVoiceHostIfIdle();
}
+export async function turnOffNarration(guideId: string | null): Promise<void> {
+ await abortVoiceNarration();
+ if (guideId) describeUnnarratedSteps(guideId, takeNarrated(guideId));
+}
+
export async function stopVoiceNarration(guideId: string): Promise<void> {
if (!supportsVoice()) return;
+ if (phase.phase === 'idle' && orphanAudio === null) return;
try {
const action = voiceStopAction({
hostAlive: await hasVoiceHost(),
@@ -319,19 +312,14 @@ export async function applyNarration(
final: boolean,
): Promise<void> {
try {
- await saveTranscript(guideId, result.transcript).catch((error: unknown) =>
- logger.warn('voice: the transcript could not be stored', error),
+ const updates = await applyNarrationResult(guideId, result);
+ recordNarrated(
+ guideId,
+ updates.map((update) => update.stepId),
);
- const narrated = result.descriptions.map((entry) => entry.stepId);
- const surviving = await findExistingStepIds(narrated);
- const updates = narrationUpdates(result, surviving);
- await applyNarrationToSteps(updates, result.transcript.epochMs);
- const narratedIds = updates.map((update) => update.stepId);
- discardDeferred(guideId, narratedIds);
- recordNarrated(guideId, narratedIds);
logger.info('voice: narration applied', {
narrated: updates.length,
- of: narrated.length,
+ of: result.descriptions.length,
final,
stats: result.stats,
});
diff --git a/src/lib/messaging.ts b/src/lib/messaging.ts
index 695fa18..cbe14ad 100644
--- a/src/lib/messaging.ts
+++ b/src/lib/messaging.ts
@@ -1,6 +1,6 @@
import { defineExtensionMessaging } from '@webext-core/messaging';
import type { GenerateGuideDescriptionResponse } from '@/core/capture/ai/guide-description';
-import type { RewriteError, RewriteSelectionResponse } from '@/core/capture/ai/rewrite';
+import type { RewriteSelectionResponse } from '@/core/capture/ai/rewrite';
import type { CaptureStateValue, PauseReason } from '@/core/capture/machine';
import type {
CaptureStepData,
@@ -19,105 +19,109 @@ export interface GetStateResponse {
pauseReason: PauseReason | null;
}
-export interface StartRecordingData {
+interface StartRecordingData {
url: string;
insertTargetGuideId?: string;
insertAtIndex?: number;
}
-export interface StartRecordingResponse {
+interface StartRecordingResponse {
guideId: string;
}
-export interface StopRecordingResponse {
+interface StopRecordingResponse {
success: boolean;
guideId?: string;
inserted?: boolean;
}
-export interface DiscardRecordingResponse {
+interface DiscardRecordingResponse {
discarded: boolean;
}
-export interface StartGuideMeData {
+interface StartGuideMeData {
guideId: string;
}
-export interface StartGuideMeResponse {
+interface StartGuideMeResponse {
started: boolean;
error?: string;
}
-export interface GuideMeStepCompletedData {
+interface GuideMeStepCompletedData {
stepIndex: number;
}
-export interface GuideMeStepCompletedResponse {
+interface GuideMeStepCompletedResponse {
advanced: boolean;
completed?: boolean;
}
-export interface GuideMe_CancelResponse {
+interface GuideMe_CancelResponse {
cancelled: boolean;
}
-export interface GuideMe_GoToData {
+interface GuideMe_GoToData {
stepIndex: number;
}
-export interface GuideMe_GoToResponse {
+interface GuideMe_GoToResponse {
moved: boolean;
}
-export interface GenerateGuideDescriptionData {
+interface GenerateGuideDescriptionData {
guideId: string;
}
-export interface RewriteSelectionData {
+interface RewriteSelectionData {
text: string;
instruction: string;
}
-export interface ValidateApiKeyData {
+interface ValidateApiKeyData {
provider: string;
apiKey: string;
baseUrl?: string;
model?: string;
}
-export interface ValidateApiKeyResponse {
+interface ValidateApiKeyResponse {
valid: boolean;
reason?: 'rejected' | 'network' | 'model-required' | 'model-invalid';
models?: string[];
warning?: 'cannot-spend';
}
-export interface ListVoicesData {
+interface ListVoicesData {
provider: VoiceoverProviderKey;
apiKey: string;
}
-export interface ListVoicesResponse {
+interface ListVoicesResponse {
voices: { id: string; name: string }[];
}
-export interface EnterBlurModeResponse {
+interface EnterBlurModeResponse {
entered: boolean;
}
-export interface StartNarrationResponse {
+interface StartNarrationResponse {
started: boolean;
}
-export interface ExitBlurModeResponse {
+interface StopNarrationResponse {
+ stopped: boolean;
+}
+
+interface ExitBlurModeResponse {
exited: boolean;
}
-export interface PauseCaptureResponse {
+interface PauseCaptureResponse {
paused: boolean;
}
-export interface ResumeCaptureResponse {
+interface ResumeCaptureResponse {
resumed: boolean;
}
@@ -138,21 +142,13 @@ interface MimikProtocol {
pauseCapture(): PauseCaptureResponse;
resumeCapture(): ResumeCaptureResponse;
startNarration(): StartNarrationResponse;
+ stopNarration(): StopNarrationResponse;
generateGuideDescription(data: GenerateGuideDescriptionData): GenerateGuideDescriptionResponse;
validateApiKey(data: ValidateApiKeyData): ValidateApiKeyResponse;
listVoices(data: ListVoicesData): ListVoicesResponse;
rewriteSelection(data: RewriteSelectionData): RewriteSelectionResponse;
}
-export type {
- CaptureStepData,
- CaptureStepResponse,
- FinalizeInputStepData,
- FinalizeInputStepResponse,
- RewriteError,
- RewriteSelectionResponse,
- UpdateInputStepData,
- UpdateInputStepResponse,
-};
+export type { CaptureStepData, CaptureStepResponse };
export const { sendMessage, onMessage } = defineExtensionMessaging<MimikProtocol>();
diff --git a/src/ui/sidepanel/MicToggle.tsx b/src/ui/sidepanel/MicToggle.tsx
index c920a9e..13821ca 100644
--- a/src/ui/sidepanel/MicToggle.tsx
+++ b/src/ui/sidepanel/MicToggle.tsx
@@ -7,7 +7,6 @@ import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@/core/capture/voice/api-key
import { getActiveTab } from '@/lib/browser-api/get-active-tab';
import { localStorage } from '@/lib/browser-api/local-storage';
import { sendMessage } from '@/lib/messaging';
-import { abortVoiceCapture } from '@/lib/offscreen/abort-voice-capture';
import { openMicPermissionPage } from '@/lib/offscreen/open-mic-permission-page';
import { microphoneGranted } from './lib/microphone-granted';
@@ -50,7 +49,11 @@ export function MicToggle({ enabled, live, paused = false, onChange }: MicToggle
await localStorage.set({ voiceEnabled: next });
if (!next) {
- if (live) await abortVoiceCapture().catch(() => undefined);
+ if (live) {
+ await sendMessage('stopNarration', undefined).catch((error) => {
+ logger.error('voice: the background did not take the request to stop narration', error);
+ });
+ }
return;
}