Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(voice): start and stop narration with the recording

Luis Ortiz9fcdd8339bed
diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.ts
index c39c53c..846bf9a 100644
--- a/src/entrypoints/background/index.ts
+++ b/src/entrypoints/background/index.ts
@@ -10,6 +10,7 @@ import { getActor, getStateUpdate, initActor, initActorFallback, waitUntilReady
import { registerNavigationListeners } from './navigation';
import { handleCaptureStep, handleFinalizeInputStep, handleUpdateInputStep } from './step-pipeline';
import { broadcastStartCapture, broadcastStopCapture, showNotificationOnTab } from './tab-manager';
+import { getVoiceUpdate, registerVoiceListeners, startVoiceNarration, stopVoiceNarration } from './voice';
async function generateTitleInBackground(guideId: string) {
try {
@@ -84,12 +85,14 @@ export default defineBackground(() => {
initActor().catch(initActorFallback);
cancelSession();
registerNavigationListeners();
+ registerVoiceListeners();
setupPortListener((port) => {
logger.debug('Panel connected via port');
waitUntilReady().then(() => {
try {
port.postMessage(getStateUpdate());
+ port.postMessage(getVoiceUpdate());
} catch {}
});
@@ -123,6 +126,8 @@ export default defineBackground(() => {
const activeTab = await getActiveTab();
if (activeTab?.id) await showNotificationOnTab(activeTab.id);
+ await startVoiceNarration(activeTab?.id);
+
await broadcastStartCapture(guideId);
return { guideId };
});
@@ -134,7 +139,10 @@ export default defineBackground(() => {
await broadcastStopCapture();
actor.send({ type: 'STOP_RECORDING' });
- if (guideId) generateTitleInBackground(guideId);
+ if (guideId) {
+ void stopVoiceNarration(guideId);
+ generateTitleInBackground(guideId);
+ }
return { success: true, guideId: guideId ?? undefined };
});
diff --git a/src/entrypoints/background/voice.ts b/src/entrypoints/background/voice.ts
new file mode 100644
index 0000000..43b1664
--- /dev/null
+++ b/src/entrypoints/background/voice.ts
@@ -0,0 +1,198 @@
+import { narrationUpdates } from '@/core/capture/voice/narration-updates';
+import { applyNarrationToSteps, getStepsForGuide } from '@/core/guides/service';
+import { localStorage, onMessage as onRuntimeMessage } from '@/lib/browser-api';
+import { logger } from '@/lib/logger';
+import {
+ closeOffscreenDocument,
+ ensureOffscreenDocument,
+ getVoiceStatus,
+ hasOffscreenDocument,
+ openMicPermissionPage,
+ queryMicPermission,
+ startVoiceCapture,
+ stopVoiceCapture,
+ supportsOffscreen,
+} from '@/lib/offscreen';
+import { broadcastVoiceToPanel, type PanelVoiceUpdate } from '@/lib/port';
+import {
+ isVoiceMessageFor,
+ VOICE_BACKGROUND_TARGET,
+ type VoiceErrorEvent,
+ type VoiceEvent,
+ VoiceMessage,
+ type VoicePermissionResultEvent,
+ type VoiceResultEvent,
+} from '@/lib/voice-messages';
+
+const START_TIMEOUT_MS = 8000;
+
+let phase: PanelVoiceUpdate = { type: 'VOICE_UPDATE', phase: 'idle' };
+
+export function getVoiceUpdate(): PanelVoiceUpdate {
+ return phase;
+}
+
+function report(update: Omit<PanelVoiceUpdate, 'type'>): void {
+ phase = { type: 'VOICE_UPDATE', ...update };
+ broadcastVoiceToPanel(phase);
+}
+
+function withTimeout(work: Promise<void>, ms: number): Promise<void> {
+ return new Promise<void>((resolve) => {
+ const timer = setTimeout(() => {
+ logger.warn('voice: giving up waiting for the microphone');
+ resolve();
+ }, ms);
+ const settle = () => {
+ clearTimeout(timer);
+ resolve();
+ };
+ work.then(settle, settle);
+ });
+}
+
+async function readVoiceSettings(): Promise<{ enabled: boolean; hasApiKey: boolean; microphoneId?: string }> {
+ const stored = await localStorage.get(['voiceEnabled', 'voiceApiKey', 'voiceMicrophoneId']);
+ const microphoneId = typeof stored.voiceMicrophoneId === 'string' ? stored.voiceMicrophoneId.trim() : '';
+ return {
+ enabled: stored.voiceEnabled === true,
+ hasApiKey: typeof stored.voiceApiKey === 'string' && stored.voiceApiKey.trim().length > 0,
+ microphoneId: microphoneId || undefined,
+ };
+}
+
+async function closeVoiceHostIfIdle(): Promise<void> {
+ const status = await getVoiceStatus().catch(() => null);
+ if (status?.recording || status?.transcribing) return;
+ await closeOffscreenDocument();
+}
+
+function requestMicPermission(tabId?: number): void {
+ void openMicPermissionPage(tabId).catch((error) => logger.error('voice: mic permission page failed to open', error));
+}
+
+async function beginVoiceCapture(microphoneId: string | undefined, tabId: number | undefined): Promise<void> {
+ if (!(await ensureOffscreenDocument())) {
+ report({ phase: 'error', reason: 'unsupported', error: 'The offscreen document could not be created' });
+ return;
+ }
+
+ const permission = await queryMicPermission().catch(() => ({ state: 'unknown' as const }));
+ if (permission.state === 'denied' || permission.state === 'prompt') {
+ logger.info('voice: microphone permission not granted yet', permission.state);
+ report({ phase: 'error', reason: 'permission-denied', error: 'Microphone access has not been granted' });
+ requestMicPermission(tabId);
+ await closeVoiceHostIfIdle();
+ return;
+ }
+
+ const started = await startVoiceCapture(microphoneId);
+ if (started.started) {
+ logger.info('voice: narration recording started', started);
+ report({ phase: 'recording' });
+ return;
+ }
+
+ logger.warn('voice: narration unavailable, recording without it', started);
+ report({ phase: 'error', reason: started.reason, error: started.error });
+ if (started.reason === 'permission-denied') requestMicPermission(tabId);
+ await closeOffscreenDocument();
+}
+
+export async function startVoiceNarration(tabId?: number): Promise<void> {
+ if (!supportsOffscreen()) return;
+ try {
+ const { enabled, hasApiKey, microphoneId } = await readVoiceSettings();
+ if (!enabled) return;
+ if (!hasApiKey) {
+ logger.warn('voice: no transcription API key, recording without narration');
+ report({ phase: 'error', reason: 'missing-api-key', error: 'No transcription API key is configured' });
+ return;
+ }
+ await withTimeout(beginVoiceCapture(microphoneId, tabId), START_TIMEOUT_MS);
+ } catch (error) {
+ logger.error('voice: narration could not be started', error);
+ report({ phase: 'error', reason: 'unknown', error: String(error) });
+ }
+}
+
+export async function stopVoiceNarration(guideId: string): Promise<void> {
+ if (!supportsOffscreen()) return;
+ try {
+ if (!(await hasOffscreenDocument())) return;
+
+ const steps = await getStepsForGuide(guideId);
+ const response = await stopVoiceCapture(
+ guideId,
+ steps.map((step) => ({ stepId: step.id, timestamp: step.timestamp })),
+ );
+
+ if (response.ok) {
+ logger.info('voice: transcribing narration', response);
+ if (phase.phase === 'recording') report({ phase: 'transcribing' });
+ return;
+ }
+
+ logger.warn('voice: no narration to apply', response);
+ report({ phase: 'error', reason: response.reason, error: response.error });
+ await closeVoiceHostIfIdle();
+ } catch (error) {
+ logger.error('voice: narration could not be stopped', error);
+ report({ phase: 'error', reason: 'unknown', error: String(error) });
+ await closeVoiceHostIfIdle();
+ }
+}
+
+async function persistNarration(event: VoiceResultEvent): Promise<void> {
+ try {
+ const steps = await getStepsForGuide(event.guideId);
+ const updates = narrationUpdates(
+ event.result,
+ steps.map((step) => step.id),
+ );
+ await applyNarrationToSteps(updates);
+ logger.info('voice: narration applied', { narrated: updates.length, of: steps.length, stats: event.result.stats });
+ report({ phase: 'idle', narrated: updates.length });
+ } catch (error) {
+ logger.error('voice: narration could not be applied', error);
+ report({ phase: 'error', reason: 'unknown', error: String(error) });
+ } finally {
+ await closeVoiceHostIfIdle();
+ }
+}
+
+function handleVoiceError(event: VoiceErrorEvent): void {
+ logger.warn('voice: microphone reported an error', event.reason, event.error);
+ report({ phase: 'error', reason: event.reason, error: event.error });
+}
+
+function handlePermissionResult(event: VoicePermissionResultEvent): void {
+ logger.info('voice: microphone permission', event.state);
+ if (event.state === 'granted' && phase.phase === 'error') report({ phase: 'idle' });
+}
+
+export function registerVoiceListeners(): void {
+ if (!supportsOffscreen()) return;
+
+ onRuntimeMessage((message) => {
+ if (!isVoiceMessageFor(VOICE_BACKGROUND_TARGET, message)) return undefined;
+ const event = message as VoiceEvent;
+
+ switch (event.type) {
+ case VoiceMessage.VOICE_RESULT:
+ void persistNarration(event);
+ return undefined;
+ case VoiceMessage.VOICE_ERROR:
+ handleVoiceError(event);
+ return undefined;
+ case VoiceMessage.VOICE_PERMISSION_RESULT:
+ handlePermissionResult(event);
+ return undefined;
+ case VoiceMessage.VOICE_EPOCH:
+ logger.debug('voice: audio epoch', event.audioEpochMs);
+ return undefined;
+ default:
+ return undefined;
+ }
+ });
+}
diff --git a/src/lib/port.ts b/src/lib/port.ts
index 2eea726..a5e8460 100644
--- a/src/lib/port.ts
+++ b/src/lib/port.ts
@@ -1,6 +1,7 @@
import { browser } from '#imports';
import type { CaptureStateValue } from '@/core/capture/machine';
import { logger } from '@/lib/logger';
+import type { VoiceErrorReason } from '@/lib/voice-messages';
const PORT_NAME = 'mimik-panel';
@@ -11,12 +12,23 @@ export interface PanelStateUpdate {
currentGuideId: string | null;
}
-type PortMessage = PanelStateUpdate;
+export type VoicePhase = 'idle' | 'recording' | 'transcribing' | 'error';
+
+export interface PanelVoiceUpdate {
+ type: 'VOICE_UPDATE';
+ phase: VoicePhase;
+ reason?: VoiceErrorReason;
+ error?: string;
+ narrated?: number;
+}
+
+type PortMessage = PanelStateUpdate | PanelVoiceUpdate;
export function connectToBackground(callbacks: {
onStateUpdate: (update: PanelStateUpdate) => void;
onConnect: () => void;
onDisconnect: () => void;
+ onVoiceUpdate?: (update: PanelVoiceUpdate) => void;
}): () => void {
let port: ReturnType<typeof browser.runtime.connect> | null = null;
let destroyed = false;
@@ -32,6 +44,8 @@ export function connectToBackground(callbacks: {
port.onMessage.addListener((msg: PortMessage) => {
if (msg.type === 'STATE_UPDATE') {
callbacks.onStateUpdate(msg);
+ } else if (msg.type === 'VOICE_UPDATE') {
+ callbacks.onVoiceUpdate?.(msg);
}
});
@@ -75,12 +89,20 @@ export function setupPortListener(onPanelConnect?: (port: ReturnType<typeof brow
});
}
-export function broadcastStateToPanel(update: PanelStateUpdate): void {
+function broadcastToPanel(message: PortMessage): void {
for (const port of connectedPorts) {
try {
- port.postMessage(update);
+ port.postMessage(message);
} catch {
connectedPorts.delete(port);
}
}
}
+
+export function broadcastStateToPanel(update: PanelStateUpdate): void {
+ broadcastToPanel(update);
+}
+
+export function broadcastVoiceToPanel(update: PanelVoiceUpdate): void {
+ broadcastToPanel(update);
+}