π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(voice): start and stop narration with the recording
Luis Ortiz9fcdd8339bed
diff --git a/src/entrypoints/background/index.ts b/src/entrypoints/background/index.tsindex c39c53c..846bf9a 100644--- a/src/entrypoints/background/index.ts+++ b/src/entrypoints/background/index.ts@@ -10,6 +10,7 @@ import { getActor, getStateUpdate, initActor, initActorFallback, waitUntilReadyimport { registerNavigationListeners } from './navigation';import { handleCaptureStep, handleFinalizeInputStep, handleUpdateInputStep } from './step-pipeline';import { broadcastStartCapture, broadcastStopCapture, showNotificationOnTab } from './tab-manager';+import { getVoiceUpdate, registerVoiceListeners, startVoiceNarration, stopVoiceNarration } from './voice';async function generateTitleInBackground(guideId: string) {try {@@ -84,12 +85,14 @@ export default defineBackground(() => {initActor().catch(initActorFallback);cancelSession();registerNavigationListeners();+ registerVoiceListeners();setupPortListener((port) => {logger.debug('Panel connected via port');waitUntilReady().then(() => {try {port.postMessage(getStateUpdate());+ port.postMessage(getVoiceUpdate());} catch {}});@@ -123,6 +126,8 @@ export default defineBackground(() => {const activeTab = await getActiveTab();if (activeTab?.id) await showNotificationOnTab(activeTab.id);+ await startVoiceNarration(activeTab?.id);+await broadcastStartCapture(guideId);return { guideId };});@@ -134,7 +139,10 @@ export default defineBackground(() => {await broadcastStopCapture();actor.send({ type: 'STOP_RECORDING' });- if (guideId) generateTitleInBackground(guideId);+ if (guideId) {+ void stopVoiceNarration(guideId);+ generateTitleInBackground(guideId);+ }return { success: true, guideId: guideId ?? undefined };});diff --git a/src/entrypoints/background/voice.ts b/src/entrypoints/background/voice.tsnew file mode 100644index 0000000..43b1664--- /dev/null+++ b/src/entrypoints/background/voice.ts@@ -0,0 +1,198 @@+import { narrationUpdates } from '@/core/capture/voice/narration-updates';+import { applyNarrationToSteps, getStepsForGuide } from '@/core/guides/service';+import { localStorage, onMessage as onRuntimeMessage } from '@/lib/browser-api';+import { logger } from '@/lib/logger';+import {+ closeOffscreenDocument,+ ensureOffscreenDocument,+ getVoiceStatus,+ hasOffscreenDocument,+ openMicPermissionPage,+ queryMicPermission,+ startVoiceCapture,+ stopVoiceCapture,+ supportsOffscreen,+} from '@/lib/offscreen';+import { broadcastVoiceToPanel, type PanelVoiceUpdate } from '@/lib/port';+import {+ isVoiceMessageFor,+ VOICE_BACKGROUND_TARGET,+ type VoiceErrorEvent,+ type VoiceEvent,+ VoiceMessage,+ type VoicePermissionResultEvent,+ type VoiceResultEvent,+} from '@/lib/voice-messages';++const START_TIMEOUT_MS = 8000;++let phase: PanelVoiceUpdate = { type: 'VOICE_UPDATE', phase: 'idle' };++export function getVoiceUpdate(): PanelVoiceUpdate {+ return phase;+}++function report(update: Omit<PanelVoiceUpdate, 'type'>): void {+ phase = { type: 'VOICE_UPDATE', ...update };+ broadcastVoiceToPanel(phase);+}++function withTimeout(work: Promise<void>, ms: number): Promise<void> {+ return new Promise<void>((resolve) => {+ const timer = setTimeout(() => {+ logger.warn('voice: giving up waiting for the microphone');+ resolve();+ }, ms);+ const settle = () => {+ clearTimeout(timer);+ resolve();+ };+ work.then(settle, settle);+ });+}++async function readVoiceSettings(): Promise<{ enabled: boolean; hasApiKey: boolean; microphoneId?: string }> {+ const stored = await localStorage.get(['voiceEnabled', 'voiceApiKey', 'voiceMicrophoneId']);+ const microphoneId = typeof stored.voiceMicrophoneId === 'string' ? stored.voiceMicrophoneId.trim() : '';+ return {+ enabled: stored.voiceEnabled === true,+ hasApiKey: typeof stored.voiceApiKey === 'string' && stored.voiceApiKey.trim().length > 0,+ microphoneId: microphoneId || undefined,+ };+}++async function closeVoiceHostIfIdle(): Promise<void> {+ const status = await getVoiceStatus().catch(() => null);+ if (status?.recording || status?.transcribing) return;+ await closeOffscreenDocument();+}++function requestMicPermission(tabId?: number): void {+ void openMicPermissionPage(tabId).catch((error) => logger.error('voice: mic permission page failed to open', error));+}++async function beginVoiceCapture(microphoneId: string | undefined, tabId: number | undefined): Promise<void> {+ if (!(await ensureOffscreenDocument())) {+ report({ phase: 'error', reason: 'unsupported', error: 'The offscreen document could not be created' });+ return;+ }++ const permission = await queryMicPermission().catch(() => ({ state: 'unknown' as const }));+ if (permission.state === 'denied' || permission.state === 'prompt') {+ logger.info('voice: microphone permission not granted yet', permission.state);+ report({ phase: 'error', reason: 'permission-denied', error: 'Microphone access has not been granted' });+ requestMicPermission(tabId);+ await closeVoiceHostIfIdle();+ return;+ }++ const started = await startVoiceCapture(microphoneId);+ if (started.started) {+ logger.info('voice: narration recording started', started);+ report({ phase: 'recording' });+ return;+ }++ logger.warn('voice: narration unavailable, recording without it', started);+ report({ phase: 'error', reason: started.reason, error: started.error });+ if (started.reason === 'permission-denied') requestMicPermission(tabId);+ await closeOffscreenDocument();+}++export async function startVoiceNarration(tabId?: number): Promise<void> {+ if (!supportsOffscreen()) return;+ try {+ const { enabled, hasApiKey, microphoneId } = await readVoiceSettings();+ if (!enabled) return;+ if (!hasApiKey) {+ logger.warn('voice: no transcription API key, recording without narration');+ report({ phase: 'error', reason: 'missing-api-key', error: 'No transcription API key is configured' });+ return;+ }+ await withTimeout(beginVoiceCapture(microphoneId, tabId), START_TIMEOUT_MS);+ } catch (error) {+ logger.error('voice: narration could not be started', error);+ report({ phase: 'error', reason: 'unknown', error: String(error) });+ }+}++export async function stopVoiceNarration(guideId: string): Promise<void> {+ if (!supportsOffscreen()) return;+ try {+ if (!(await hasOffscreenDocument())) return;++ const steps = await getStepsForGuide(guideId);+ const response = await stopVoiceCapture(+ guideId,+ steps.map((step) => ({ stepId: step.id, timestamp: step.timestamp })),+ );++ if (response.ok) {+ logger.info('voice: transcribing narration', response);+ if (phase.phase === 'recording') report({ phase: 'transcribing' });+ return;+ }++ logger.warn('voice: no narration to apply', response);+ report({ phase: 'error', reason: response.reason, error: response.error });+ await closeVoiceHostIfIdle();+ } catch (error) {+ logger.error('voice: narration could not be stopped', error);+ report({ phase: 'error', reason: 'unknown', error: String(error) });+ await closeVoiceHostIfIdle();+ }+}++async function persistNarration(event: VoiceResultEvent): Promise<void> {+ try {+ const steps = await getStepsForGuide(event.guideId);+ const updates = narrationUpdates(+ event.result,+ steps.map((step) => step.id),+ );+ await applyNarrationToSteps(updates);+ logger.info('voice: narration applied', { narrated: updates.length, of: steps.length, stats: event.result.stats });+ report({ phase: 'idle', narrated: updates.length });+ } catch (error) {+ logger.error('voice: narration could not be applied', error);+ report({ phase: 'error', reason: 'unknown', error: String(error) });+ } finally {+ await closeVoiceHostIfIdle();+ }+}++function handleVoiceError(event: VoiceErrorEvent): void {+ logger.warn('voice: microphone reported an error', event.reason, event.error);+ report({ phase: 'error', reason: event.reason, error: event.error });+}++function handlePermissionResult(event: VoicePermissionResultEvent): void {+ logger.info('voice: microphone permission', event.state);+ if (event.state === 'granted' && phase.phase === 'error') report({ phase: 'idle' });+}++export function registerVoiceListeners(): void {+ if (!supportsOffscreen()) return;++ onRuntimeMessage((message) => {+ if (!isVoiceMessageFor(VOICE_BACKGROUND_TARGET, message)) return undefined;+ const event = message as VoiceEvent;++ switch (event.type) {+ case VoiceMessage.VOICE_RESULT:+ void persistNarration(event);+ return undefined;+ case VoiceMessage.VOICE_ERROR:+ handleVoiceError(event);+ return undefined;+ case VoiceMessage.VOICE_PERMISSION_RESULT:+ handlePermissionResult(event);+ return undefined;+ case VoiceMessage.VOICE_EPOCH:+ logger.debug('voice: audio epoch', event.audioEpochMs);+ return undefined;+ default:+ return undefined;+ }+ });+}diff --git a/src/lib/port.ts b/src/lib/port.tsindex 2eea726..a5e8460 100644--- a/src/lib/port.ts+++ b/src/lib/port.ts@@ -1,6 +1,7 @@import { browser } from '#imports';import type { CaptureStateValue } from '@/core/capture/machine';import { logger } from '@/lib/logger';+import type { VoiceErrorReason } from '@/lib/voice-messages';const PORT_NAME = 'mimik-panel';@@ -11,12 +12,23 @@ export interface PanelStateUpdate {currentGuideId: string | null;}-type PortMessage = PanelStateUpdate;+export type VoicePhase = 'idle' | 'recording' | 'transcribing' | 'error';++export interface PanelVoiceUpdate {+ type: 'VOICE_UPDATE';+ phase: VoicePhase;+ reason?: VoiceErrorReason;+ error?: string;+ narrated?: number;+}++type PortMessage = PanelStateUpdate | PanelVoiceUpdate;export function connectToBackground(callbacks: {onStateUpdate: (update: PanelStateUpdate) => void;onConnect: () => void;onDisconnect: () => void;+ onVoiceUpdate?: (update: PanelVoiceUpdate) => void;}): () => void {let port: ReturnType<typeof browser.runtime.connect> | null = null;let destroyed = false;@@ -32,6 +44,8 @@ export function connectToBackground(callbacks: {port.onMessage.addListener((msg: PortMessage) => {if (msg.type === 'STATE_UPDATE') {callbacks.onStateUpdate(msg);+ } else if (msg.type === 'VOICE_UPDATE') {+ callbacks.onVoiceUpdate?.(msg);}});@@ -75,12 +89,20 @@ export function setupPortListener(onPanelConnect?: (port: ReturnType<typeof brow});}-export function broadcastStateToPanel(update: PanelStateUpdate): void {+function broadcastToPanel(message: PortMessage): void {for (const port of connectedPorts) {try {- port.postMessage(update);+ port.postMessage(message);} catch {connectedPorts.delete(port);}}}++export function broadcastStateToPanel(update: PanelStateUpdate): void {+ broadcastToPanel(update);+}++export function broadcastVoiceToPanel(update: PanelVoiceUpdate): void {+ broadcastToPanel(update);+}