π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(desktop): narrate recordings from a mic button and live meter on the card
Luis Ortiz17f06d82032a
diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.ymlindex 962933d..6d10bf2 100644--- a/apps/desktop/electron-builder.yml+++ b/apps/desktop/electron-builder.yml@@ -36,6 +36,8 @@ mac:target:- dmg- zip+ entitlements: resources/entitlements.mac.plist+ entitlementsInherit: resources/entitlements.mac.plistextendInfo:NSMicrophoneUsageDescription: Mimik listens while you narrate a recording, when you turn narration on.win:diff --git a/apps/desktop/resources/entitlements.mac.plist b/apps/desktop/resources/entitlements.mac.plistnew file mode 100644index 0000000..281212c--- /dev/null+++ b/apps/desktop/resources/entitlements.mac.plist@@ -0,0 +1,14 @@+<?xml version="1.0" encoding="UTF-8"?>+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">+<plist version="1.0">+ <dict>+ <key>com.apple.security.cs.allow-jit</key>+ <true/>+ <key>com.apple.security.cs.allow-unsigned-executable-memory</key>+ <true/>+ <key>com.apple.security.cs.disable-library-validation</key>+ <true/>+ <key>com.apple.security.device.audio-input</key>+ <true/>+ </dict>+</plist>diff --git a/apps/desktop/scripts/check-overlay.ts b/apps/desktop/scripts/check-overlay.tsindex ff6342a..74812d6 100644--- a/apps/desktop/scripts/check-overlay.ts+++ b/apps/desktop/scripts/check-overlay.ts@@ -268,6 +268,28 @@ app.whenReady().then(async () => {await settle();check('removing the step reaches the host', commands.includes('remove'), `commands: ${commands.join(', ')}`);+ overlay.run('mic:on');+ await settle();+ check(+ 'turning narration on reaches the host and keeps recording',+ commands.includes('mic:on') && overlay.state === 'recording',+ `state ${overlay.state}, commands: ${commands.join(', ')}`,+ );++ overlay.setNarration({ level: 0.8, speaking: true });+ await settle();+ const narrating = await card(+ "shown: !document.querySelector('#voice').hidden, text: document.querySelector('#voice').textContent, height: document.body.scrollHeight",+ );+ overlay.setNarration(null);+ await settle();+ const quiet = await card("shown: !document.querySelector('#voice').hidden, height: document.body.scrollHeight");+ check(+ 'narrating shows the meter, and it goes when narration stops',+ narrating.shown === true && String(narrating.text).includes('Hearing you') && quiet.shown === false,+ `while narrating: ${narrating.text} (${narrating.height}px); after: shown ${quiet.shown} (${quiet.height}px)`,+ );+overlay.setBusy(true);await settle();const whileBusy = await windowWithHash('controls')?.webContents.executeJavaScript(@@ -357,8 +379,8 @@ app.whenReady().then(async () => {"JSON.stringify([...document.querySelectorAll('#foot button')].map((b) => b.id + ':' + b.textContent.trim()))",);check(- 'Finish sits on the right, as the primary action',- String(order).includes('secondary:Pause') && String(order).indexOf('primary:Finish') > String(order).indexOf('secondary:Pause'),+ 'Finish leads the footer, with the mic and Pause after it',+ /^\["primary:Finish","mic:","secondary:"\]$/.test(String(order)),String(order),);check(diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.tsindex a2297bf..57fe76e 100644--- a/apps/desktop/src/main/index.ts+++ b/apps/desktop/src/main/index.ts@@ -11,8 +11,22 @@ import { grabDisplay } from './capture/screenshot';import { registerScreenshotProtocol, SCREENSHOT_SCHEME, sweepScreenshots } from './capture/screenshot-store';import { type CaptureMode, type CaptureSettings, loadSettings, saveSettings } from './capture/settings';import { mainI18n } from './i18n';-import { CaptureOverlay, type OverlayAiFailure, type OverlayCommand, type OverlayStep } from './overlay';-import { captureWasHeld, holdCapture, type PermissionKind, readPermissions, requestPermission } from './permissions';+import {+ CaptureOverlay,+ type OverlayAiFailure,+ type OverlayCommand,+ type OverlayNarration,+ type OverlayStep,+} from './overlay';+import {+ askMicrophone,+ captureWasHeld,+ holdCapture,+ microphoneAccess,+ type PermissionKind,+ readPermissions,+ requestPermission,+} from './permissions';import { bindShortcuts, type ShortcutName, shortcutMap, unbindShortcuts } from './shortcuts';import { checkForUpdates } from './updater';@@ -287,6 +301,10 @@ async function onOverlayCommand(command: OverlayCommand): Promise<void> {overlay?.refresh();return;}+ if (command === 'mic:on' || command === 'mic:off') {+ if (command === 'mic:off' || (await askMicrophone())) broadcastOverlay(command, guideId);+ return;+ }if (command === 'remove') {const removed = steps.pop();if (removed && guideId) {@@ -468,15 +486,24 @@ if (!app.requestSingleInstanceLock()) {applyShortcuts();return captureSettings;});+ ipcMain.on('mimik:capture:narration', (_event, narration: OverlayNarration | null) =>+ overlay?.setNarration(narration),+ );ipcMain.on('mimik:capture:described',- (_event, stepId: string, description: string | null, failure: OverlayAiFailure | null) => {+ (+ _event,+ stepId: string,+ description: string | null,+ failure: OverlayAiFailure | null,+ source: 'ai' | 'narration' = 'ai',+ ) => {if (failure) overlay?.setAiFailure(failure);const step = steps.find((candidate) => candidate.id === stepId);if (!step) return;if (description) {step.title = description;- step.source = 'ai';+ step.source = source;}step.pending = false;if (step === steps.at(-1)) {@@ -505,6 +532,11 @@ if (!app.requestSingleInstanceLock()) {return { ...readPermissions(), pending: pendingStart !== null };});ipcMain.handle('mimik:permissions:request', (_event, kind: PermissionKind) => requestPermission(kind));+ ipcMain.handle('mimik:microphone:get', () => microphoneAccess());+ ipcMain.handle('mimik:microphone:request', async () => {+ await askMicrophone();+ return microphoneAccess();+ });ipcMain.on('mimik:permissions:continue', () => {const start = pendingStart;pendingStart = null;diff --git a/apps/desktop/src/main/overlay/index.ts b/apps/desktop/src/main/overlay/index.tsindex 0cfd841..b3382e1 100644--- a/apps/desktop/src/main/overlay/index.ts+++ b/apps/desktop/src/main/overlay/index.ts@@ -17,6 +17,8 @@ export type OverlayCommand =| 'mode:screen'| 'mode:region'| 'remove'+ | 'mic:on'+ | 'mic:off'| 'intro:done';export interface OverlayStep {@@ -25,7 +27,7 @@ export interface OverlayStep {title: string;action: string;src: string;- source: 'heuristic' | 'ai';+ source: 'heuristic' | 'ai' | 'narration';pending: boolean;app: string | null;}@@ -45,6 +47,11 @@ export interface OverlayAiFailure {provider: string;}+export interface OverlayNarration {+ level: number;+ speaking: boolean;+}+export interface OverlayAim {x: number;y: number;@@ -67,6 +74,7 @@ export interface OverlayView {starting: boolean;shortcuts: OverlayShortcuts;aiFailure: OverlayAiFailure | null;+ narration: OverlayNarration | null;}export interface OverlayOptions {@@ -144,6 +152,7 @@ export class CaptureOverlay {private progress: OverlayProgress = { percent: 0, ms: 0 };private print: OverlayPrint = { src: null, aim: null };private aiFailure: OverlayAiFailure | null = null;+ private narration: OverlayNarration | null = null;private editingFrom: OverlayState = 'hidden';private modeBeforeEdit: CaptureMode | null = null;private hiding: { shown: BrowserWindow[]; ready: Promise<unknown>; users: number } | null = null;@@ -192,6 +201,7 @@ export class CaptureOverlay {starting: this.starting,shortcuts: this.options.shortcuts?.() ?? NO_SHORTCUTS,aiFailure: this.aiFailure,+ narration: this.narration,};}@@ -223,6 +233,11 @@ export class CaptureOverlay {this.broadcast();}+ setNarration(narration: OverlayNarration | null): void {+ this.narration = narration;+ this.broadcast();+ }+setBusy(busy: boolean): void {if (this.busy === busy) return;this.busy = busy;@@ -408,6 +423,7 @@ export class CaptureOverlay {hide(): void {this.introDone?.(false);this.starting = false;+ this.narration = null;this.current = 'hidden';this.closeEditors();for (const win of this.windows()) win.hide();@@ -451,6 +467,10 @@ export class CaptureOverlay {if (!this.busy) this.onCommand(command);return;}+ if (command === 'mic:on' || command === 'mic:off') {+ this.onCommand(command);+ return;+ }if (this.starting && command !== 'cancel' && command !== 'stop') return;if (command === 'edit') this.edit();else if (command === 'arm') this.arm();diff --git a/apps/desktop/src/main/permissions.ts b/apps/desktop/src/main/permissions.tsindex f7ec307..bf69ad2 100644--- a/apps/desktop/src/main/permissions.ts+++ b/apps/desktop/src/main/permissions.ts@@ -14,6 +14,8 @@ const SETTINGS_PANE: Record<PermissionKind, string> = {screen: 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture',};+const MICROPHONE_PANE = 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone';+const PROMPTED_FILE = 'permission-prompts.json';const HELD_FILE = 'held-capture.json';const HELD_FOR_MS = 10 * 60 * 1000;@@ -60,3 +62,19 @@ export async function requestPermission(kind: PermissionKind): Promise<void> {if (kind === 'accessibility') systemPreferences.isTrustedAccessibilityClient(true);else await desktopCapturer.getSources({ types: ['screen'], thumbnailSize: { width: 1, height: 1 } }).catch(() => []);}++export type MicrophoneAccess = 'granted' | 'ask' | 'denied';++export function microphoneAccess(): MicrophoneAccess {+ if (process.platform !== 'darwin') return 'granted';+ const status = systemPreferences.getMediaAccessStatus('microphone');+ return status === 'granted' ? 'granted' : status === 'not-determined' ? 'ask' : 'denied';+}++export async function askMicrophone(): Promise<boolean> {+ const access = microphoneAccess();+ if (access === 'granted') return true;+ if (access === 'ask') return systemPreferences.askForMediaAccess('microphone');+ await shell.openExternal(MICROPHONE_PANE);+ return false;+}diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.tsindex 53e5f1b..e5e7dcb 100644--- a/apps/desktop/src/preload/index.ts+++ b/apps/desktop/src/preload/index.ts@@ -1,8 +1,8 @@import { contextBridge, ipcRenderer } from 'electron';import type { CaptureInsert } from '../main/capture/insert';import type { CaptureSettings } from '../main/capture/settings';-import type { OverlayAiFailure } from '../main/overlay';-import type { CapturePermissions, PermissionKind } from '../main/permissions';+import type { OverlayAiFailure, OverlayNarration } from '../main/overlay';+import type { CapturePermissions, MicrophoneAccess, PermissionKind } from '../main/permissions';interface Region {x: number;@@ -17,8 +17,13 @@ const api = {region: (): Promise<Region> => ipcRenderer.invoke('mimik:capture:region'),edit: (insert?: CaptureInsert): Promise<void> => ipcRenderer.invoke('mimik:capture:edit', insert),arm: (insert?: CaptureInsert): Promise<void> => ipcRenderer.invoke('mimik:capture:arm', insert),- described: (stepId: string, description: string | null, failure: OverlayAiFailure | null): void =>- ipcRenderer.send('mimik:capture:described', stepId, description, failure),+ described: (+ stepId: string,+ description: string | null,+ failure: OverlayAiFailure | null,+ source: 'ai' | 'narration' = 'ai',+ ): void => ipcRenderer.send('mimik:capture:described', stepId, description, failure, source),+ narration: (narration: OverlayNarration | null): void => ipcRenderer.send('mimik:capture:narration', narration),settings: {get: (): Promise<CaptureSettings> => ipcRenderer.invoke('mimik:capture:settings:get'),set: (patch: Partial<CaptureSettings>): Promise<CaptureSettings> =>@@ -35,6 +40,10 @@ const api = {);},},+ microphone: {+ get: (): Promise<MicrophoneAccess> => ipcRenderer.invoke('mimik:microphone:get'),+ request: (): Promise<MicrophoneAccess> => ipcRenderer.invoke('mimik:microphone:request'),+ },permissions: {get: (): Promise<CapturePermissions & { pending: boolean }> => ipcRenderer.invoke('mimik:permissions:get'),request: (kind: PermissionKind): Promise<void> => ipcRenderer.invoke('mimik:permissions:request', kind),diff --git a/apps/desktop/src/renderer/capture-host.ts b/apps/desktop/src/renderer/capture-host.tsindex 3619ea9..a0fb653 100644--- a/apps/desktop/src/renderer/capture-host.ts+++ b/apps/desktop/src/renderer/capture-host.ts@@ -1,7 +1,7 @@import { resolveGuideMetaInputs } from '@mimik/core/capture/ai/guide-description';import { settleDescriptions } from '@mimik/core/capture/ai/settle-descriptions';import type { CaptureStepData } from '@mimik/core/capture/sink';-import { i18n } from '@mimik/core/env';+import { i18n, localStorage } from '@mimik/core/env';import { getMostCommonApp } from '@mimik/core/guides/most-common-app';import {allScreenshotIds,@@ -15,14 +15,25 @@ import {} from '@mimik/core/guides/service';import { nameGuide } from './ai/name-guide';import { DesktopCaptureSink } from './capture-sink';+import { DesktopNarration } from './narration';-const sink = new DesktopCaptureSink();+const narration = new DesktopNarration();+const sink = new DesktopCaptureSink(narration);allScreenshotIds().then((ids) => window.mimik.screenshots.sweep(ids));window.mimik.onRequest('mimik:capture:startGuide', (payload) => sink.startGuide(payload === true));window.mimik.onRequest('mimik:capture:step', (payload) => sink.captureStep(payload as CaptureStepData));+window.mimik.capture.onCommand(async (command, state, _region, id) => {+ if (command === 'mic:on' || command === 'mic:off') await localStorage.set({ voiceEnabled: command === 'mic:on' });+ if (!id) return;+ if (command === 'start' || command === 'resume' || (command === 'mic:on' && state === 'recording')) {+ narration.listen(id);+ } else if (command === 'pause' || command === 'mic:off') narration.cut();+ else if (command === 'cancel') narration.abort();+});+window.mimik.onRequest('mimik:capture:removeStep', async (payload) => {const { guideId, stepId } = payload as { guideId: string; stepId: string };await deleteStep(guideId, stepId);@@ -31,6 +42,7 @@ window.mimik.onRequest('mimik:capture:removeStep', async (payload) => {window.mimik.onRequest('mimik:capture:finishGuide', async (payload) => {const guideId = payload as string;+ await narration.finish();const steps = await getStepsForGuide(guideId);if (steps.length === 0) return true;const app = getMostCommonApp(steps)?.name;@@ -49,6 +61,7 @@ window.mimik.onRequest('mimik:capture:finishGuide', async (payload) => {window.mimik.onRequest('mimik:capture:insertGuide', async (payload) => {const { guideId, targetGuideId, atIndex } = payload as { guideId: string; targetGuideId: string; atIndex: number };+ await narration.finish();await settleDescriptions(guideId);await createSnapshot(targetGuideId);await mergeGuideInto(guideId, targetGuideId, atIndex);diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.tsindex 602ef7c..5034b29 100644--- a/apps/desktop/src/renderer/capture-sink.ts+++ b/apps/desktop/src/renderer/capture-sink.ts@@ -15,6 +15,7 @@ import {clearStepAiPending,createGuide,createStep,+ getStep,getStepsForGuide,saveScreenshot,updateStepDescription,@@ -22,6 +23,7 @@ import {import { screenshotForElement } from '@mimik/core/screenshot/record';import { credentials } from './ai/credentials';import { describeStep } from './ai/describe-step';+import type { DesktopNarration } from './narration';export type DesktopCaptureStepData = CaptureStepData & {inputValue?: string;@@ -29,6 +31,8 @@ export type DesktopCaptureStepData = CaptureStepData & {};export class DesktopCaptureSink implements CaptureSink {+ constructor(private readonly narration?: DesktopNarration) {}+async startGuide(staging = false): Promise<string> {const guide = await createGuide(crypto.randomUUID(), staging);return guide.id;@@ -63,7 +67,10 @@ export class DesktopCaptureSink implements CaptureSink {}const description = buildFallbackDescription(data.action, meta, data.inputValue);- const pending = (await credentials()) !== null;+ const hasKey = (await credentials()) !== null;+ const narrating = this.narration?.capturing === true;+ const pending = hasKey || narrating;+ const timestamp = Date.now();await createStep({id: stepId,@@ -74,7 +81,7 @@ export class DesktopCaptureSink implements CaptureSink {url: '',app: meta.app,window: meta.window,- timestamp: Date.now(),+ timestamp,screenshotId,elementMeta: meta,descriptionSource: 'heuristic',@@ -83,15 +90,15 @@ export class DesktopCaptureSink implements CaptureSink {});await addStepToGuide(data.guideId, stepId);- if (pending) {- queueDescription(data.guideId, async () => {- const previous = (await getStepsForGuide(data.guideId)).find((step) => step.index === index - 1);- const { text, failure } = await describeStep(data.action, meta, previous?.description);- if (text) await updateStepDescription(stepId, text, 'ai');- await clearStepAiPending(stepId);- window.mimik.capture.described(stepId, text, failure);- });- }+ const describe = async () => {+ const previous = (await getStepsForGuide(data.guideId)).find((step) => step.index === index - 1);+ const { text, failure } = await describeStep(data.action, meta, previous?.description);+ await clearStepAiPending(stepId, text ?? undefined);+ const written = (await getStep(stepId))?.descriptionSource === 'ai';+ window.mimik.capture.described(stepId, written ? text : null, failure);+ };+ if (narrating) this.narration?.step(data.guideId, { stepId, timestamp }, hasKey ? describe : null);+ else if (hasKey) queueDescription(data.guideId, describe);return { stepId, title: description, pending };}diff --git a/apps/desktop/src/renderer/hooks/use-microphone-access.ts b/apps/desktop/src/renderer/hooks/use-microphone-access.tsnew file mode 100644index 0000000..6a45839--- /dev/null+++ b/apps/desktop/src/renderer/hooks/use-microphone-access.ts@@ -0,0 +1,26 @@+import { useEffect, useState } from 'react';+import type { MicrophoneAccess } from '../../main/permissions';++const POLL_MS = 2000;++export function useMicrophoneAccess() {+ const [access, setAccess] = useState<MicrophoneAccess | null>(null);++ useEffect(() => {+ let alive = true;+ const read = () =>+ window.mimik.microphone.get().then((found) => {+ if (alive) setAccess(found);+ });+ void read();+ const timer = setInterval(read, POLL_MS);+ return () => {+ alive = false;+ clearInterval(timer);+ };+ }, []);++ const request = () => window.mimik.microphone.request().then(setAccess);++ return { access, request };+}diff --git a/apps/desktop/src/renderer/icons.ts b/apps/desktop/src/renderer/icons.tsindex 59ed196..20b4f32 100644--- a/apps/desktop/src/renderer/icons.ts+++ b/apps/desktop/src/renderer/icons.ts@@ -7,6 +7,8 @@ import {createElement,Keyboard,LoaderCircle,+ Mic,+ MicOff,Monitor,MousePointer2,Pause,@@ -14,6 +16,7 @@ import {Trash2,TriangleAlert,Video,+ X,} from 'lucide';const ICONS = {@@ -31,6 +34,9 @@ const ICONS = {keyboard: Keyboard,pointer: MousePointer2,alert: TriangleAlert,+ mic: Mic,+ micOff: MicOff,+ close: X,};export type IconName = keyof typeof ICONS;diff --git a/apps/desktop/src/renderer/narration.ts b/apps/desktop/src/renderer/narration.tsnew file mode 100644index 0000000..b8a0a02--- /dev/null+++ b/apps/desktop/src/renderer/narration.ts@@ -0,0 +1,169 @@+import { queueDescription } from '@mimik/core/capture/ai/description-queue';+import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@mimik/core/capture/voice/api-key';+import { MicRecorder } from '@mimik/core/capture/voice/mic-recorder';+import { narrateRecording, type VoiceRecording } from '@mimik/core/capture/voice/narrate-recording';+import { narrationUpdates } from '@mimik/core/capture/voice/narration-updates';+import { partialRecording } from '@mimik/core/capture/voice/partial-recording';+import { readTranscriptionSettings } from '@mimik/core/capture/voice/read-transcription-settings';+import type { StepMark } from '@mimik/core/capture/voice/step-windows';+import { usableRecording } from '@mimik/core/capture/voice/usable-recording';+import { assetUrl, localStorage } from '@mimik/core/env';+import {+ applyNarrationToSteps,+ clearStepAiPending,+ findExistingStepIds,+ getStepsForGuide,+ saveTranscript,+} from '@mimik/core/guides/service';+import { logger } from '@mimik/core/logger';++const SETTLE_MS = 30_000;+const REFUSED = ['NotAllowedError', 'SecurityError'];++type Describe = () => Promise<void>;++export class DesktopNarration {+ private live: Promise<MicRecorder | null> | null = null;+ private mic: MicRecorder | null = null;+ private guideId: string | null = null;+ private flushedUpTo = 0;+ private readonly pending = new Set<Promise<void>>();+ private readonly deferred = new Map<string, { guideId: string; describe: Describe | null }>();++ async wanted(): Promise<boolean> {+ const stored = await localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled']);+ return stored.voiceEnabled === true && hasVoiceApiKey(stored);+ }++ get capturing(): boolean {+ return this.mic !== null;+ }++ listen(guideId: string): void {+ this.guideId = guideId;+ this.live ??= this.open();+ }++ step(guideId: string, mark: StepMark, describe: Describe | null): void {+ this.deferred.set(mark.stepId, { guideId, describe });+ const mic = this.mic;+ this.track(async () => {+ const full = mic && usableRecording(mic.snapshot());+ const closesAt = full ? (mark.timestamp - full.audioEpochMs) / 1000 : 0;+ const slice = full && closesAt > this.flushedUpTo ? partialRecording(full, this.flushedUpTo, closesAt) : null;+ if (!slice) {+ this.describeNow(mark.stepId);+ return;+ }+ this.flushedUpTo = closesAt;+ const narrated = await this.transcribe(guideId, slice, [mark]);+ if (!narrated.includes(mark.stepId)) this.describeNow(mark.stepId);+ });+ }++ cut(): void {+ const opening = this.live;+ const guideId = this.guideId;+ const from = this.flushedUpTo;+ this.live = null;+ this.mic = null;+ this.flushedUpTo = 0;+ window.mimik.capture.narration(null);+ if (!opening || !guideId) return;+ const inflight = [...this.pending];+ this.track(async () => {+ const mic = await opening;+ const audio = mic && usableRecording(mic.stop());+ window.mimik.capture.narration(null);+ await Promise.all(inflight);+ const tail = audio && from > 0 ? partialRecording(audio, from, audio.durationSeconds) : audio;+ const steps = await getStepsForGuide(guideId);+ if (tail && steps.length > 0) {+ await this.transcribe(+ guideId,+ tail,+ steps.map((step) => ({ stepId: step.id, timestamp: step.timestamp })),+ );+ }+ for (const [stepId, entry] of [...this.deferred]) {+ if (entry.guideId === guideId) this.describeNow(stepId);+ }+ });+ }++ abort(): void {+ const opening = this.live;+ this.live = null;+ this.mic = null;+ this.flushedUpTo = 0;+ this.deferred.clear();+ window.mimik.capture.narration(null);+ void opening?.then((mic) => {+ mic?.release();+ window.mimik.capture.narration(null);+ });+ }++ async finish(): Promise<void> {+ this.cut();+ await Promise.race([Promise.all(this.pending), new Promise((resolve) => setTimeout(resolve, SETTLE_MS))]);+ }++ private track(work: () => Promise<void>): void {+ const running = work()+ .catch((error) => logger.error('voice: narration could not be applied', error))+ .finally(() => this.pending.delete(running));+ this.pending.add(running);+ }++ private describeNow(stepId: string): void {+ const entry = this.deferred.get(stepId);+ if (!entry) return;+ this.deferred.delete(stepId);+ if (entry.describe) {+ queueDescription(entry.guideId, entry.describe);+ return;+ }+ void clearStepAiPending(stepId).then(() => window.mimik.capture.described(stepId, null, null));+ }++ private async open(): Promise<MicRecorder | null> {+ if (!(await this.wanted())) return null;+ const { voiceMicrophoneId } = await localStorage.get(['voiceMicrophoneId']);+ const mic = new MicRecorder(assetUrl('pcm-processor.js'), {+ onEpoch: () => undefined,+ onLevel: (level, speaking) => window.mimik.capture.narration({ level, speaking }),+ onStreamEnded: () => this.cut(),+ });+ try {+ await mic.start(voiceMicrophoneId || undefined);+ if (!this.live) {+ mic.release();+ return null;+ }+ this.mic = mic;+ window.mimik.capture.narration({ level: 0, speaking: false });+ return mic;+ } catch (error) {+ mic.release();+ logger.warn('voice: narration could not start, recording without it', error);+ if (error instanceof Error && REFUSED.includes(error.name)) await localStorage.set({ voiceEnabled: false });+ return null;+ }+ }++ private async transcribe(guideId: string, audio: VoiceRecording, marks: StepMark[]): Promise<string[]> {+ const settings = await readTranscriptionSettings();+ if (!settings.apiKey) return [];+ const result = await narrateRecording(audio, marks, settings);+ await saveTranscript(guideId, result.transcript);+ const surviving = await findExistingStepIds(result.descriptions.map((entry) => entry.stepId));+ const updates = narrationUpdates(result, surviving);+ await applyNarrationToSteps(updates, result.transcript.epochMs);+ for (const { stepId, description } of updates) {+ this.deferred.delete(stepId);+ window.mimik.capture.described(stepId, description, null, 'narration');+ }+ return updates.map((update) => update.stepId);+ }+}diff --git a/apps/desktop/src/renderer/overlay.css b/apps/desktop/src/renderer/overlay.cssindex 2a4f8ff..fce3ccf 100644--- a/apps/desktop/src/renderer/overlay.css+++ b/apps/desktop/src/renderer/overlay.css@@ -668,6 +668,11 @@ body.controls #source.basic {color: var(--lavender);}+body.controls #source.voice {+ background: rgba(5, 150, 105, 0.1);+ color: var(--success);+}+body.controls #source.ai {background: #eef2ff;color: var(--accent);@@ -814,10 +819,87 @@ body.controls #foot {}body.controls #foot button {- flex-grow: 1;- flex-basis: 0;+ flex: none;justify-content: center;height: 40px;+ border-radius: 999px;+}++body.controls #foot #primary {+ flex: 1 1 0;+}++body.controls #foot #mic,+body.controls #foot #secondary {+ width: 40px;+ padding: 0;+ color: var(--muted);+}++body.controls #foot #secondary:hover,+body.controls #foot #mic:hover {+ color: var(--accent);+}++body.controls #foot #mic.on {+ border-color: var(--accent);+ background: #eef2ff;+ color: var(--accent);+}++body.controls #foot #mic.locked {+ border-style: dashed;+ opacity: 0.6;+ cursor: not-allowed;+}++body.controls #voice {+ margin-top: 12px;+ padding: 8px 10px;+ border-radius: 9px;+ background: #eef2ff;+}++body.controls #voiceLine {+ display: flex;+ align-items: center;+ gap: 8px;+ font-size: 11.5px;+ font-weight: 600;+ color: var(--deep);+}++body.controls #voiceHint {+ margin: 4px 0 0;+ font-size: 10.5px;+ line-height: 1.5;+ color: var(--muted);+}++body.controls #voiceBars {+ display: flex;+ flex-shrink: 0;+ align-items: center;+ gap: 2px;+ height: 14px;+}++body.controls #voiceBars i {+ width: 3px;+ height: 14px;+ border-radius: 999px;+ background: var(--lavender);+ transition: transform 0.3s ease-out;+}++body.controls #voiceBars.speaking i {+ background: var(--accent);+}++@media (prefers-reduced-motion: reduce) {+ body.controls #voiceBars i {+ transition: none;+ }}body.controls button {diff --git a/apps/desktop/src/renderer/overlay/controls.ts b/apps/desktop/src/renderer/overlay/controls.tsindex 676c4cd..cc930c3 100644--- a/apps/desktop/src/renderer/overlay/controls.ts+++ b/apps/desktop/src/renderer/overlay/controls.ts@@ -1,6 +1,7 @@import { aiFailureNotice } from '@mimik/core/capture/ai/errors';import { splitAtShortcut } from '@mimik/core/capture/split-at-shortcut';-import { i18n } from '@mimik/core/env';+import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@mimik/core/capture/voice/api-key';+import { i18n, localStorage } from '@mimik/core/env';import type { OverlayView } from '../../main/overlay';import { icon } from '../icons';import { cameraMascot } from './camera-mascot';@@ -22,6 +23,9 @@ const MODE_ID: Record<string, string> = { window: 'window', screen: 'screen', reconst AI_WAIT_MS = 8000;+const VOICE_BARS = [0.45, 0.75, 1, 0.75, 0.45];+const VOICE_BAR_FLOOR = 0.14;+export function controls(): void {document.body.className = 'controls';@@ -78,7 +82,17 @@ export function controls(): void {const flyerShot = el('img', { alt: '' });const flyer = el('div', { id: 'flyer' }, flyerShot);const stage = el('div', { id: 'stage' }, preview, stepTitle, stepMeta, printer, flyer);- const body = el('div', { id: 'body' }, stage, intro, hint);+ const voiceBars = el('span', { id: 'voiceBars' });+ const bars = VOICE_BARS.map(() => voiceBars.appendChild(el('i', {})));+ const voiceLabel = el('strong', {});+ const voiceHint = el('p', { id: 'voiceHint' }, i18n.t('voice_orderHint'));+ const voice = el(+ 'div',+ { id: 'voice', role: 'status' },+ el('div', { id: 'voiceLine' }, voiceBars, voiceLabel),+ voiceHint,+ );+ const body = el('div', { id: 'body' }, stage, intro, hint, voice);const modeLabel = el('p', { id: 'modeLabel' }, i18n.t('desktop_captureMode'));const modeRow = el('div', { id: 'modes' });@@ -102,12 +116,16 @@ export function controls(): void {const secondary = el('button', { id: 'secondary', type: 'button' });const primary = el('button', { id: 'primary', type: 'button', className: 'primary' });- const foot = el('div', { id: 'foot' }, secondary, primary);+ const mic = el('button', { id: 'mic', type: 'button' });+ const foot = el('div', { id: 'foot' }, primary, mic, secondary);document.body.append(head, body, modes, aiNotice, foot);const report = () => window.mimikOverlay.size(document.body.scrollWidth, document.body.scrollHeight);+ let micOn = false;+ let micKeyed = false;+ let shownMic = '';let shownIcon = '';let shownChevron = '';let shownMode = '';@@ -146,7 +164,7 @@ export function controls(): void {};const render = (view: OverlayView) => {- const { state, step, busy, progress, print, starting, mode, shortcuts, aiFailure } = view;+ const { state, step, busy, progress, print, starting, mode, shortcuts, aiFailure, narration } = view;const armed = state === 'armed';const recording = state === 'recording';const paused = state === 'paused';@@ -209,6 +227,18 @@ export function controls(): void {}badge.hidden = paused || collapsed;+ const micLocked = !micKeyed && !micOn;+ const micState = micOn ? 'on' : micLocked ? 'locked' : 'off';+ if (micState !== shownMic) {+ shownMic = micState;+ const micLabel = i18n.t(micLocked ? 'voice_needsApiKey' : micOn ? 'voice_turnOff' : 'voice_turnOn');+ mic.className = micState;+ mic.title = micLabel;+ mic.setAttribute('aria-label', micLabel);+ mic.setAttribute('aria-pressed', String(micOn));+ mic.replaceChildren(icon(micOn ? 'mic' : 'micOff', 16));+ }+const chevron = collapsed ? 'up' : 'down';if (chevron !== shownChevron) {shownChevron = chevron;@@ -242,8 +272,9 @@ export function controls(): void {const extra = stepTitle.getBoundingClientRect().height - Number.parseFloat(getComputedStyle(stepTitle).lineHeight);preview.style.setProperty('--title-extra', `${step ? Math.max(0, extra) : 0}px`);source.hidden = !step;- source.textContent = i18n.t(step?.source === 'ai' ? 'stepSource_ai' : 'stepSource_basic');- source.className = step?.source === 'ai' ? 'ai' : 'basic';+ const sourceKind = step?.source === 'ai' ? 'ai' : step?.source === 'narration' ? 'voice' : 'basic';+ source.textContent = i18n.t(`stepSource_${sourceKind}`);+ source.className = sourceKind;metaText.textContent =[step ? i18n.t('export_stepLabel', [String(step.index)]) : '', step?.app ?? ''].filter(Boolean).join(' Β· ') ||'\u00a0';@@ -255,7 +286,15 @@ export function controls(): void {? 'Nothing is recorded while paused.': 'Your first click will show up here.';const key = armed ? shortcuts.startStop : shortcuts.capture;- hint.hidden = intro.hidden || resting || !key;+ hint.hidden = intro.hidden || resting || !key || Boolean(narration);+ voice.hidden = collapsed || !narration;+ voiceHint.hidden = intro.hidden;+ voiceBars.classList.toggle('speaking', narration?.speaking === true);+ voiceLabel.textContent = i18n.t(narration?.speaking ? 'voice_micHearing' : 'voice_micQuiet');+ bars.forEach((bar, index) => {+ const scale = Math.max(VOICE_BAR_FLOOR, Math.min(1, (narration?.level ?? 0) * VOICE_BARS[index]));+ bar.style.transform = `scaleY(${scale})`;+ });hintKey.textContent = key ?? '';hintText.textContent = armed ? ' to start and stop.' : ' to capture without clicking.';body.hidden = collapsed;@@ -265,9 +304,10 @@ export function controls(): void {if (state !== shownIcon) {shownIcon = state;- secondary.replaceChildren();- if (!armed) secondary.append(icon(recording ? 'pause' : 'play'));- secondary.append(i18n.t(armed ? 'common_close' : recording ? 'desktop_pause' : 'desktop_resume'));+ const secondaryLabel = i18n.t(armed ? 'common_close' : recording ? 'desktop_pause' : 'desktop_resume');+ secondary.replaceChildren(icon(armed ? 'close' : recording ? 'pause' : 'play', 16));+ secondary.title = secondaryLabel;+ secondary.setAttribute('aria-label', secondaryLabel);primary.replaceChildren(icon(armed ? 'video' : 'check'));primary.append(i18n.t(armed ? 'desktop_startButton' : 'desktop_finish'));}@@ -324,6 +364,17 @@ export function controls(): void {primary.addEventListener('click', () => window.mimikOverlay.command(primary.dataset.command ?? 'start'));secondary.addEventListener('click', () => window.mimikOverlay.command(secondary.dataset.command ?? 'cancel'));remove.addEventListener('click', () => window.mimikOverlay.command('remove'));+ mic.addEventListener('click', () => {+ if (shownMic !== 'locked') window.mimikOverlay.command(micOn ? 'mic:off' : 'mic:on');+ });+ const readMic = () =>+ localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled']).then((stored) => {+ micOn = stored.voiceEnabled === true;+ micKeyed = hasVoiceApiKey(stored);+ paint();+ });+ void readMic();+ window.addEventListener('storage', () => void readMic());collapse.addEventListener('click', () => {collapsed = !collapsed;paint();diff --git a/apps/desktop/src/renderer/settings/SettingsPanel.tsx b/apps/desktop/src/renderer/settings/SettingsPanel.tsxindex 37d093c..57329c7 100644--- a/apps/desktop/src/renderer/settings/SettingsPanel.tsx+++ b/apps/desktop/src/renderer/settings/SettingsPanel.tsx@@ -5,6 +5,8 @@ import {AiSettings,ApiKeysSettings,BrandingSettings,+ MicrophoneAccessRow,+ NarrationSettings,Segmented,Switch,useApiKeys,@@ -24,6 +26,8 @@ import {} from 'lucide-react';import { useEffect, useState } from 'react';import type { CaptureSettings, CaptureShortcuts } from '../../main/capture/settings';+import { useMicrophoneAccess } from '../hooks/use-microphone-access';+import { requestMicrophoneAccess } from '../lib/request-microphone-access';import { Card } from './Card';import { GeneralSettings } from './GeneralSettings';import { REOPEN_SETTINGS } from './lib/reopen-settings';@@ -61,6 +65,7 @@ export function SettingsPanel() {return reopened ?? 'general';});const keys = useApiKeys();+ const microphone = useMicrophoneAccess();const [settings, setSettings] = useState<CaptureSettings | null>(null);useEffect(() => {@@ -100,6 +105,28 @@ export function SettingsPanel() {{section === 'ai' && (<><AiSettings keys={keys} onOpenKeys={() => setSection('keys')} />+ <NarrationSettings+ keys={keys}+ onOpenKeys={() => setSection('keys')}+ onRequestAccess={requestMicrophoneAccess}+ access={+ microphone.access && microphone.access !== 'granted' ? (+ <MicrophoneAccessRow+ refused={microphone.access === 'denied'}+ hint={i18n.t(+ microphone.access === 'denied' ? 'desktop_micAccessRefused' : 'desktop_micAccessAsk',+ )}+ action={i18n.t(+ microphone.access === 'denied' ? 'desktop_micAccessOpen' : 'settings_microphoneAccessAllow',+ )}+ external={microphone.access === 'denied'}+ onRequest={() => void microphone.request()}+ />+ ) : null+ }+ microphoneDisabled={microphone.access !== 'granted'}+ liveMeter+ /><VoiceoverSettings keys={keys.keys} listVoices={fetchVoices} onOpenKeys={() => setSection('keys')} /></>)}