Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(desktop): narrate recordings from a mic button and live meter on the card

Luis Ortiz17f06d82032a
diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml
index 962933d..6d10bf2 100644
--- a/apps/desktop/electron-builder.yml
+++ b/apps/desktop/electron-builder.yml
@@ -36,6 +36,8 @@ mac:
target:
- dmg
- zip
+ entitlements: resources/entitlements.mac.plist
+ entitlementsInherit: resources/entitlements.mac.plist
extendInfo:
NSMicrophoneUsageDescription: Mimik listens while you narrate a recording, when you turn narration on.
win:
diff --git a/apps/desktop/resources/entitlements.mac.plist b/apps/desktop/resources/entitlements.mac.plist
new file mode 100644
index 0000000..281212c
--- /dev/null
+++ b/apps/desktop/resources/entitlements.mac.plist
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+ <dict>
+ <key>com.apple.security.cs.allow-jit</key>
+ <true/>
+ <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+ <true/>
+ <key>com.apple.security.cs.disable-library-validation</key>
+ <true/>
+ <key>com.apple.security.device.audio-input</key>
+ <true/>
+ </dict>
+</plist>
diff --git a/apps/desktop/scripts/check-overlay.ts b/apps/desktop/scripts/check-overlay.ts
index ff6342a..74812d6 100644
--- a/apps/desktop/scripts/check-overlay.ts
+++ b/apps/desktop/scripts/check-overlay.ts
@@ -268,6 +268,28 @@ app.whenReady().then(async () => {
await settle();
check('removing the step reaches the host', commands.includes('remove'), `commands: ${commands.join(', ')}`);
+ overlay.run('mic:on');
+ await settle();
+ check(
+ 'turning narration on reaches the host and keeps recording',
+ commands.includes('mic:on') && overlay.state === 'recording',
+ `state ${overlay.state}, commands: ${commands.join(', ')}`,
+ );
+
+ overlay.setNarration({ level: 0.8, speaking: true });
+ await settle();
+ const narrating = await card(
+ "shown: !document.querySelector('#voice').hidden, text: document.querySelector('#voice').textContent, height: document.body.scrollHeight",
+ );
+ overlay.setNarration(null);
+ await settle();
+ const quiet = await card("shown: !document.querySelector('#voice').hidden, height: document.body.scrollHeight");
+ check(
+ 'narrating shows the meter, and it goes when narration stops',
+ narrating.shown === true && String(narrating.text).includes('Hearing you') && quiet.shown === false,
+ `while narrating: ${narrating.text} (${narrating.height}px); after: shown ${quiet.shown} (${quiet.height}px)`,
+ );
+
overlay.setBusy(true);
await settle();
const whileBusy = await windowWithHash('controls')?.webContents.executeJavaScript(
@@ -357,8 +379,8 @@ app.whenReady().then(async () => {
"JSON.stringify([...document.querySelectorAll('#foot button')].map((b) => b.id + ':' + b.textContent.trim()))",
);
check(
- 'Finish sits on the right, as the primary action',
- String(order).includes('secondary:Pause') && String(order).indexOf('primary:Finish') > String(order).indexOf('secondary:Pause'),
+ 'Finish leads the footer, with the mic and Pause after it',
+ /^\["primary:Finish","mic:","secondary:"\]$/.test(String(order)),
String(order),
);
check(
diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts
index a2297bf..57fe76e 100644
--- a/apps/desktop/src/main/index.ts
+++ b/apps/desktop/src/main/index.ts
@@ -11,8 +11,22 @@ import { grabDisplay } from './capture/screenshot';
import { registerScreenshotProtocol, SCREENSHOT_SCHEME, sweepScreenshots } from './capture/screenshot-store';
import { type CaptureMode, type CaptureSettings, loadSettings, saveSettings } from './capture/settings';
import { mainI18n } from './i18n';
-import { CaptureOverlay, type OverlayAiFailure, type OverlayCommand, type OverlayStep } from './overlay';
-import { captureWasHeld, holdCapture, type PermissionKind, readPermissions, requestPermission } from './permissions';
+import {
+ CaptureOverlay,
+ type OverlayAiFailure,
+ type OverlayCommand,
+ type OverlayNarration,
+ type OverlayStep,
+} from './overlay';
+import {
+ askMicrophone,
+ captureWasHeld,
+ holdCapture,
+ microphoneAccess,
+ type PermissionKind,
+ readPermissions,
+ requestPermission,
+} from './permissions';
import { bindShortcuts, type ShortcutName, shortcutMap, unbindShortcuts } from './shortcuts';
import { checkForUpdates } from './updater';
@@ -287,6 +301,10 @@ async function onOverlayCommand(command: OverlayCommand): Promise<void> {
overlay?.refresh();
return;
}
+ if (command === 'mic:on' || command === 'mic:off') {
+ if (command === 'mic:off' || (await askMicrophone())) broadcastOverlay(command, guideId);
+ return;
+ }
if (command === 'remove') {
const removed = steps.pop();
if (removed && guideId) {
@@ -468,15 +486,24 @@ if (!app.requestSingleInstanceLock()) {
applyShortcuts();
return captureSettings;
});
+ ipcMain.on('mimik:capture:narration', (_event, narration: OverlayNarration | null) =>
+ overlay?.setNarration(narration),
+ );
ipcMain.on(
'mimik:capture:described',
- (_event, stepId: string, description: string | null, failure: OverlayAiFailure | null) => {
+ (
+ _event,
+ stepId: string,
+ description: string | null,
+ failure: OverlayAiFailure | null,
+ source: 'ai' | 'narration' = 'ai',
+ ) => {
if (failure) overlay?.setAiFailure(failure);
const step = steps.find((candidate) => candidate.id === stepId);
if (!step) return;
if (description) {
step.title = description;
- step.source = 'ai';
+ step.source = source;
}
step.pending = false;
if (step === steps.at(-1)) {
@@ -505,6 +532,11 @@ if (!app.requestSingleInstanceLock()) {
return { ...readPermissions(), pending: pendingStart !== null };
});
ipcMain.handle('mimik:permissions:request', (_event, kind: PermissionKind) => requestPermission(kind));
+ ipcMain.handle('mimik:microphone:get', () => microphoneAccess());
+ ipcMain.handle('mimik:microphone:request', async () => {
+ await askMicrophone();
+ return microphoneAccess();
+ });
ipcMain.on('mimik:permissions:continue', () => {
const start = pendingStart;
pendingStart = null;
diff --git a/apps/desktop/src/main/overlay/index.ts b/apps/desktop/src/main/overlay/index.ts
index 0cfd841..b3382e1 100644
--- a/apps/desktop/src/main/overlay/index.ts
+++ b/apps/desktop/src/main/overlay/index.ts
@@ -17,6 +17,8 @@ export type OverlayCommand =
| 'mode:screen'
| 'mode:region'
| 'remove'
+ | 'mic:on'
+ | 'mic:off'
| 'intro:done';
export interface OverlayStep {
@@ -25,7 +27,7 @@ export interface OverlayStep {
title: string;
action: string;
src: string;
- source: 'heuristic' | 'ai';
+ source: 'heuristic' | 'ai' | 'narration';
pending: boolean;
app: string | null;
}
@@ -45,6 +47,11 @@ export interface OverlayAiFailure {
provider: string;
}
+export interface OverlayNarration {
+ level: number;
+ speaking: boolean;
+}
+
export interface OverlayAim {
x: number;
y: number;
@@ -67,6 +74,7 @@ export interface OverlayView {
starting: boolean;
shortcuts: OverlayShortcuts;
aiFailure: OverlayAiFailure | null;
+ narration: OverlayNarration | null;
}
export interface OverlayOptions {
@@ -144,6 +152,7 @@ export class CaptureOverlay {
private progress: OverlayProgress = { percent: 0, ms: 0 };
private print: OverlayPrint = { src: null, aim: null };
private aiFailure: OverlayAiFailure | null = null;
+ private narration: OverlayNarration | null = null;
private editingFrom: OverlayState = 'hidden';
private modeBeforeEdit: CaptureMode | null = null;
private hiding: { shown: BrowserWindow[]; ready: Promise<unknown>; users: number } | null = null;
@@ -192,6 +201,7 @@ export class CaptureOverlay {
starting: this.starting,
shortcuts: this.options.shortcuts?.() ?? NO_SHORTCUTS,
aiFailure: this.aiFailure,
+ narration: this.narration,
};
}
@@ -223,6 +233,11 @@ export class CaptureOverlay {
this.broadcast();
}
+ setNarration(narration: OverlayNarration | null): void {
+ this.narration = narration;
+ this.broadcast();
+ }
+
setBusy(busy: boolean): void {
if (this.busy === busy) return;
this.busy = busy;
@@ -408,6 +423,7 @@ export class CaptureOverlay {
hide(): void {
this.introDone?.(false);
this.starting = false;
+ this.narration = null;
this.current = 'hidden';
this.closeEditors();
for (const win of this.windows()) win.hide();
@@ -451,6 +467,10 @@ export class CaptureOverlay {
if (!this.busy) this.onCommand(command);
return;
}
+ if (command === 'mic:on' || command === 'mic:off') {
+ this.onCommand(command);
+ return;
+ }
if (this.starting && command !== 'cancel' && command !== 'stop') return;
if (command === 'edit') this.edit();
else if (command === 'arm') this.arm();
diff --git a/apps/desktop/src/main/permissions.ts b/apps/desktop/src/main/permissions.ts
index f7ec307..bf69ad2 100644
--- a/apps/desktop/src/main/permissions.ts
+++ b/apps/desktop/src/main/permissions.ts
@@ -14,6 +14,8 @@ const SETTINGS_PANE: Record<PermissionKind, string> = {
screen: 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture',
};
+const MICROPHONE_PANE = 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone';
+
const PROMPTED_FILE = 'permission-prompts.json';
const HELD_FILE = 'held-capture.json';
const HELD_FOR_MS = 10 * 60 * 1000;
@@ -60,3 +62,19 @@ export async function requestPermission(kind: PermissionKind): Promise<void> {
if (kind === 'accessibility') systemPreferences.isTrustedAccessibilityClient(true);
else await desktopCapturer.getSources({ types: ['screen'], thumbnailSize: { width: 1, height: 1 } }).catch(() => []);
}
+
+export type MicrophoneAccess = 'granted' | 'ask' | 'denied';
+
+export function microphoneAccess(): MicrophoneAccess {
+ if (process.platform !== 'darwin') return 'granted';
+ const status = systemPreferences.getMediaAccessStatus('microphone');
+ return status === 'granted' ? 'granted' : status === 'not-determined' ? 'ask' : 'denied';
+}
+
+export async function askMicrophone(): Promise<boolean> {
+ const access = microphoneAccess();
+ if (access === 'granted') return true;
+ if (access === 'ask') return systemPreferences.askForMediaAccess('microphone');
+ await shell.openExternal(MICROPHONE_PANE);
+ return false;
+}
diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts
index 53e5f1b..e5e7dcb 100644
--- a/apps/desktop/src/preload/index.ts
+++ b/apps/desktop/src/preload/index.ts
@@ -1,8 +1,8 @@
import { contextBridge, ipcRenderer } from 'electron';
import type { CaptureInsert } from '../main/capture/insert';
import type { CaptureSettings } from '../main/capture/settings';
-import type { OverlayAiFailure } from '../main/overlay';
-import type { CapturePermissions, PermissionKind } from '../main/permissions';
+import type { OverlayAiFailure, OverlayNarration } from '../main/overlay';
+import type { CapturePermissions, MicrophoneAccess, PermissionKind } from '../main/permissions';
interface Region {
x: number;
@@ -17,8 +17,13 @@ const api = {
region: (): Promise<Region> => ipcRenderer.invoke('mimik:capture:region'),
edit: (insert?: CaptureInsert): Promise<void> => ipcRenderer.invoke('mimik:capture:edit', insert),
arm: (insert?: CaptureInsert): Promise<void> => ipcRenderer.invoke('mimik:capture:arm', insert),
- described: (stepId: string, description: string | null, failure: OverlayAiFailure | null): void =>
- ipcRenderer.send('mimik:capture:described', stepId, description, failure),
+ described: (
+ stepId: string,
+ description: string | null,
+ failure: OverlayAiFailure | null,
+ source: 'ai' | 'narration' = 'ai',
+ ): void => ipcRenderer.send('mimik:capture:described', stepId, description, failure, source),
+ narration: (narration: OverlayNarration | null): void => ipcRenderer.send('mimik:capture:narration', narration),
settings: {
get: (): Promise<CaptureSettings> => ipcRenderer.invoke('mimik:capture:settings:get'),
set: (patch: Partial<CaptureSettings>): Promise<CaptureSettings> =>
@@ -35,6 +40,10 @@ const api = {
);
},
},
+ microphone: {
+ get: (): Promise<MicrophoneAccess> => ipcRenderer.invoke('mimik:microphone:get'),
+ request: (): Promise<MicrophoneAccess> => ipcRenderer.invoke('mimik:microphone:request'),
+ },
permissions: {
get: (): Promise<CapturePermissions & { pending: boolean }> => ipcRenderer.invoke('mimik:permissions:get'),
request: (kind: PermissionKind): Promise<void> => ipcRenderer.invoke('mimik:permissions:request', kind),
diff --git a/apps/desktop/src/renderer/capture-host.ts b/apps/desktop/src/renderer/capture-host.ts
index 3619ea9..a0fb653 100644
--- a/apps/desktop/src/renderer/capture-host.ts
+++ b/apps/desktop/src/renderer/capture-host.ts
@@ -1,7 +1,7 @@
import { resolveGuideMetaInputs } from '@mimik/core/capture/ai/guide-description';
import { settleDescriptions } from '@mimik/core/capture/ai/settle-descriptions';
import type { CaptureStepData } from '@mimik/core/capture/sink';
-import { i18n } from '@mimik/core/env';
+import { i18n, localStorage } from '@mimik/core/env';
import { getMostCommonApp } from '@mimik/core/guides/most-common-app';
import {
allScreenshotIds,
@@ -15,14 +15,25 @@ import {
} from '@mimik/core/guides/service';
import { nameGuide } from './ai/name-guide';
import { DesktopCaptureSink } from './capture-sink';
+import { DesktopNarration } from './narration';
-const sink = new DesktopCaptureSink();
+const narration = new DesktopNarration();
+const sink = new DesktopCaptureSink(narration);
allScreenshotIds().then((ids) => window.mimik.screenshots.sweep(ids));
window.mimik.onRequest('mimik:capture:startGuide', (payload) => sink.startGuide(payload === true));
window.mimik.onRequest('mimik:capture:step', (payload) => sink.captureStep(payload as CaptureStepData));
+window.mimik.capture.onCommand(async (command, state, _region, id) => {
+ if (command === 'mic:on' || command === 'mic:off') await localStorage.set({ voiceEnabled: command === 'mic:on' });
+ if (!id) return;
+ if (command === 'start' || command === 'resume' || (command === 'mic:on' && state === 'recording')) {
+ narration.listen(id);
+ } else if (command === 'pause' || command === 'mic:off') narration.cut();
+ else if (command === 'cancel') narration.abort();
+});
+
window.mimik.onRequest('mimik:capture:removeStep', async (payload) => {
const { guideId, stepId } = payload as { guideId: string; stepId: string };
await deleteStep(guideId, stepId);
@@ -31,6 +42,7 @@ window.mimik.onRequest('mimik:capture:removeStep', async (payload) => {
window.mimik.onRequest('mimik:capture:finishGuide', async (payload) => {
const guideId = payload as string;
+ await narration.finish();
const steps = await getStepsForGuide(guideId);
if (steps.length === 0) return true;
const app = getMostCommonApp(steps)?.name;
@@ -49,6 +61,7 @@ window.mimik.onRequest('mimik:capture:finishGuide', async (payload) => {
window.mimik.onRequest('mimik:capture:insertGuide', async (payload) => {
const { guideId, targetGuideId, atIndex } = payload as { guideId: string; targetGuideId: string; atIndex: number };
+ await narration.finish();
await settleDescriptions(guideId);
await createSnapshot(targetGuideId);
await mergeGuideInto(guideId, targetGuideId, atIndex);
diff --git a/apps/desktop/src/renderer/capture-sink.ts b/apps/desktop/src/renderer/capture-sink.ts
index 602ef7c..5034b29 100644
--- a/apps/desktop/src/renderer/capture-sink.ts
+++ b/apps/desktop/src/renderer/capture-sink.ts
@@ -15,6 +15,7 @@ import {
clearStepAiPending,
createGuide,
createStep,
+ getStep,
getStepsForGuide,
saveScreenshot,
updateStepDescription,
@@ -22,6 +23,7 @@ import {
import { screenshotForElement } from '@mimik/core/screenshot/record';
import { credentials } from './ai/credentials';
import { describeStep } from './ai/describe-step';
+import type { DesktopNarration } from './narration';
export type DesktopCaptureStepData = CaptureStepData & {
inputValue?: string;
@@ -29,6 +31,8 @@ export type DesktopCaptureStepData = CaptureStepData & {
};
export class DesktopCaptureSink implements CaptureSink {
+ constructor(private readonly narration?: DesktopNarration) {}
+
async startGuide(staging = false): Promise<string> {
const guide = await createGuide(crypto.randomUUID(), staging);
return guide.id;
@@ -63,7 +67,10 @@ export class DesktopCaptureSink implements CaptureSink {
}
const description = buildFallbackDescription(data.action, meta, data.inputValue);
- const pending = (await credentials()) !== null;
+ const hasKey = (await credentials()) !== null;
+ const narrating = this.narration?.capturing === true;
+ const pending = hasKey || narrating;
+ const timestamp = Date.now();
await createStep({
id: stepId,
@@ -74,7 +81,7 @@ export class DesktopCaptureSink implements CaptureSink {
url: '',
app: meta.app,
window: meta.window,
- timestamp: Date.now(),
+ timestamp,
screenshotId,
elementMeta: meta,
descriptionSource: 'heuristic',
@@ -83,15 +90,15 @@ export class DesktopCaptureSink implements CaptureSink {
});
await addStepToGuide(data.guideId, stepId);
- if (pending) {
- queueDescription(data.guideId, async () => {
- const previous = (await getStepsForGuide(data.guideId)).find((step) => step.index === index - 1);
- const { text, failure } = await describeStep(data.action, meta, previous?.description);
- if (text) await updateStepDescription(stepId, text, 'ai');
- await clearStepAiPending(stepId);
- window.mimik.capture.described(stepId, text, failure);
- });
- }
+ const describe = async () => {
+ const previous = (await getStepsForGuide(data.guideId)).find((step) => step.index === index - 1);
+ const { text, failure } = await describeStep(data.action, meta, previous?.description);
+ await clearStepAiPending(stepId, text ?? undefined);
+ const written = (await getStep(stepId))?.descriptionSource === 'ai';
+ window.mimik.capture.described(stepId, written ? text : null, failure);
+ };
+ if (narrating) this.narration?.step(data.guideId, { stepId, timestamp }, hasKey ? describe : null);
+ else if (hasKey) queueDescription(data.guideId, describe);
return { stepId, title: description, pending };
}
diff --git a/apps/desktop/src/renderer/hooks/use-microphone-access.ts b/apps/desktop/src/renderer/hooks/use-microphone-access.ts
new file mode 100644
index 0000000..6a45839
--- /dev/null
+++ b/apps/desktop/src/renderer/hooks/use-microphone-access.ts
@@ -0,0 +1,26 @@
+import { useEffect, useState } from 'react';
+import type { MicrophoneAccess } from '../../main/permissions';
+
+const POLL_MS = 2000;
+
+export function useMicrophoneAccess() {
+ const [access, setAccess] = useState<MicrophoneAccess | null>(null);
+
+ useEffect(() => {
+ let alive = true;
+ const read = () =>
+ window.mimik.microphone.get().then((found) => {
+ if (alive) setAccess(found);
+ });
+ void read();
+ const timer = setInterval(read, POLL_MS);
+ return () => {
+ alive = false;
+ clearInterval(timer);
+ };
+ }, []);
+
+ const request = () => window.mimik.microphone.request().then(setAccess);
+
+ return { access, request };
+}
diff --git a/apps/desktop/src/renderer/icons.ts b/apps/desktop/src/renderer/icons.ts
index 59ed196..20b4f32 100644
--- a/apps/desktop/src/renderer/icons.ts
+++ b/apps/desktop/src/renderer/icons.ts
@@ -7,6 +7,8 @@ import {
createElement,
Keyboard,
LoaderCircle,
+ Mic,
+ MicOff,
Monitor,
MousePointer2,
Pause,
@@ -14,6 +16,7 @@ import {
Trash2,
TriangleAlert,
Video,
+ X,
} from 'lucide';
const ICONS = {
@@ -31,6 +34,9 @@ const ICONS = {
keyboard: Keyboard,
pointer: MousePointer2,
alert: TriangleAlert,
+ mic: Mic,
+ micOff: MicOff,
+ close: X,
};
export type IconName = keyof typeof ICONS;
diff --git a/apps/desktop/src/renderer/narration.ts b/apps/desktop/src/renderer/narration.ts
new file mode 100644
index 0000000..b8a0a02
--- /dev/null
+++ b/apps/desktop/src/renderer/narration.ts
@@ -0,0 +1,169 @@
+import { queueDescription } from '@mimik/core/capture/ai/description-queue';
+import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@mimik/core/capture/voice/api-key';
+import { MicRecorder } from '@mimik/core/capture/voice/mic-recorder';
+import { narrateRecording, type VoiceRecording } from '@mimik/core/capture/voice/narrate-recording';
+import { narrationUpdates } from '@mimik/core/capture/voice/narration-updates';
+import { partialRecording } from '@mimik/core/capture/voice/partial-recording';
+import { readTranscriptionSettings } from '@mimik/core/capture/voice/read-transcription-settings';
+import type { StepMark } from '@mimik/core/capture/voice/step-windows';
+import { usableRecording } from '@mimik/core/capture/voice/usable-recording';
+import { assetUrl, localStorage } from '@mimik/core/env';
+import {
+ applyNarrationToSteps,
+ clearStepAiPending,
+ findExistingStepIds,
+ getStepsForGuide,
+ saveTranscript,
+} from '@mimik/core/guides/service';
+import { logger } from '@mimik/core/logger';
+
+const SETTLE_MS = 30_000;
+const REFUSED = ['NotAllowedError', 'SecurityError'];
+
+type Describe = () => Promise<void>;
+
+export class DesktopNarration {
+ private live: Promise<MicRecorder | null> | null = null;
+ private mic: MicRecorder | null = null;
+ private guideId: string | null = null;
+ private flushedUpTo = 0;
+ private readonly pending = new Set<Promise<void>>();
+ private readonly deferred = new Map<string, { guideId: string; describe: Describe | null }>();
+
+ async wanted(): Promise<boolean> {
+ const stored = await localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled']);
+ return stored.voiceEnabled === true && hasVoiceApiKey(stored);
+ }
+
+ get capturing(): boolean {
+ return this.mic !== null;
+ }
+
+ listen(guideId: string): void {
+ this.guideId = guideId;
+ this.live ??= this.open();
+ }
+
+ step(guideId: string, mark: StepMark, describe: Describe | null): void {
+ this.deferred.set(mark.stepId, { guideId, describe });
+ const mic = this.mic;
+ this.track(async () => {
+ const full = mic && usableRecording(mic.snapshot());
+ const closesAt = full ? (mark.timestamp - full.audioEpochMs) / 1000 : 0;
+ const slice = full && closesAt > this.flushedUpTo ? partialRecording(full, this.flushedUpTo, closesAt) : null;
+ if (!slice) {
+ this.describeNow(mark.stepId);
+ return;
+ }
+ this.flushedUpTo = closesAt;
+ const narrated = await this.transcribe(guideId, slice, [mark]);
+ if (!narrated.includes(mark.stepId)) this.describeNow(mark.stepId);
+ });
+ }
+
+ cut(): void {
+ const opening = this.live;
+ const guideId = this.guideId;
+ const from = this.flushedUpTo;
+ this.live = null;
+ this.mic = null;
+ this.flushedUpTo = 0;
+ window.mimik.capture.narration(null);
+ if (!opening || !guideId) return;
+ const inflight = [...this.pending];
+ this.track(async () => {
+ const mic = await opening;
+ const audio = mic && usableRecording(mic.stop());
+ window.mimik.capture.narration(null);
+ await Promise.all(inflight);
+ const tail = audio && from > 0 ? partialRecording(audio, from, audio.durationSeconds) : audio;
+ const steps = await getStepsForGuide(guideId);
+ if (tail && steps.length > 0) {
+ await this.transcribe(
+ guideId,
+ tail,
+ steps.map((step) => ({ stepId: step.id, timestamp: step.timestamp })),
+ );
+ }
+ for (const [stepId, entry] of [...this.deferred]) {
+ if (entry.guideId === guideId) this.describeNow(stepId);
+ }
+ });
+ }
+
+ abort(): void {
+ const opening = this.live;
+ this.live = null;
+ this.mic = null;
+ this.flushedUpTo = 0;
+ this.deferred.clear();
+ window.mimik.capture.narration(null);
+ void opening?.then((mic) => {
+ mic?.release();
+ window.mimik.capture.narration(null);
+ });
+ }
+
+ async finish(): Promise<void> {
+ this.cut();
+ await Promise.race([Promise.all(this.pending), new Promise((resolve) => setTimeout(resolve, SETTLE_MS))]);
+ }
+
+ private track(work: () => Promise<void>): void {
+ const running = work()
+ .catch((error) => logger.error('voice: narration could not be applied', error))
+ .finally(() => this.pending.delete(running));
+ this.pending.add(running);
+ }
+
+ private describeNow(stepId: string): void {
+ const entry = this.deferred.get(stepId);
+ if (!entry) return;
+ this.deferred.delete(stepId);
+ if (entry.describe) {
+ queueDescription(entry.guideId, entry.describe);
+ return;
+ }
+ void clearStepAiPending(stepId).then(() => window.mimik.capture.described(stepId, null, null));
+ }
+
+ private async open(): Promise<MicRecorder | null> {
+ if (!(await this.wanted())) return null;
+ const { voiceMicrophoneId } = await localStorage.get(['voiceMicrophoneId']);
+ const mic = new MicRecorder(assetUrl('pcm-processor.js'), {
+ onEpoch: () => undefined,
+ onLevel: (level, speaking) => window.mimik.capture.narration({ level, speaking }),
+ onStreamEnded: () => this.cut(),
+ });
+ try {
+ await mic.start(voiceMicrophoneId || undefined);
+ if (!this.live) {
+ mic.release();
+ return null;
+ }
+ this.mic = mic;
+ window.mimik.capture.narration({ level: 0, speaking: false });
+ return mic;
+ } catch (error) {
+ mic.release();
+ logger.warn('voice: narration could not start, recording without it', error);
+ if (error instanceof Error && REFUSED.includes(error.name)) await localStorage.set({ voiceEnabled: false });
+ return null;
+ }
+ }
+
+ private async transcribe(guideId: string, audio: VoiceRecording, marks: StepMark[]): Promise<string[]> {
+ const settings = await readTranscriptionSettings();
+ if (!settings.apiKey) return [];
+ const result = await narrateRecording(audio, marks, settings);
+ await saveTranscript(guideId, result.transcript);
+ const surviving = await findExistingStepIds(result.descriptions.map((entry) => entry.stepId));
+ const updates = narrationUpdates(result, surviving);
+ await applyNarrationToSteps(updates, result.transcript.epochMs);
+ for (const { stepId, description } of updates) {
+ this.deferred.delete(stepId);
+ window.mimik.capture.described(stepId, description, null, 'narration');
+ }
+ return updates.map((update) => update.stepId);
+ }
+}
diff --git a/apps/desktop/src/renderer/overlay.css b/apps/desktop/src/renderer/overlay.css
index 2a4f8ff..fce3ccf 100644
--- a/apps/desktop/src/renderer/overlay.css
+++ b/apps/desktop/src/renderer/overlay.css
@@ -668,6 +668,11 @@ body.controls #source.basic {
color: var(--lavender);
}
+body.controls #source.voice {
+ background: rgba(5, 150, 105, 0.1);
+ color: var(--success);
+}
+
body.controls #source.ai {
background: #eef2ff;
color: var(--accent);
@@ -814,10 +819,87 @@ body.controls #foot {
}
body.controls #foot button {
- flex-grow: 1;
- flex-basis: 0;
+ flex: none;
justify-content: center;
height: 40px;
+ border-radius: 999px;
+}
+
+body.controls #foot #primary {
+ flex: 1 1 0;
+}
+
+body.controls #foot #mic,
+body.controls #foot #secondary {
+ width: 40px;
+ padding: 0;
+ color: var(--muted);
+}
+
+body.controls #foot #secondary:hover,
+body.controls #foot #mic:hover {
+ color: var(--accent);
+}
+
+body.controls #foot #mic.on {
+ border-color: var(--accent);
+ background: #eef2ff;
+ color: var(--accent);
+}
+
+body.controls #foot #mic.locked {
+ border-style: dashed;
+ opacity: 0.6;
+ cursor: not-allowed;
+}
+
+body.controls #voice {
+ margin-top: 12px;
+ padding: 8px 10px;
+ border-radius: 9px;
+ background: #eef2ff;
+}
+
+body.controls #voiceLine {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ font-size: 11.5px;
+ font-weight: 600;
+ color: var(--deep);
+}
+
+body.controls #voiceHint {
+ margin: 4px 0 0;
+ font-size: 10.5px;
+ line-height: 1.5;
+ color: var(--muted);
+}
+
+body.controls #voiceBars {
+ display: flex;
+ flex-shrink: 0;
+ align-items: center;
+ gap: 2px;
+ height: 14px;
+}
+
+body.controls #voiceBars i {
+ width: 3px;
+ height: 14px;
+ border-radius: 999px;
+ background: var(--lavender);
+ transition: transform 0.3s ease-out;
+}
+
+body.controls #voiceBars.speaking i {
+ background: var(--accent);
+}
+
+@media (prefers-reduced-motion: reduce) {
+ body.controls #voiceBars i {
+ transition: none;
+ }
}
body.controls button {
diff --git a/apps/desktop/src/renderer/overlay/controls.ts b/apps/desktop/src/renderer/overlay/controls.ts
index 676c4cd..cc930c3 100644
--- a/apps/desktop/src/renderer/overlay/controls.ts
+++ b/apps/desktop/src/renderer/overlay/controls.ts
@@ -1,6 +1,7 @@
import { aiFailureNotice } from '@mimik/core/capture/ai/errors';
import { splitAtShortcut } from '@mimik/core/capture/split-at-shortcut';
-import { i18n } from '@mimik/core/env';
+import { hasVoiceApiKey, VOICE_KEY_SETTINGS } from '@mimik/core/capture/voice/api-key';
+import { i18n, localStorage } from '@mimik/core/env';
import type { OverlayView } from '../../main/overlay';
import { icon } from '../icons';
import { cameraMascot } from './camera-mascot';
@@ -22,6 +23,9 @@ const MODE_ID: Record<string, string> = { window: 'window', screen: 'screen', re
const AI_WAIT_MS = 8000;
+const VOICE_BARS = [0.45, 0.75, 1, 0.75, 0.45];
+const VOICE_BAR_FLOOR = 0.14;
+
export function controls(): void {
document.body.className = 'controls';
@@ -78,7 +82,17 @@ export function controls(): void {
const flyerShot = el('img', { alt: '' });
const flyer = el('div', { id: 'flyer' }, flyerShot);
const stage = el('div', { id: 'stage' }, preview, stepTitle, stepMeta, printer, flyer);
- const body = el('div', { id: 'body' }, stage, intro, hint);
+ const voiceBars = el('span', { id: 'voiceBars' });
+ const bars = VOICE_BARS.map(() => voiceBars.appendChild(el('i', {})));
+ const voiceLabel = el('strong', {});
+ const voiceHint = el('p', { id: 'voiceHint' }, i18n.t('voice_orderHint'));
+ const voice = el(
+ 'div',
+ { id: 'voice', role: 'status' },
+ el('div', { id: 'voiceLine' }, voiceBars, voiceLabel),
+ voiceHint,
+ );
+ const body = el('div', { id: 'body' }, stage, intro, hint, voice);
const modeLabel = el('p', { id: 'modeLabel' }, i18n.t('desktop_captureMode'));
const modeRow = el('div', { id: 'modes' });
@@ -102,12 +116,16 @@ export function controls(): void {
const secondary = el('button', { id: 'secondary', type: 'button' });
const primary = el('button', { id: 'primary', type: 'button', className: 'primary' });
- const foot = el('div', { id: 'foot' }, secondary, primary);
+ const mic = el('button', { id: 'mic', type: 'button' });
+ const foot = el('div', { id: 'foot' }, primary, mic, secondary);
document.body.append(head, body, modes, aiNotice, foot);
const report = () => window.mimikOverlay.size(document.body.scrollWidth, document.body.scrollHeight);
+ let micOn = false;
+ let micKeyed = false;
+ let shownMic = '';
let shownIcon = '';
let shownChevron = '';
let shownMode = '';
@@ -146,7 +164,7 @@ export function controls(): void {
};
const render = (view: OverlayView) => {
- const { state, step, busy, progress, print, starting, mode, shortcuts, aiFailure } = view;
+ const { state, step, busy, progress, print, starting, mode, shortcuts, aiFailure, narration } = view;
const armed = state === 'armed';
const recording = state === 'recording';
const paused = state === 'paused';
@@ -209,6 +227,18 @@ export function controls(): void {
}
badge.hidden = paused || collapsed;
+ const micLocked = !micKeyed && !micOn;
+ const micState = micOn ? 'on' : micLocked ? 'locked' : 'off';
+ if (micState !== shownMic) {
+ shownMic = micState;
+ const micLabel = i18n.t(micLocked ? 'voice_needsApiKey' : micOn ? 'voice_turnOff' : 'voice_turnOn');
+ mic.className = micState;
+ mic.title = micLabel;
+ mic.setAttribute('aria-label', micLabel);
+ mic.setAttribute('aria-pressed', String(micOn));
+ mic.replaceChildren(icon(micOn ? 'mic' : 'micOff', 16));
+ }
+
const chevron = collapsed ? 'up' : 'down';
if (chevron !== shownChevron) {
shownChevron = chevron;
@@ -242,8 +272,9 @@ export function controls(): void {
const extra = stepTitle.getBoundingClientRect().height - Number.parseFloat(getComputedStyle(stepTitle).lineHeight);
preview.style.setProperty('--title-extra', `${step ? Math.max(0, extra) : 0}px`);
source.hidden = !step;
- source.textContent = i18n.t(step?.source === 'ai' ? 'stepSource_ai' : 'stepSource_basic');
- source.className = step?.source === 'ai' ? 'ai' : 'basic';
+ const sourceKind = step?.source === 'ai' ? 'ai' : step?.source === 'narration' ? 'voice' : 'basic';
+ source.textContent = i18n.t(`stepSource_${sourceKind}`);
+ source.className = sourceKind;
metaText.textContent =
[step ? i18n.t('export_stepLabel', [String(step.index)]) : '', step?.app ?? ''].filter(Boolean).join(' Β· ') ||
'\u00a0';
@@ -255,7 +286,15 @@ export function controls(): void {
? 'Nothing is recorded while paused.'
: 'Your first click will show up here.';
const key = armed ? shortcuts.startStop : shortcuts.capture;
- hint.hidden = intro.hidden || resting || !key;
+ hint.hidden = intro.hidden || resting || !key || Boolean(narration);
+ voice.hidden = collapsed || !narration;
+ voiceHint.hidden = intro.hidden;
+ voiceBars.classList.toggle('speaking', narration?.speaking === true);
+ voiceLabel.textContent = i18n.t(narration?.speaking ? 'voice_micHearing' : 'voice_micQuiet');
+ bars.forEach((bar, index) => {
+ const scale = Math.max(VOICE_BAR_FLOOR, Math.min(1, (narration?.level ?? 0) * VOICE_BARS[index]));
+ bar.style.transform = `scaleY(${scale})`;
+ });
hintKey.textContent = key ?? '';
hintText.textContent = armed ? ' to start and stop.' : ' to capture without clicking.';
body.hidden = collapsed;
@@ -265,9 +304,10 @@ export function controls(): void {
if (state !== shownIcon) {
shownIcon = state;
- secondary.replaceChildren();
- if (!armed) secondary.append(icon(recording ? 'pause' : 'play'));
- secondary.append(i18n.t(armed ? 'common_close' : recording ? 'desktop_pause' : 'desktop_resume'));
+ const secondaryLabel = i18n.t(armed ? 'common_close' : recording ? 'desktop_pause' : 'desktop_resume');
+ secondary.replaceChildren(icon(armed ? 'close' : recording ? 'pause' : 'play', 16));
+ secondary.title = secondaryLabel;
+ secondary.setAttribute('aria-label', secondaryLabel);
primary.replaceChildren(icon(armed ? 'video' : 'check'));
primary.append(i18n.t(armed ? 'desktop_startButton' : 'desktop_finish'));
}
@@ -324,6 +364,17 @@ export function controls(): void {
primary.addEventListener('click', () => window.mimikOverlay.command(primary.dataset.command ?? 'start'));
secondary.addEventListener('click', () => window.mimikOverlay.command(secondary.dataset.command ?? 'cancel'));
remove.addEventListener('click', () => window.mimikOverlay.command('remove'));
+ mic.addEventListener('click', () => {
+ if (shownMic !== 'locked') window.mimikOverlay.command(micOn ? 'mic:off' : 'mic:on');
+ });
+ const readMic = () =>
+ localStorage.get([...VOICE_KEY_SETTINGS, 'voiceEnabled']).then((stored) => {
+ micOn = stored.voiceEnabled === true;
+ micKeyed = hasVoiceApiKey(stored);
+ paint();
+ });
+ void readMic();
+ window.addEventListener('storage', () => void readMic());
collapse.addEventListener('click', () => {
collapsed = !collapsed;
paint();
diff --git a/apps/desktop/src/renderer/settings/SettingsPanel.tsx b/apps/desktop/src/renderer/settings/SettingsPanel.tsx
index 37d093c..57329c7 100644
--- a/apps/desktop/src/renderer/settings/SettingsPanel.tsx
+++ b/apps/desktop/src/renderer/settings/SettingsPanel.tsx
@@ -5,6 +5,8 @@ import {
AiSettings,
ApiKeysSettings,
BrandingSettings,
+ MicrophoneAccessRow,
+ NarrationSettings,
Segmented,
Switch,
useApiKeys,
@@ -24,6 +26,8 @@ import {
} from 'lucide-react';
import { useEffect, useState } from 'react';
import type { CaptureSettings, CaptureShortcuts } from '../../main/capture/settings';
+import { useMicrophoneAccess } from '../hooks/use-microphone-access';
+import { requestMicrophoneAccess } from '../lib/request-microphone-access';
import { Card } from './Card';
import { GeneralSettings } from './GeneralSettings';
import { REOPEN_SETTINGS } from './lib/reopen-settings';
@@ -61,6 +65,7 @@ export function SettingsPanel() {
return reopened ?? 'general';
});
const keys = useApiKeys();
+ const microphone = useMicrophoneAccess();
const [settings, setSettings] = useState<CaptureSettings | null>(null);
useEffect(() => {
@@ -100,6 +105,28 @@ export function SettingsPanel() {
{section === 'ai' && (
<>
<AiSettings keys={keys} onOpenKeys={() => setSection('keys')} />
+ <NarrationSettings
+ keys={keys}
+ onOpenKeys={() => setSection('keys')}
+ onRequestAccess={requestMicrophoneAccess}
+ access={
+ microphone.access && microphone.access !== 'granted' ? (
+ <MicrophoneAccessRow
+ refused={microphone.access === 'denied'}
+ hint={i18n.t(
+ microphone.access === 'denied' ? 'desktop_micAccessRefused' : 'desktop_micAccessAsk',
+ )}
+ action={i18n.t(
+ microphone.access === 'denied' ? 'desktop_micAccessOpen' : 'settings_microphoneAccessAllow',
+ )}
+ external={microphone.access === 'denied'}
+ onRequest={() => void microphone.request()}
+ />
+ ) : null
+ }
+ microphoneDisabled={microphone.access !== 'granted'}
+ liveMeter
+ />
<VoiceoverSettings keys={keys.keys} listVoices={fetchVoices} onOpenKeys={() => setSection('keys')} />
</>
)}