Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix(export): stop shipping a silent video that reports success

davbiod6a1b5f75db0
diff --git a/src/core/export/__tests__/video-export-encode.test.ts b/src/core/export/__tests__/video-export-encode.test.ts
index fda3167..45ea2a1 100644
--- a/src/core/export/__tests__/video-export-encode.test.ts
+++ b/src/core/export/__tests__/video-export-encode.test.ts
@@ -7,7 +7,7 @@ const rec = vi.hoisted(() => ({
calls: [] as { m: string; a: unknown[] }[],
added: [] as { at: number; dur: number }[],
closed: 0,
- container: 'mp4' as 'mp4' | 'webm' | null,
+ containers: ['mp4'] as ('mp4' | 'webm')[],
probed: [] as string[],
started: 0,
finalized: 0,
@@ -18,7 +18,7 @@ const rec = vi.hoisted(() => ({
}));
const voice = vi.hoisted(() => ({
- codec: 'aac' as 'aac' | 'opus' | null,
+ codecs: ['aac', 'opus'] as ('aac' | 'opus')[],
settings: { voiceoverProvider: 'openai', voiceoverApiKeys: { openai: 'sk_test' } } as Record<string, unknown>,
clips: new Map<number, AudioBuffer>(),
}));
@@ -102,9 +102,9 @@ vi.mock('@/core/export/video-support', async () => {
const actual = await vi.importActual<typeof import('@/core/export/video-support')>('@/core/export/video-support');
return {
...actual,
- pickContainer: vi.fn(async (r = '720p') => {
+ availableContainers: vi.fn(async (r = '720p') => {
rec.probed.push(r);
- return rec.container;
+ return [...rec.containers];
}),
};
});
@@ -117,7 +117,13 @@ vi.mock('@/core/export/voiceover/audio', async () => {
const actual = await vi.importActual<typeof import('@/core/export/voiceover/audio')>('@/core/export/voiceover/audio');
return {
...actual,
- pickVoiceCodec: vi.fn(async () => voice.codec),
+ pickVoiceContainer: vi.fn(async (containers: ('mp4' | 'webm')[]) => {
+ for (const container of containers) {
+ const codec = container === 'mp4' ? ('aac' as const) : ('opus' as const);
+ if (voice.codecs.includes(codec)) return { container, codec };
+ }
+ return null;
+ }),
};
});
@@ -189,7 +195,7 @@ beforeEach(() => {
rec.calls = [];
rec.added = [];
rec.closed = 0;
- rec.container = 'mp4';
+ rec.containers = ['mp4'];
rec.probed = [];
rec.started = 0;
rec.finalized = 0;
@@ -197,7 +203,7 @@ beforeEach(() => {
rec.buffer = true;
rec.audio = [];
rec.audioTracks = 0;
- voice.codec = 'aac';
+ voice.codecs = ['aac', 'opus'];
voice.settings = { voiceoverProvider: 'openai', voiceoverApiKeys: { openai: 'sk_test' } };
voice.clips = new Map();
renderVoiceover.mockReset();
@@ -245,7 +251,7 @@ describe('exportGuideAsVideo guards', () => {
});
it('refuses when the browser cannot encode at all', async () => {
- rec.container = null;
+ rec.containers = [];
const steps = [makeStep(0)];
await expect(exportGuideAsVideo(guide, steps, shotsFor(steps), opts())).rejects.toThrow(/cannot encode/i);
});
@@ -270,7 +276,7 @@ describe('exportGuideAsVideo output', () => {
});
it('falls back to webm when mp4 is unavailable', async () => {
- rec.container = 'webm';
+ rec.containers = ['webm'];
const steps = [makeStep(0)];
const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false }));
@@ -533,7 +539,8 @@ describe('exportGuideAsVideo voiceover', () => {
const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
expect(result.blob).toBeInstanceOf(Blob);
- expect(result.voiceoverError).toMatch(/401/);
+ expect(result.voiceoverError?.reason).toBe('failed');
+ expect(result.voiceoverError?.detail).toMatch(/401/);
expect(rec.audioTracks).toBe(0);
expect(rec.added).toHaveLength(157 * 2 - 10);
});
@@ -556,24 +563,77 @@ describe('exportGuideAsVideo voiceover', () => {
expect(rec.audioTracks).toBe(1);
});
- it('falls back to a silent video when no key is stored', async () => {
+ it('falls back to a silent video when no key is stored, and says so', async () => {
voice.settings = {};
voice.clips = new Map([[0, clip(9)]]);
const steps = [makeStep(0), makeStep(1)];
- await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+ const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
expect(rec.audioTracks).toBe(0);
expect(rec.added).toHaveLength(157 * 2 - 10);
+ expect(result.voiceoverError).toEqual({ reason: 'noKey' });
});
- it('falls back to a silent video when the browser cannot encode audio', async () => {
- voice.codec = null;
+ it('falls back to a silent video when the browser cannot encode audio, and says so', async () => {
+ voice.codecs = [];
voice.clips = new Map([[0, clip(9)]]);
const steps = [makeStep(0)];
- await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+ const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
expect(rec.audioTracks).toBe(0);
expect(rec.added).toHaveLength(157);
+ expect(result.voiceoverError).toEqual({ reason: 'noAudioCodec' });
+ });
+
+ it('reports the skip when the guide has nothing to read aloud', async () => {
+ voice.clips = new Map([[0, clip(9)]]);
+ const steps = [{ ...makeStep(0), description: '' }];
+ const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ expect(rec.audioTracks).toBe(0);
+ expect(result.voiceoverError).toEqual({ reason: 'nothingToSay' });
+ });
+
+ it('reports no error at all when narration lands', async () => {
+ voice.clips = new Map([[0, clip(2)]]);
+ const steps = [makeStep(0)];
+ const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ expect(rec.audioTracks).toBe(1);
+ expect(result.voiceoverError).toBeUndefined();
+ });
+
+ it('prefers the container whose audio codec this browser can encode', async () => {
+ // Chromium on Linux: H.264 and VP9 both encode, but AAC does not.
+ rec.containers = ['mp4', 'webm'];
+ voice.codecs = ['opus'];
+ voice.clips = new Map([[0, clip(2)]]);
+ const steps = [makeStep(0)];
+ const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ expect(result.extension).toBe('webm');
+ expect(rec.audioTracks).toBe(1);
+ expect(result.voiceoverError).toBeUndefined();
+ });
+
+ it('keeps mp4 when its audio codec is available too', async () => {
+ rec.containers = ['mp4', 'webm'];
+ voice.clips = new Map([[0, clip(2)]]);
+ const steps = [makeStep(0)];
+ const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ expect(result.extension).toBe('mp4');
+ expect(rec.audioTracks).toBe(1);
+ });
+
+ it('leaves a silent export on the preferred container', async () => {
+ rec.containers = ['mp4', 'webm'];
+ voice.codecs = ['opus'];
+ const steps = [makeStep(0)];
+ const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false }));
+
+ expect(result.extension).toBe('mp4');
+ expect(rec.audioTracks).toBe(0);
});
it('stretches the chapter marks to match the narrated timeline', async () => {
diff --git a/src/core/export/__tests__/video-support.test.ts b/src/core/export/__tests__/video-support.test.ts
index 1d509bf..aa750cf 100644
--- a/src/core/export/__tests__/video-support.test.ts
+++ b/src/core/export/__tests__/video-support.test.ts
@@ -48,6 +48,33 @@ describe('codec strings', () => {
});
});
+describe('availableContainers', () => {
+ it('lists both containers when both video codecs encode, so audio support can break the tie', async () => {
+ stubEncoder(() => true);
+ const { availableContainers } = await freshModule();
+ expect(await availableContainers()).toEqual(['mp4', 'webm']);
+ });
+
+ it('still probes vp9 when avc encodes, since the caller may need the webm option', async () => {
+ const probed = stubEncoder(() => true);
+ const { availableContainers } = await freshModule();
+ await availableContainers();
+ expect(probed.some((c) => c.codec === VP9_CODEC)).toBe(true);
+ });
+
+ it('lists only what encodes', async () => {
+ stubEncoder((codec) => codec === VP9_CODEC);
+ const { availableContainers } = await freshModule();
+ expect(await availableContainers()).toEqual(['webm']);
+ });
+
+ it('is empty when nothing encodes', async () => {
+ stubEncoder(() => false);
+ const { availableContainers } = await freshModule();
+ expect(await availableContainers()).toEqual([]);
+ });
+});
+
describe('pickContainer', () => {
it('returns mp4 when the High Profile string encodes', async () => {
stubEncoder((codec) => codec === AVC_CODEC);
diff --git a/src/core/export/video-export.ts b/src/core/export/video-export.ts
index 09ba1f5..7c4787b 100644
--- a/src/core/export/video-export.ts
+++ b/src/core/export/video-export.ts
@@ -5,12 +5,12 @@ import type { ExportOptions } from '@/core/export/options';
import { loadExportOptions } from '@/core/export/options';
import { extractDomain, formatDate } from '@/core/export/utils';
import {
+ availableContainers,
COVER_SECONDS,
FPS,
FRAME_FILL,
FRAME_HEIGHT,
FRAME_WIDTH,
- pickContainer,
RESOLUTION_SPECS,
STEP_SECONDS,
STEP_ZOOM_TRANSITION_SEC,
@@ -716,11 +716,23 @@ export function stepKind(step: Step): StepKind {
return 'click';
}
+/**
+ * Why narration did not make it into the video. `failed` carries the thrown message in `detail`;
+ * the other three are the deliberate skips, which used to return silently and leave the export
+ * panel reporting a narrated video that had no audio in it.
+ */
+export type VoiceoverSkipReason = 'failed' | 'noAudioCodec' | 'noKey' | 'nothingToSay';
+
+export interface VoiceoverSkip {
+ reason: VoiceoverSkipReason;
+ detail?: string;
+}
+
export interface VideoExportResult {
blob: Blob;
extension: string;
chapters: VideoChapter[];
- voiceoverError?: string;
+ voiceoverError?: VoiceoverSkip;
}
export function videoChapters(frames: Step[], cover: boolean, fps = FPS, timeline?: VideoTimeline): VideoChapter[] {
@@ -860,24 +872,27 @@ export async function composeGuideFrames(
}
interface PreparedVoiceover {
+ container: VideoContainer;
codec: 'aac' | 'opus';
clips: Map<number, AudioBuffer>;
timeline: VideoTimeline;
}
+type NarrationOutcome = { voice: PreparedVoiceover; error?: undefined } | { voice: null; error: VoiceoverSkip };
+
async function narrateOrSkip(
guide: Guide,
frames: Step[],
cover: boolean,
- container: VideoContainer,
+ containers: VideoContainer[],
controls: VideoExportControls,
-): Promise<{ voice: PreparedVoiceover | null; error?: string }> {
+): Promise<NarrationOutcome> {
try {
- return { voice: await prepareVoiceover(guide, frames, cover, container, controls) };
+ return await prepareVoiceover(guide, frames, cover, containers, controls);
} catch (error) {
if (error instanceof DOMException && error.name === 'AbortError') throw error;
logger.error('[voiceover] narration failed, exporting a silent video', error);
- return { voice: null, error: error instanceof Error ? error.message : String(error) };
+ return { voice: null, error: { reason: 'failed', detail: error instanceof Error ? error.message : String(error) } };
}
}
@@ -885,30 +900,41 @@ async function prepareVoiceover(
guide: Guide,
frames: Step[],
cover: boolean,
- container: VideoContainer,
+ containers: VideoContainer[],
controls: VideoExportControls,
-): Promise<PreparedVoiceover | null> {
- const [{ pickVoiceCodec }, { resolveVoiceoverConfig, VOICEOVER_SETTINGS }, { voiceoverScript }, { renderVoiceover }] =
- await Promise.all([
- import('./voiceover/audio'),
- import('./voiceover/config'),
- import('./voiceover/script'),
- import('./voiceover/render'),
- ]);
+): Promise<NarrationOutcome> {
+ const [
+ { pickVoiceContainer },
+ { resolveVoiceoverConfig, VOICEOVER_SETTINGS },
+ { voiceoverProvider },
+ { voiceoverScript },
+ { renderVoiceover },
+ ] = await Promise.all([
+ import('./voiceover/audio'),
+ import('./voiceover/config'),
+ import('./voiceover/providers'),
+ import('./voiceover/script'),
+ import('./voiceover/render'),
+ ]);
const config = resolveVoiceoverConfig(await localStorage.get([...VOICEOVER_SETTINGS]));
if (!config.apiKey) {
- logger.warn('[voiceover] skipped: no ElevenLabs API key');
- return null;
+ // logger.warn is compiled out of a built extension, so every skip logs at error level: a
+ // silent skip that neither reaches the UI nor the console is undebuggable.
+ logger.error('[voiceover] skipped: no', voiceoverProvider(config.provider).label, 'API key');
+ return { voice: null, error: { reason: 'noKey' } };
}
const segments = voiceoverScript(guide, frames, cover);
- if (segments.length === 0) return null;
+ if (segments.length === 0) {
+ logger.error('[voiceover] skipped: this guide has no text to read aloud');
+ return { voice: null, error: { reason: 'nothingToSay' } };
+ }
- const codec = await pickVoiceCodec(container);
- if (!codec) {
- logger.warn('[voiceover] skipped: this browser cannot encode', container === 'mp4' ? 'AAC' : 'Opus');
- return null;
+ const picked = await pickVoiceContainer(containers);
+ if (!picked) {
+ logger.error('[voiceover] skipped: this browser cannot encode audio for', containers.join('/') || 'any container');
+ return { voice: null, error: { reason: 'noAudioCodec' } };
}
const clips = await renderVoiceover(segments, config, {
@@ -917,7 +943,14 @@ async function prepareVoiceover(
});
const seconds = new Map(Array.from(clips, ([index, clip]) => [index, clip.duration]));
- return { codec, clips, timeline: voiceTimeline(frames.length, seconds) };
+ return {
+ voice: {
+ container: picked.container,
+ codec: picked.codec,
+ clips,
+ timeline: voiceTimeline(frames.length, seconds),
+ },
+ };
}
export function voiceoverPlacement(
@@ -959,17 +992,20 @@ export async function exportGuideAsVideo(
]);
const requested = RESOLUTION_SPECS[options.resolution] ? options.resolution : '720p';
- const preferred = await pickContainer(requested);
- const resolution = preferred ? requested : '720p';
- const container = preferred ?? (await pickContainer('720p'));
- if (!container) throw new Error('This browser cannot encode video');
- const mp4 = container === 'mp4';
+ const preferred = await availableContainers(requested);
+ const resolution = preferred.length > 0 ? requested : '720p';
+ const containers = preferred.length > 0 ? preferred : await availableContainers('720p');
+ if (containers.length === 0) throw new Error('This browser cannot encode video');
const spec = RESOLUTION_SPECS[resolution];
- const narration = options.voiceover
- ? await narrateOrSkip(guide, frames, Boolean(options.cover), container, controls)
+ // Narration runs first because it also settles the container: a browser that encodes H.264 but
+ // not AAC has to fall back to WebM/Opus rather than mp4, which it could not write audio into.
+ const narration: NarrationOutcome | { voice: null; error?: undefined } = options.voiceover
+ ? await narrateOrSkip(guide, frames, Boolean(options.cover), containers, controls)
: { voice: null };
const voice = narration.voice;
+ const container = voice?.container ?? containers[0];
+ const mp4 = container === 'mp4';
const timeline = voice?.timeline ?? uniformTimeline(frames.length);
const canvas = document.createElement('canvas');
diff --git a/src/core/export/video-support.ts b/src/core/export/video-support.ts
index 4f8f632..f95b190 100644
--- a/src/core/export/video-support.ts
+++ b/src/core/export/video-support.ts
@@ -78,15 +78,21 @@ async function encodes(codec: string, width: number, height: number): Promise<bo
}
}
-async function probe(spec: ResolutionSpec): Promise<VideoContainer | null> {
- if (typeof VideoEncoder === 'undefined') return null;
- if (await encodes(spec.avc, spec.width, spec.height)) return 'mp4';
- return (await encodes(spec.vp9, spec.width, spec.height)) ? 'webm' : null;
+async function probe(spec: ResolutionSpec): Promise<VideoContainer[]> {
+ if (typeof VideoEncoder === 'undefined') return [];
+ const [avc, vp9] = await Promise.all([
+ encodes(spec.avc, spec.width, spec.height),
+ encodes(spec.vp9, spec.width, spec.height),
+ ]);
+ const containers: VideoContainer[] = [];
+ if (avc) containers.push('mp4');
+ if (vp9) containers.push('webm');
+ return containers;
}
-const pending = new Map<VideoResolution, Promise<VideoContainer | null>>();
+const pending = new Map<VideoResolution, Promise<VideoContainer[]>>();
-export function pickContainer(resolution: VideoResolution = '720p'): Promise<VideoContainer | null> {
+export function availableContainers(resolution: VideoResolution = '720p'): Promise<VideoContainer[]> {
const cached = pending.get(resolution);
if (cached) return cached;
const next = probe(RESOLUTION_SPECS[resolution]);
@@ -94,6 +100,10 @@ export function pickContainer(resolution: VideoResolution = '720p'): Promise<Vid
return next;
}
+export async function pickContainer(resolution: VideoResolution = '720p'): Promise<VideoContainer | null> {
+ return (await availableContainers(resolution))[0] ?? null;
+}
+
export async function canExportVideo(): Promise<boolean> {
- return (await pickContainer()) !== null;
+ return (await availableContainers()).length > 0;
}
diff --git a/src/core/export/voiceover/audio.ts b/src/core/export/voiceover/audio.ts
index ce1bc6d..aac6e08 100644
--- a/src/core/export/voiceover/audio.ts
+++ b/src/core/export/voiceover/audio.ts
@@ -1,3 +1,5 @@
+import type { VideoContainer } from '@/core/export/video-support';
+
export const VOICE_SAMPLE_RATE = 44100;
export type VoiceCodec = 'aac' | 'opus';
@@ -72,8 +74,23 @@ export async function writeVoiceTrack(
}
}
-export async function pickVoiceCodec(container: 'mp4' | 'webm'): Promise<VoiceCodec | null> {
+export async function pickVoiceCodec(container: VideoContainer): Promise<VoiceCodec | null> {
const { canEncodeAudio } = await import('mediabunny');
const codec: VoiceCodec = container === 'mp4' ? 'aac' : 'opus';
return (await canEncodeAudio(codec, { numberOfChannels: 1, sampleRate: VOICE_SAMPLE_RATE })) ? codec : null;
}
+
+/**
+ * Picks the first container this browser can encode *audio* into. Video support alone is not
+ * enough: Chromium on Linux encodes H.264 but not AAC, so preferring mp4 there lands the export in
+ * a container it cannot write the voice track into, while WebM/Opus would have worked.
+ */
+export async function pickVoiceContainer(
+ containers: VideoContainer[],
+): Promise<{ container: VideoContainer; codec: VoiceCodec } | null> {
+ for (const container of containers) {
+ const codec = await pickVoiceCodec(container);
+ if (codec) return { container, codec };
+ }
+ return null;
+}
diff --git a/src/ui/fullview/ExportPreviewModal.tsx b/src/ui/fullview/ExportPreviewModal.tsx
index c02e2a9..ffd010a 100644
--- a/src/ui/fullview/ExportPreviewModal.tsx
+++ b/src/ui/fullview/ExportPreviewModal.tsx
@@ -15,7 +15,7 @@ import {
} from '@/core/export/options';
import { exportGuideAsPDF } from '@/core/export/pdf-export';
import { paginatePreview, withPreviewStyles } from '@/core/export/preview';
-import type { VideoChapter } from '@/core/export/video-export';
+import type { VideoChapter, VoiceoverSkip } from '@/core/export/video-export';
import { COVER_SECONDS, canExportVideo, STEP_SECONDS } from '@/core/export/video-support';
import { hasVoiceoverKey, VOICEOVER_SETTINGS } from '@/core/export/voiceover/config';
import type { Guide, Screenshot, Step } from '@/core/guides/types';
@@ -30,6 +30,13 @@ const VIDEO_AUTOPLAY_STEP_LIMIT = 25;
const IMAGE_SCALES: ImageScale[] = ['small', 'medium', 'large'];
+const VOICEOVER_SKIP_MESSAGES = {
+ failed: 'exportPreview.voiceoverFailed',
+ noAudioCodec: 'exportPreview.voiceoverNoAudioCodec',
+ noKey: 'exportPreview.voiceoverNoKeySkip',
+ nothingToSay: 'exportPreview.voiceoverNothingToSay',
+} as const satisfies Record<VoiceoverSkip['reason'], string>;
+
interface ExportPreviewModalProps {
open: boolean;
onOpenChange: (open: boolean) => void;
@@ -59,7 +66,7 @@ export default function ExportPreviewModal({ open, onOpenChange, guide, steps, s
const [voiceoverReady, setVoiceoverReady] = useState(false);
const [voiceProgress, setVoiceProgress] = useState<{ done: number; total: number } | null>(null);
const [, setNarratedSeconds] = useState<number | null>(null);
- const [voiceoverError, setVoiceoverError] = useState<string | null>(null);
+ const [voiceoverError, setVoiceoverError] = useState<VoiceoverSkip | null>(null);
useEffect(() => {
if (open) loadExportOptions().then(setOptions);
@@ -204,7 +211,11 @@ export default function ExportPreviewModal({ open, onOpenChange, guide, steps, s
downloadAbort.current = controller;
setDownloadProgress(0);
const { exportGuideAsVideo } = await import('@/core/export/video-export');
- const { blob, extension } = await exportGuideAsVideo(
+ const {
+ blob,
+ extension,
+ voiceoverError: failed,
+ } = await exportGuideAsVideo(
guide,
steps,
screenshots,
@@ -215,6 +226,9 @@ export default function ExportPreviewModal({ open, onOpenChange, guide, steps, s
onVoiceProgress: (done, total) => setVoiceProgress(done < total ? { done, total } : null),
},
);
+ // A download that never opened the video tab must still say the narration was dropped:
+ // the guide can be exported to video straight from the format list.
+ setVoiceoverError(failed ?? null);
downloadBlob(blob, safeFilename(guide.title, extension));
} else if (format === 'bundle') {
const { exportGuideAsBundle } = await import('@/core/transfer/bundle');
@@ -386,7 +400,7 @@ export default function ExportPreviewModal({ open, onOpenChange, guide, steps, s
className="mt-1.5 rounded-lg px-2.5 py-2 text-[10px] leading-snug text-destructive bg-destructive/10"
role="alert"
>
- {i18n.t('exportPreview.voiceoverFailed')}
+ {i18n.t(VOICEOVER_SKIP_MESSAGES[voiceoverError.reason])}
</div>
)}
</div>