Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(export): hold each step until its narration ends

davbio142da0087f75
diff --git a/src/core/export/__tests__/options.test.ts b/src/core/export/__tests__/options.test.ts
index 79b371a..149833c 100644
--- a/src/core/export/__tests__/options.test.ts
+++ b/src/core/export/__tests__/options.test.ts
@@ -32,6 +32,7 @@ describe('normaliseExportOptions', () => {
gifQuality: DEFAULT_EXPORT_OPTIONS.gifQuality,
bundleStripInputs: DEFAULT_EXPORT_OPTIONS.bundleStripInputs,
bundleUrls: DEFAULT_EXPORT_OPTIONS.bundleUrls,
+ voiceover: DEFAULT_EXPORT_OPTIONS.voiceover,
});
});
diff --git a/src/core/export/__tests__/video-export-encode.test.ts b/src/core/export/__tests__/video-export-encode.test.ts
index c24dd90..fda3167 100644
--- a/src/core/export/__tests__/video-export-encode.test.ts
+++ b/src/core/export/__tests__/video-export-encode.test.ts
@@ -13,6 +13,14 @@ const rec = vi.hoisted(() => ({
finalized: 0,
cancelled: 0,
buffer: true,
+ audio: [] as { length: number }[],
+ audioTracks: 0,
+}));
+
+const voice = vi.hoisted(() => ({
+ codec: 'aac' as 'aac' | 'opus' | null,
+ settings: { voiceoverProvider: 'openai', voiceoverApiKeys: { openai: 'sk_test' } } as Record<string, unknown>,
+ clips: new Map<number, AudioBuffer>(),
}));
const branding = vi.hoisted(() => ({
@@ -51,6 +59,12 @@ function fakeCtx() {
vi.mock('mediabunny', () => ({
QUALITY_HIGH: 'high',
+ QUALITY_MEDIUM: 'medium',
+ AudioBufferSource: class {
+ async add(buffer: { length: number }) {
+ rec.audio.push(buffer);
+ }
+ },
Mp4OutputFormat: class {},
WebMOutputFormat: class {},
BufferTarget: class {
@@ -69,6 +83,9 @@ vi.mock('mediabunny', () => ({
this.target = o.target;
}
addVideoTrack() {}
+ addAudioTrack() {
+ rec.audioTracks += 1;
+ }
async start() {
rec.started += 1;
}
@@ -92,6 +109,22 @@ vi.mock('@/core/export/video-support', async () => {
};
});
+const renderVoiceover = vi.hoisted(() => vi.fn());
+
+vi.mock('@/core/export/voiceover/render', () => ({ renderVoiceover }));
+
+vi.mock('@/core/export/voiceover/audio', async () => {
+ const actual = await vi.importActual<typeof import('@/core/export/voiceover/audio')>('@/core/export/voiceover/audio');
+ return {
+ ...actual,
+ pickVoiceCodec: vi.fn(async () => voice.codec),
+ };
+});
+
+vi.mock('@/lib/browser-api', () => ({
+ localStorage: { get: vi.fn(async () => voice.settings), set: vi.fn(async () => undefined) },
+}));
+
vi.mock('@/core/screenshot/render', () => ({
renderScreenshot: vi.fn(async () => new Blob(['webp'])),
}));
@@ -103,7 +136,10 @@ vi.mock('@/core/export/branding', async () => {
const { exportGuideAsVideo } = await import('@/core/export/video-export');
const { renderScreenshot } = await import('@/core/screenshot/render');
-const { FPS } = await import('@/core/export/video-support');
+const { COVER_SECONDS, FPS, STEP_SECONDS, VOICE_LEAD_SEC, VOICE_TAIL_SEC } = await import(
+ '@/core/export/video-support'
+);
+const { toFrames } = await import('@/core/export/video-export');
const guide: Guide = {
id: 'g1',
@@ -159,6 +195,13 @@ beforeEach(() => {
rec.finalized = 0;
rec.cancelled = 0;
rec.buffer = true;
+ rec.audio = [];
+ rec.audioTracks = 0;
+ voice.codec = 'aac';
+ voice.settings = { voiceoverProvider: 'openai', voiceoverApiKeys: { openai: 'sk_test' } };
+ voice.clips = new Map();
+ renderVoiceover.mockReset();
+ renderVoiceover.mockImplementation(async () => voice.clips);
branding.value = { logo: null, footer: '', attribution: false, accent: '#4F46E5', custom: false };
vi.mocked(renderScreenshot).mockClear();
vi.mocked(renderScreenshot).mockResolvedValue(new Blob(['webp']));
@@ -173,6 +216,14 @@ beforeEach(() => {
}
}
vi.stubGlobal('OffscreenCanvas', FakeOffscreen);
+ vi.stubGlobal(
+ 'OfflineAudioContext',
+ class {
+ createBuffer(_channels: number, length: number) {
+ return { length };
+ }
+ },
+ );
vi.stubGlobal(
'createImageBitmap',
vi.fn(async () => ({
@@ -429,3 +480,113 @@ describe('exportGuideAsVideo layers', () => {
expect(alphas.some((a) => a > 0 && a < 1)).toBe(true);
});
});
+
+describe('exportGuideAsVideo voiceover', () => {
+ const clip = (seconds: number) => ({ duration: seconds, length: Math.round(seconds * 44100) }) as AudioBuffer;
+
+ it('stays silent and on the structural timeline when the option is off', async () => {
+ const steps = [makeStep(0), makeStep(1)];
+ await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: false }));
+
+ expect(rec.audioTracks).toBe(0);
+ expect(rec.added).toHaveLength(157 * 2 - 10);
+ });
+
+ it('holds a narrated step until its clip ends', async () => {
+ voice.clips = new Map([[0, clip(9)]]);
+ const steps = [makeStep(0), makeStep(1)];
+ await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ const first = toFrames(VOICE_LEAD_SEC + 9 + VOICE_TAIL_SEC);
+ expect(rec.audioTracks).toBe(1);
+ expect(rec.added).toHaveLength(first + 157 - 10);
+ });
+
+ it('lays each clip down at its own step, after the lead-in', async () => {
+ const coverClip = clip(1);
+ const secondStepClip = clip(2);
+ voice.clips = new Map([
+ [-1, coverClip],
+ [1, secondStepClip],
+ ]);
+ const steps = [makeStep(0), makeStep(1)];
+ await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: true, voiceover: true }));
+
+ const startsAt = (buffer: AudioBuffer) => {
+ let cursor = 0;
+ for (const written of rec.audio) {
+ if (written === (buffer as unknown as { length: number })) return cursor / 44100;
+ cursor += written.length;
+ }
+ return null;
+ };
+
+ const stepTwoOpens = COVER_SECONDS + (toFrames(STEP_SECONDS) - 10) / FPS;
+ expect(startsAt(coverClip)).toBeCloseTo(VOICE_LEAD_SEC, 4);
+ expect(startsAt(secondStepClip)).toBeCloseTo(stepTwoOpens + VOICE_LEAD_SEC, 4);
+ });
+
+ it('exports a silent video when narration fails, instead of failing the export', async () => {
+ renderVoiceover.mockRejectedValueOnce(new Error('ElevenLabs rejected the API key (401)'));
+ const steps = [makeStep(0), makeStep(1)];
+
+ const result = await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ expect(result.blob).toBeInstanceOf(Blob);
+ expect(result.voiceoverError).toMatch(/401/);
+ expect(rec.audioTracks).toBe(0);
+ expect(rec.added).toHaveLength(157 * 2 - 10);
+ });
+
+ it('still aborts the export when the user cancels during narration', async () => {
+ renderVoiceover.mockRejectedValueOnce(new DOMException('Voiceover was aborted', 'AbortError'));
+ const steps = [makeStep(0)];
+
+ await expect(
+ exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true })),
+ ).rejects.toThrow(/aborted/i);
+ });
+
+ it('narrates on the AI key alone, with no voice-over key of its own', async () => {
+ voice.settings = { voiceoverProvider: 'openai', aiProvider: 'openai', aiApiKeys: { openai: 'sk-ai' } };
+ voice.clips = new Map([[0, clip(9)]]);
+ const steps = [makeStep(0), makeStep(1)];
+ await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ expect(rec.audioTracks).toBe(1);
+ });
+
+ it('falls back to a silent video when no key is stored', async () => {
+ voice.settings = {};
+ voice.clips = new Map([[0, clip(9)]]);
+ const steps = [makeStep(0), makeStep(1)];
+ await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ expect(rec.audioTracks).toBe(0);
+ expect(rec.added).toHaveLength(157 * 2 - 10);
+ });
+
+ it('falls back to a silent video when the browser cannot encode audio', async () => {
+ voice.codec = null;
+ voice.clips = new Map([[0, clip(9)]]);
+ const steps = [makeStep(0)];
+ await exportGuideAsVideo(guide, steps, shotsFor(steps), opts({ cover: false, voiceover: true }));
+
+ expect(rec.audioTracks).toBe(0);
+ expect(rec.added).toHaveLength(157);
+ });
+
+ it('stretches the chapter marks to match the narrated timeline', async () => {
+ voice.clips = new Map([[0, clip(9)]]);
+ const steps = [makeStep(0), makeStep(1)];
+ const { chapters } = await exportGuideAsVideo(
+ guide,
+ steps,
+ shotsFor(steps),
+ opts({ cover: false, voiceover: true }),
+ );
+
+ expect(chapters[1].start).toBeCloseTo((toFrames(VOICE_LEAD_SEC + 9 + VOICE_TAIL_SEC) - 10) / FPS, 6);
+ expect(chapters[0].end).toBeCloseTo(chapters[1].start, 6);
+ });
+});
diff --git a/src/core/export/__tests__/video-export.test.ts b/src/core/export/__tests__/video-export.test.ts
index 0ed7355..e1aad7f 100644
--- a/src/core/export/__tests__/video-export.test.ts
+++ b/src/core/export/__tests__/video-export.test.ts
@@ -16,8 +16,11 @@ import {
ringProgress,
STEP_SECONDS,
sharpZoomCeiling,
+ spannedFrames,
stepFrames,
stepKind,
+ stepSpans,
+ stepStarts,
toFrames,
tooltipBand,
tooltipPlacement,
@@ -28,7 +31,13 @@ import {
zoomCrop,
zoomProgress,
} from '@/core/export/video-export';
-import { FRAME_HEIGHT, FRAME_WIDTH, RESOLUTION_SPECS } from '@/core/export/video-support';
+import {
+ FRAME_HEIGHT,
+ FRAME_WIDTH,
+ RESOLUTION_SPECS,
+ uniformTimeline,
+ voiceTimeline,
+} from '@/core/export/video-support';
const measure = (line: string) => line.length * 10;
@@ -659,3 +668,48 @@ describe('cursorOriginFor', () => {
expect(CURSOR_ENTRY_ORIGIN.y).toBeGreaterThan(1);
});
});
+
+describe('narrated step spans', () => {
+ it('matches the uniform layout when nothing was narrated', () => {
+ const spans = stepSpans(uniformTimeline(4), 4);
+ expect(spans).toEqual([157, 157, 157, 157]);
+ expect(stepStarts(spans)).toEqual([0, 147, 294, 441]);
+ expect(spannedFrames(spans)).toBe(totalStepFrames(4));
+ });
+
+ it('never runs a step shorter than the animation, whatever the timeline claims', () => {
+ expect(stepSpans({ coverSeconds: 3, stepSeconds: [0.1, 1] }, 2)).toEqual([157, 157]);
+ });
+
+ it('pushes later steps back by exactly what a long narration added', () => {
+ const spans = stepSpans(voiceTimeline(3, new Map([[0, 8]])), 3);
+ expect(spans[0]).toBe(toFrames(0.35 + 8 + 0.6));
+ expect(stepStarts(spans)).toEqual([0, spans[0] - 10, spans[0] - 10 + 147]);
+ expect(spannedFrames(spans)).toBe(spans[0] + 157 * 2 - 10 * 2);
+ });
+
+ it('holds the last step for its full span', () => {
+ const spans = stepSpans(voiceTimeline(2, new Map([[1, 9]])), 2);
+ expect(spannedFrames(spans)).toBe(157 - 10 + spans[1]);
+ });
+});
+
+describe('videoChapters on a narrated timeline', () => {
+ const steps = (n: number) =>
+ Array.from({ length: n }, (_, i) => ({ id: `s${i}`, description: `Do thing ${i}`, action: 'click' }) as never);
+
+ it('opens the first chapter after the stretched cover card', () => {
+ const timeline = voiceTimeline(2, new Map([[-1, 6]]));
+ expect(videoChapters(steps(2), true, FPS, timeline)[0].start).toBeCloseTo(timeline.coverSeconds, 6);
+ });
+
+ it('gives a narrated step the longer chapter', () => {
+ const marks = videoChapters(steps(3), false, FPS, voiceTimeline(3, new Map([[1, 8]])));
+ expect(marks[1].end - marks[1].start).toBeGreaterThan(marks[0].end - marks[0].start);
+ expect(marks[2].start).toBeCloseTo(marks[1].end, 6);
+ });
+
+ it('is unchanged for a guide with no narration', () => {
+ expect(videoChapters(steps(4), true, FPS, uniformTimeline(4))).toEqual(videoChapters(steps(4), true));
+ });
+});
diff --git a/src/core/export/__tests__/video-support.test.ts b/src/core/export/__tests__/video-support.test.ts
index 3d0e9d9..1d509bf 100644
--- a/src/core/export/__tests__/video-support.test.ts
+++ b/src/core/export/__tests__/video-support.test.ts
@@ -1,6 +1,19 @@
// @vitest-environment jsdom
import { afterEach, describe, expect, it, vi } from 'vitest';
-import { AVC_CODEC, FRAME_HEIGHT, FRAME_WIDTH, VP9_CODEC } from '@/core/export/video-support';
+import {
+ AVC_CODEC,
+ COVER_SECONDS,
+ FRAME_HEIGHT,
+ FRAME_WIDTH,
+ STEP_SECONDS,
+ TRANSITION_SECONDS,
+ uniformTimeline,
+ VOICE_LEAD_SEC,
+ VOICE_TAIL_SEC,
+ VP9_CODEC,
+ videoSeconds,
+ voiceTimeline,
+} from '@/core/export/video-support';
interface ProbedConfig {
codec: string;
@@ -124,3 +137,56 @@ describe('canExportVideo', () => {
expect(probed.length).toBe(afterFirst);
});
});
+
+describe('voiceTimeline', () => {
+ it('leaves an unnarrated guide on the structural timeline', () => {
+ expect(voiceTimeline(3, new Map())).toEqual(uniformTimeline(3));
+ });
+
+ it('never shortens a step whose narration is briefer than the animation', () => {
+ expect(voiceTimeline(1, new Map([[0, 0.5]])).stepSeconds[0]).toBe(STEP_SECONDS);
+ });
+
+ it('holds a step until a long narration finishes, with a lead-in and a tail', () => {
+ expect(voiceTimeline(1, new Map([[0, 9]])).stepSeconds[0]).toBeCloseTo(VOICE_LEAD_SEC + 9 + VOICE_TAIL_SEC, 10);
+ });
+
+ it('stretches the cover card for the spoken title', () => {
+ expect(voiceTimeline(1, new Map([[-1, 6]])).coverSeconds).toBeCloseTo(VOICE_LEAD_SEC + 6 + VOICE_TAIL_SEC, 10);
+ expect(voiceTimeline(1, new Map([[-1, 0.2]])).coverSeconds).toBe(COVER_SECONDS);
+ });
+
+ it('sizes each step from its own clip', () => {
+ const timeline = voiceTimeline(3, new Map([[1, 8]]));
+ expect(timeline.stepSeconds[0]).toBe(STEP_SECONDS);
+ expect(timeline.stepSeconds[1]).toBeGreaterThan(STEP_SECONDS);
+ expect(timeline.stepSeconds[2]).toBe(STEP_SECONDS);
+ });
+});
+
+describe('videoSeconds', () => {
+ it('is nothing for a guide with no frames', () => {
+ expect(videoSeconds(0, true)).toBe(0);
+ });
+
+ it('counts one step as its full hold', () => {
+ expect(videoSeconds(1, false)).toBeCloseTo(STEP_SECONDS, 10);
+ });
+
+ it('takes the cross-dissolve off each join', () => {
+ expect(videoSeconds(3, false)).toBeCloseTo(3 * STEP_SECONDS - 2 * TRANSITION_SECONDS, 10);
+ });
+
+ it('adds both cards when the cover is on', () => {
+ expect(videoSeconds(1, true) - videoSeconds(1, false)).toBeCloseTo(2 * COVER_SECONDS, 10);
+ });
+
+ it('grows with a narrated timeline, which is what the estimate is for', () => {
+ const narrated = voiceTimeline(2, new Map([[0, 9]]));
+ expect(videoSeconds(2, false, narrated)).toBeGreaterThan(videoSeconds(2, false));
+ expect(videoSeconds(2, false, narrated)).toBeCloseTo(
+ VOICE_LEAD_SEC + 9 + VOICE_TAIL_SEC + STEP_SECONDS - TRANSITION_SECONDS,
+ 10,
+ );
+ });
+});
diff --git a/src/core/export/options.ts b/src/core/export/options.ts
index 28ddda3..a194ff1 100644
--- a/src/core/export/options.ts
+++ b/src/core/export/options.ts
@@ -43,6 +43,7 @@ export interface ExportOptions {
gifQuality: GifQuality;
bundleStripInputs: boolean;
bundleUrls: BundleUrlMode;
+ voiceover: boolean;
}
export const DEFAULT_EXPORT_OPTIONS: ExportOptions = {
@@ -55,6 +56,7 @@ export const DEFAULT_EXPORT_OPTIONS: ExportOptions = {
gifQuality: 'medium',
bundleStripInputs: true,
bundleUrls: 'path',
+ voiceover: false,
};
const bool = (value: unknown, fallback: boolean) => (typeof value === 'boolean' ? value : fallback);
@@ -77,6 +79,7 @@ export function normaliseExportOptions(value: unknown): ExportOptions {
bundleUrls: BUNDLE_URL_MODES.includes(raw.bundleUrls as BundleUrlMode)
? (raw.bundleUrls as BundleUrlMode)
: DEFAULT_EXPORT_OPTIONS.bundleUrls,
+ voiceover: bool(raw.voiceover, DEFAULT_EXPORT_OPTIONS.voiceover),
};
}
diff --git a/src/core/export/video-export.ts b/src/core/export/video-export.ts
index b2e4515..6a65b93 100644
--- a/src/core/export/video-export.ts
+++ b/src/core/export/video-export.ts
@@ -5,6 +5,7 @@ import type { ExportOptions } from '@/core/export/options';
import { loadExportOptions } from '@/core/export/options';
import { extractDomain, formatDate } from '@/core/export/utils';
import {
+ COVER_SECONDS,
FPS,
FRAME_FILL,
FRAME_HEIGHT,
@@ -14,16 +15,23 @@ import {
STEP_SECONDS,
STEP_ZOOM_TRANSITION_SEC,
STEP_ZOOMED_OUT_SEC,
+ TRANSITION_SECONDS,
+ uniformTimeline,
+ type VideoContainer,
+ type VideoTimeline,
+ VOICE_LEAD_SEC,
+ voiceTimeline,
} from '@/core/export/video-support';
+import { COVER_SEGMENT } from '@/core/export/voiceover/script';
import { actionSteps, calloutAccent, isBlock } from '@/core/guides/blocks';
import type { BlockType, Guide, Screenshot, Step } from '@/core/guides/types';
import type { Ctx } from '@/core/screenshot/draw';
import { drawRoundedRect, TARGET_RADIUS, TARGET_STROKE } from '@/core/screenshot/draw';
import { clamp, resolveFrameViewport, resolveTarget } from '@/core/screenshot/geometry';
import { renderScreenshot } from '@/core/screenshot/render';
+import { localStorage } from '@/lib/browser-api';
+import { logger } from '@/lib/logger';
-const TRANSITION_DURATION_SEC = 0.33;
-const COVER_SECONDS = 3;
const KEY_FRAME_INTERVAL_SEC = 2;
const ZOOM_MIN = 1;
@@ -93,7 +101,7 @@ export function stepFrames(fps = FPS): number {
}
export function overlapFrames(fps = FPS): number {
- return toFrames(TRANSITION_DURATION_SEC, fps);
+ return toFrames(TRANSITION_SECONDS, fps);
}
export function totalStepFrames(stepCount: number, fps = FPS): number {
@@ -101,6 +109,29 @@ export function totalStepFrames(stepCount: number, fps = FPS): number {
return stepCount * stepFrames(fps) - (stepCount - 1) * overlapFrames(fps);
}
+export function stepSpans(timeline: VideoTimeline, count: number, fps = FPS): number[] {
+ return Array.from({ length: count }, (_, index) =>
+ Math.max(stepFrames(fps), toFrames(timeline.stepSeconds[index] ?? STEP_SECONDS, fps)),
+ );
+}
+
+export function stepStarts(spans: number[], fps = FPS): number[] {
+ const overlap = overlapFrames(fps);
+ const starts: number[] = [];
+ let at = 0;
+ for (const span of spans) {
+ starts.push(at);
+ at += span - overlap;
+ }
+ return starts;
+}
+
+export function spannedFrames(spans: number[], fps = FPS): number {
+ if (spans.length === 0) return 0;
+ const starts = stepStarts(spans, fps);
+ return starts[spans.length - 1] + spans[spans.length - 1];
+}
+
export function zoomProgress(frame: number, fps = FPS): number {
const held = toFrames(STEP_ZOOMED_OUT_SEC, fps);
const moving = toFrames(STEP_ZOOM_TRANSITION_SEC, fps);
@@ -655,12 +686,13 @@ async function drawCardFrame(ctx: Ctx, guide: Guide, steps: Step[], brand: Brand
}
}
-export type VideoOptions = Pick<ExportOptions, 'cover' | 'stepDescriptions' | 'resolution'>;
+export type VideoOptions = Pick<ExportOptions, 'cover' | 'stepDescriptions' | 'resolution' | 'voiceover'>;
-export type FrameOptions = Pick<ExportOptions, 'cover' | 'stepDescriptions'>;
+export type FrameOptions = Pick<ExportOptions, 'cover' | 'stepDescriptions'> & { timeline?: VideoTimeline };
export interface VideoExportControls {
onProgress?: (done: number, total: number) => void;
+ onVoiceProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
@@ -687,20 +719,23 @@ export interface VideoExportResult {
blob: Blob;
extension: string;
chapters: VideoChapter[];
+ voiceoverError?: string;
}
-export function videoChapters(frames: Step[], cover: boolean, fps = FPS): VideoChapter[] {
+export function videoChapters(frames: Step[], cover: boolean, fps = FPS, timeline?: VideoTimeline): VideoChapter[] {
if (frames.length === 0) return [];
- const offset = cover ? COVER_SECONDS : 0;
- const stride = (stepFrames(fps) - overlapFrames(fps)) / fps;
- const last = offset + totalStepFrames(frames.length, fps) / fps;
+ const plan = timeline ?? uniformTimeline(frames.length);
+ const offset = cover ? plan.coverSeconds : 0;
+ const spans = stepSpans(plan, frames.length, fps);
+ const starts = stepStarts(spans, fps);
+ const last = offset + spannedFrames(spans, fps) / fps;
return frames.map((step, index) => ({
stepId: step.id,
title: step.description?.trim() || i18n.t('export.stepLabel', [String(index + 1)]),
kind: stepKind(step),
- start: offset + index * stride,
- end: index === frames.length - 1 ? last : offset + (index + 1) * stride,
+ start: offset + starts[index] / fps,
+ end: index === frames.length - 1 ? last : offset + starts[index + 1] / fps,
}));
}
@@ -718,10 +753,11 @@ export async function composeGuideFrames(
controls: { onProgress?: (done: number, total: number) => void; signal?: AbortSignal } = {},
fps = FPS,
): Promise<void> {
- const span = stepFrames(fps);
const overlap = overlapFrames(fps);
- const stride = span - overlap;
- const stepTotal = totalStepFrames(frames.length, fps);
+ const timeline = options.timeline ?? uniformTimeline(frames.length);
+ const spans = stepSpans(timeline, frames.length, fps);
+ const starts = stepStarts(spans, fps);
+ const stepTotal = spannedFrames(spans, fps);
const total = stepTotal + (options.cover ? 2 : 0);
const loaded = new Map<number, StepLayer>();
const { onProgress, signal } = controls;
@@ -766,23 +802,24 @@ export async function composeGuideFrames(
};
try {
- const offset = options.cover ? COVER_SECONDS : 0;
+ const offset = options.cover ? timeline.coverSeconds : 0;
const cards = actionSteps(frames);
if (options.cover) {
abortIfRequested();
await drawCardFrame(ctx, guide, cards, brand, i18n.t('export.guideLabel'));
- await sink(0, COVER_SECONDS);
+ await sink(0, timeline.coverSeconds);
done += 1;
onProgress?.(done, total);
}
+ let index = 0;
for (let frame = 0; frame < stepTotal; frame++) {
abortIfRequested();
- const index = Math.min(Math.floor(frame / stride), frames.length - 1);
- const local = frame - index * stride;
+ while (index + 1 < starts.length && frame >= starts[index + 1]) index += 1;
+ const local = frame - starts[index];
const outgoing = index > 0 && local < overlap ? index - 1 : -1;
const current = await layerAt(index);
@@ -794,7 +831,7 @@ export async function composeGuideFrames(
ctx.fillRect(0, 0, FRAME_WIDTH, FRAME_HEIGHT);
if (previous) {
- drawStepFrame(ctx, previous, local + stride, device, fps);
+ drawStepFrame(ctx, previous, frame - starts[outgoing], device, fps);
ctx.globalAlpha = (local + 1) / overlap;
drawStepFrame(ctx, current, local, device, fps);
ctx.globalAlpha = 1;
@@ -820,6 +857,79 @@ export async function composeGuideFrames(
}
}
+interface PreparedVoiceover {
+ codec: 'aac' | 'opus';
+ clips: Map<number, AudioBuffer>;
+ timeline: VideoTimeline;
+}
+
+async function narrateOrSkip(
+ guide: Guide,
+ frames: Step[],
+ cover: boolean,
+ container: VideoContainer,
+ controls: VideoExportControls,
+): Promise<{ voice: PreparedVoiceover | null; error?: string }> {
+ try {
+ return { voice: await prepareVoiceover(guide, frames, cover, container, controls) };
+ } catch (error) {
+ if (error instanceof DOMException && error.name === 'AbortError') throw error;
+ logger.error('[voiceover] narration failed, exporting a silent video', error);
+ return { voice: null, error: error instanceof Error ? error.message : String(error) };
+ }
+}
+
+async function prepareVoiceover(
+ guide: Guide,
+ frames: Step[],
+ cover: boolean,
+ container: VideoContainer,
+ controls: VideoExportControls,
+): Promise<PreparedVoiceover | null> {
+ const [{ pickVoiceCodec }, { resolveVoiceoverConfig, VOICEOVER_SETTINGS }, { voiceoverScript }, { renderVoiceover }] =
+ await Promise.all([
+ import('./voiceover/audio'),
+ import('./voiceover/config'),
+ import('./voiceover/script'),
+ import('./voiceover/render'),
+ ]);
+
+ const config = resolveVoiceoverConfig(await localStorage.get([...VOICEOVER_SETTINGS]));
+ if (!config.apiKey) {
+ logger.warn('[voiceover] skipped: no ElevenLabs API key');
+ return null;
+ }
+
+ const segments = voiceoverScript(guide, frames, cover);
+ if (segments.length === 0) return null;
+
+ const codec = await pickVoiceCodec(container);
+ if (!codec) {
+ logger.warn('[voiceover] skipped: this browser cannot encode', container === 'mp4' ? 'AAC' : 'Opus');
+ return null;
+ }
+
+ const clips = await renderVoiceover(segments, config, {
+ signal: controls.signal,
+ onProgress: controls.onVoiceProgress,
+ });
+
+ const seconds = new Map(Array.from(clips, ([index, clip]) => [index, clip.duration]));
+ return { codec, clips, timeline: voiceTimeline(frames.length, seconds) };
+}
+
+export function voiceoverPlacement(
+ clips: Map<number, AudioBuffer>,
+ starts: number[],
+ offsetSec: number,
+ fps = FPS,
+): { startSec: number; buffer: AudioBuffer }[] {
+ return Array.from(clips, ([index, buffer]) => ({
+ startSec: (index === COVER_SEGMENT ? 0 : offsetSec + starts[index] / fps) + VOICE_LEAD_SEC,
+ buffer,
+ }));
+}
+
export async function exportGuideAsVideo(
guide: Guide,
steps: Step[],
@@ -830,9 +940,16 @@ export async function exportGuideAsVideo(
const frames = steps.filter((step) => isBlock(step) || screenshots.has(step.id));
if (frames.length === 0) throw new Error('This guide has no screenshots to turn into a video');
- const { BufferTarget, CanvasSource, Mp4OutputFormat, Output, QUALITY_HIGH, WebMOutputFormat } = await import(
- 'mediabunny'
- );
+ const {
+ AudioBufferSource,
+ BufferTarget,
+ CanvasSource,
+ Mp4OutputFormat,
+ Output,
+ QUALITY_HIGH,
+ QUALITY_MEDIUM,
+ WebMOutputFormat,
+ } = await import('mediabunny');
const [brand, options] = await Promise.all([
loadBranding(),
@@ -847,6 +964,12 @@ export async function exportGuideAsVideo(
const mp4 = container === 'mp4';
const spec = RESOLUTION_SPECS[resolution];
+ const narration = options.voiceover
+ ? await narrateOrSkip(guide, frames, Boolean(options.cover), container, controls)
+ : { voice: null };
+ const voice = narration.voice;
+ const timeline = voice?.timeline ?? uniformTimeline(frames.length);
+
const canvas = document.createElement('canvas');
canvas.width = spec.width;
canvas.height = spec.height;
@@ -865,6 +988,10 @@ export async function exportGuideAsVideo(
keyFrameInterval: KEY_FRAME_INTERVAL_SEC,
});
output.addVideoTrack(source);
+
+ const audio = voice ? new AudioBufferSource({ codec: voice.codec, quality: QUALITY_MEDIUM }) : null;
+ if (audio) output.addAudioTrack(audio);
+
await output.start();
try {
@@ -872,13 +999,24 @@ export async function exportGuideAsVideo(
guide,
frames,
screenshots,
- options,
+ { ...options, timeline },
brand,
ctx,
device,
(at, dur) => source.add(at, dur),
controls,
);
+
+ if (voice && audio) {
+ const { writeVoiceTrack } = await import('./voiceover/audio');
+ const spans = stepSpans(timeline, frames.length);
+ const offset = options.cover ? timeline.coverSeconds : 0;
+ const total = offset + spannedFrames(spans) / FPS + (options.cover ? COVER_SECONDS : 0);
+ await writeVoiceTrack(voiceoverPlacement(voice.clips, stepStarts(spans), offset), total, (buffer) =>
+ audio.add(buffer),
+ );
+ }
+
await output.finalize();
} catch (error) {
await output.cancel();
@@ -891,6 +1029,7 @@ export async function exportGuideAsVideo(
return {
blob: new Blob([buffer], { type: mp4 ? 'video/mp4' : 'video/webm' }),
extension: mp4 ? 'mp4' : 'webm',
- chapters: videoChapters(frames, Boolean(options.cover)),
+ chapters: videoChapters(frames, Boolean(options.cover), FPS, timeline),
+ ...(narration.error ? { voiceoverError: narration.error } : {}),
};
}
diff --git a/src/core/export/video-support.ts b/src/core/export/video-support.ts
index 99e2f3f..4f8f632 100644
--- a/src/core/export/video-support.ts
+++ b/src/core/export/video-support.ts
@@ -11,6 +11,47 @@ export const STEP_ZOOM_TRANSITION_SEC = 0.73;
export const STEP_ZOOMED_IN_SEC = 3;
export const STEP_SECONDS = STEP_ZOOMED_OUT_SEC + STEP_ZOOM_TRANSITION_SEC + STEP_ZOOMED_IN_SEC;
+export const COVER_SECONDS = 3;
+
+export const TRANSITION_SECONDS = 0.33;
+
+export const VOICE_LEAD_SEC = 0.35;
+export const VOICE_TAIL_SEC = 0.6;
+
+export interface VideoTimeline {
+ coverSeconds: number;
+ stepSeconds: number[];
+}
+
+export function uniformTimeline(stepCount: number): VideoTimeline {
+ return { coverSeconds: COVER_SECONDS, stepSeconds: Array.from({ length: stepCount }, () => STEP_SECONDS) };
+}
+
+export function narratedSeconds(clipSeconds: number): number {
+ return VOICE_LEAD_SEC + clipSeconds + VOICE_TAIL_SEC;
+}
+
+export function videoSeconds(stepCount: number, cover: boolean, timeline?: VideoTimeline): number {
+ if (stepCount <= 0) return 0;
+ const plan = timeline ?? uniformTimeline(stepCount);
+ const held = Array.from({ length: stepCount }, (_, index) =>
+ Math.max(STEP_SECONDS, plan.stepSeconds[index] ?? STEP_SECONDS),
+ ).reduce((total, seconds) => total + seconds, 0);
+ const cards = cover ? plan.coverSeconds + COVER_SECONDS : 0;
+ return held - (stepCount - 1) * TRANSITION_SECONDS + cards;
+}
+
+export function voiceTimeline(stepCount: number, clipSeconds: Map<number, number>, coverIndex = -1): VideoTimeline {
+ const cover = clipSeconds.get(coverIndex);
+ return {
+ coverSeconds: cover === undefined ? COVER_SECONDS : Math.max(COVER_SECONDS, narratedSeconds(cover)),
+ stepSeconds: Array.from({ length: stepCount }, (_, index) => {
+ const clip = clipSeconds.get(index);
+ return clip === undefined ? STEP_SECONDS : Math.max(STEP_SECONDS, narratedSeconds(clip));
+ }),
+ };
+}
+
export interface ResolutionSpec {
width: number;
height: number;