Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

fix(export): pay once per clip and never cache one that cannot play

davbiob239a478d931
diff --git a/src/core/export/voiceover/__tests__/render.test.ts b/src/core/export/voiceover/__tests__/render.test.ts
new file mode 100644
index 0000000..cb2ddcd
--- /dev/null
+++ b/src/core/export/voiceover/__tests__/render.test.ts
@@ -0,0 +1,94 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const synthesizeSpeech = vi.fn();
+const decodeClip = vi.fn();
+const readClip = vi.fn();
+const writeClip = vi.fn();
+const deleteClip = vi.fn();
+
+vi.mock('../client', () => ({ synthesizeSpeech: (...args: unknown[]) => synthesizeSpeech(...args) }));
+vi.mock('../audio', () => ({ decodeClip: (...args: unknown[]) => decodeClip(...args) }));
+vi.mock('../cache', () => ({
+ clipKey: async (_provider: string, _voice: string, _model: string, text: string) => `key:${text}`,
+ readClip: (...args: unknown[]) => readClip(...args),
+ writeClip: (...args: unknown[]) => writeClip(...args),
+ deleteClip: (...args: unknown[]) => deleteClip(...args),
+}));
+vi.mock('@/lib/logger', () => ({ logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn(), debug: vi.fn() } }));
+
+import type { VoiceoverConfig } from '../config';
+import { renderVoiceover } from '../render';
+import type { VoiceoverSegment } from '../script';
+
+const CONFIG: VoiceoverConfig = {
+ provider: 'openai',
+ apiKey: 'sk',
+ voiceId: 'alloy',
+ modelId: 'tts-1',
+ source: 'voiceover',
+};
+const SEGMENTS: VoiceoverSegment[] = [{ index: 0, text: 'Open billing' }];
+const BUFFER = { length: 10 } as AudioBuffer;
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ readClip.mockResolvedValue(null);
+ writeClip.mockResolvedValue(undefined);
+ deleteClip.mockResolvedValue(undefined);
+ decodeClip.mockResolvedValue(BUFFER);
+});
+
+describe('renderVoiceover', () => {
+ it('pays for a clip once when two exports ask for it at the same time', async () => {
+ let finish!: (bytes: ArrayBuffer) => void;
+ synthesizeSpeech.mockReturnValue(new Promise((resolve) => (finish = resolve)));
+
+ const preview = renderVoiceover(SEGMENTS, CONFIG);
+ const download = renderVoiceover(SEGMENTS, CONFIG);
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ finish(new ArrayBuffer(4));
+
+ const [a, b] = await Promise.all([preview, download]);
+ expect(synthesizeSpeech).toHaveBeenCalledTimes(1);
+ expect(a.get(0)).toBe(BUFFER);
+ expect(b.get(0)).toBe(BUFFER);
+ });
+
+ it('does not cache a clip that cannot be decoded', async () => {
+ synthesizeSpeech.mockResolvedValue(new ArrayBuffer(0));
+ decodeClip.mockRejectedValue(new Error('Unable to decode audio data'));
+
+ await expect(renderVoiceover(SEGMENTS, CONFIG)).rejects.toThrow('Unable to decode');
+ expect(writeClip).not.toHaveBeenCalled();
+ });
+
+ it('throws out a cached clip that no longer decodes and fetches it again', async () => {
+ readClip.mockResolvedValue(new ArrayBuffer(0));
+ decodeClip.mockRejectedValueOnce(new Error('Unable to decode audio data')).mockResolvedValue(BUFFER);
+ synthesizeSpeech.mockResolvedValue(new ArrayBuffer(4));
+
+ const clips = await renderVoiceover(SEGMENTS, CONFIG);
+
+ expect(deleteClip).toHaveBeenCalledWith('key:Open billing');
+ expect(synthesizeSpeech).toHaveBeenCalledTimes(1);
+ expect(clips.get(0)).toBe(BUFFER);
+ });
+
+ it('starts its own request when the export it was waiting on is cancelled', async () => {
+ const preview = new AbortController();
+ let rejectFirst!: (error: unknown) => void;
+ synthesizeSpeech
+ .mockReturnValueOnce(new Promise((_resolve, reject) => (rejectFirst = reject)))
+ .mockResolvedValueOnce(new ArrayBuffer(4));
+
+ const previewing = renderVoiceover(SEGMENTS, CONFIG, { signal: preview.signal });
+ const downloading = renderVoiceover(SEGMENTS, CONFIG);
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ preview.abort();
+ rejectFirst(new DOMException('Voiceover was aborted', 'AbortError'));
+
+ await expect(previewing).rejects.toMatchObject({ name: 'AbortError' });
+ expect((await downloading).get(0)).toBe(BUFFER);
+ expect(synthesizeSpeech).toHaveBeenCalledTimes(2);
+ });
+});
diff --git a/src/core/export/voiceover/cache.ts b/src/core/export/voiceover/cache.ts
index 7deaa41..04b8318 100644
--- a/src/core/export/voiceover/cache.ts
+++ b/src/core/export/voiceover/cache.ts
@@ -38,6 +38,14 @@ export async function writeClip(id: string, bytes: ArrayBuffer): Promise<void> {
}
}
+export async function deleteClip(id: string): Promise<void> {
+ try {
+ await db.voiceClips.delete(id);
+ } catch (error) {
+ logger.error('[voiceover] cache delete failed', error);
+ }
+}
+
export async function clearClips(): Promise<void> {
await db.voiceClips.clear();
}
diff --git a/src/core/export/voiceover/render.ts b/src/core/export/voiceover/render.ts
index d8f9bdc..d1e2367 100644
--- a/src/core/export/voiceover/render.ts
+++ b/src/core/export/voiceover/render.ts
@@ -1,6 +1,6 @@
import { logger } from '@/lib/logger';
import { decodeClip } from './audio';
-import { clipKey, readClip, writeClip } from './cache';
+import { clipKey, deleteClip, readClip, writeClip } from './cache';
import { synthesizeSpeech } from './client';
import type { VoiceoverConfig } from './config';
import type { VoiceoverSegment } from './script';
@@ -30,27 +30,79 @@ function wait(ms: number, signal?: AbortSignal): Promise<void> {
});
}
-async function clipBytes(
- segment: VoiceoverSegment,
+const inFlight = new Map<string, Promise<AudioBuffer>>();
+
+function untilAborted<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
+ if (!signal) return promise;
+ return new Promise<T>((resolve, reject) => {
+ const onAbort = () => reject(new DOMException('Voiceover was aborted', 'AbortError'));
+ if (signal.aborted) return onAbort();
+ signal.addEventListener('abort', onAbort, { once: true });
+ promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort));
+ });
+}
+
+async function synthesizeAndStore(
+ key: string,
+ text: string,
config: VoiceoverConfig,
signal?: AbortSignal,
-): Promise<ArrayBuffer> {
- const key = await clipKey(config.provider, config.voiceId, config.modelId, segment.text);
- const cached = await readClip(key);
- if (cached) return cached;
-
+): Promise<AudioBuffer> {
let bytes: ArrayBuffer;
try {
- bytes = await synthesizeSpeech(config, segment.text, signal);
+ bytes = await synthesizeSpeech(config, text, signal);
} catch (error) {
if (aborted(error)) throw error;
logger.error('[voiceover] synthesis retrying', error);
await wait(RETRY_DELAY_MS, signal);
- bytes = await synthesizeSpeech(config, segment.text, signal);
+ bytes = await synthesizeSpeech(config, text, signal);
}
+ const buffer = await decodeClip(bytes);
await writeClip(key, bytes);
- return bytes;
+ return buffer;
+}
+
+async function sharedSynthesis(
+ key: string,
+ text: string,
+ config: VoiceoverConfig,
+ signal?: AbortSignal,
+): Promise<AudioBuffer> {
+ const pending = inFlight.get(key);
+ if (pending) {
+ try {
+ return await untilAborted(pending, signal);
+ } catch (error) {
+ if (!aborted(error) || signal?.aborted) throw error;
+ }
+ }
+
+ const request = synthesizeAndStore(key, text, config, signal);
+ inFlight.set(key, request);
+ try {
+ return await request;
+ } finally {
+ if (inFlight.get(key) === request) inFlight.delete(key);
+ }
+}
+
+async function clipBuffer(
+ segment: VoiceoverSegment,
+ config: VoiceoverConfig,
+ signal?: AbortSignal,
+): Promise<AudioBuffer> {
+ const key = await clipKey(config.provider, config.voiceId, config.modelId, segment.text);
+ const cached = await readClip(key);
+ if (cached) {
+ try {
+ return await decodeClip(cached);
+ } catch (error) {
+ logger.error('[voiceover] cached clip could not be decoded, synthesising it again', error);
+ await deleteClip(key);
+ }
+ }
+ return sharedSynthesis(key, segment.text, config, signal);
}
export async function renderVoiceover(
@@ -66,7 +118,7 @@ export async function renderVoiceover(
for (const segment of segments) {
if (signal?.aborted) throw new DOMException('Voiceover was aborted', 'AbortError');
- clips.set(segment.index, await decodeClip(await clipBytes(segment, config, signal)));
+ clips.set(segment.index, await clipBuffer(segment, config, signal));
onProgress?.(++done, segments.length);
}