VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
import i18n from 'i18next';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
// Streaming TTS preview (feat: streaming-tts-preview): the NDJSON client must
// start Web Audio playback from the FIRST chunk, register as a tracked
// 'output' playback (mini-player bar), flip its label on completion, resolve
// with the "done" metadata — and turn ANY mid-stream failure into
// StreamingPreviewError so useTTS can fall back to the classic flow.
vi.mock('../api/client', () => ({
apiFetch: vi.fn(),
}));
const { apiFetch } = await import('../api/client');
const {
streamGenerateSpeech,
createStreamingChunkPlayer,
supportsStreamingPreview,
decodePcm16Base64,
pcm16BytesToFloat32,
peaksFromChunkList,
StreamingPreviewError,
shouldFallbackToClassic,
} = await import('../utils/streamingTts');
const { getPlaybackTrack, stopActivePlayback, seekActivePlayback } =
await import('../utils/playback');
// ── Web Audio fakes ─────────────────────────────────────────────────────────
class FakeGainParam {
setValueAtTime() {}
linearRampToValueAtTime() {}
}
class FakeGain {
constructor() {
this.gain = new FakeGainParam();
}
connect() {}
disconnect() {}
}
class FakeSource {
constructor(ctx) {
this.ctx = ctx;
}
connect() {}
disconnect() {}
start(when, offset) {
this.startedAt = { when, offset: offset || 0 };
this.ctx.started.push(this);
}
stop() {
this.stopped = true;
}
}
class FakeAudioContext {
static instances = [];
static constructorArgs = [];
static rejectSampleRate = false;
constructor(options) {
FakeAudioContext.constructorArgs.push(options);
if (FakeAudioContext.rejectSampleRate && options?.sampleRate) {
throw new Error('unsupported sample rate');
}
this.currentTime = 0;
this.state = 'running';
this.sampleRate = options?.sampleRate ?? 48000;
this.destination = {};
this.started = []; // every source that called start()
FakeAudioContext.instances.push(this);
}
resume() {
this.state = 'running';
return Promise.resolve();
}
suspend() {
this.state = 'suspended';
return Promise.resolve();
}
close() {
this.state = 'closed';
}
createBuffer(channels, length, sampleRate) {
const data = new Float32Array(length);
return {
length,
duration: length / sampleRate,
numberOfChannels: channels,
sampleRate,
copyToChannel: (src) => data.set(src),
getChannelData: () => data,
};
}
createBufferSource() {
return new FakeSource(this);
}
createGain() {
return new FakeGain();
}
}
// ── NDJSON fixtures ─────────────────────────────────────────────────────────
const b64Pcm = (samples) => {
// Int16 PCM little-endian → base64.
const pcm = new Int16Array(samples);
const bytes = new Uint8Array(pcm.buffer);
let bin = '';
for (const b of bytes) bin += String.fromCharCode(b);
return btoa(bin);
};
const CHUNK_SAMPLES = 2400; // 100 ms @ 24 kHz
const startEvent = (totalChunks) => ({
type: 'start',
sample_rate: 24000,
channels: 1,
format: 'pcm16',
total_chunks: totalChunks,
crossfade_ms: 50,
seed: 7,
});
const chunkEvent = (seq) => ({
type: 'chunk',
seq,
pcm: b64Pcm(Array.from({ length: CHUNK_SAMPLES }, (_, i) => (i % 100) * 50)),
});
const doneEvent = {
type: 'done',
id: 'abc12345',
audio_path: 'abc12345.wav',
duration: 0.3,
gen_time: 1.2,
seed: 7,
sample_rate: 24000,
};
const ndjsonResponse = (events, headers = {}) => {
const enc = new TextEncoder();
const stream = new ReadableStream({
start(controller) {
for (const e of events) controller.enqueue(enc.encode(JSON.stringify(e) + '\n'));
controller.close();
},
});
return { headers: new Headers(headers), body: stream };
};
beforeEach(() => {
FakeAudioContext.instances = [];
FakeAudioContext.constructorArgs = [];
FakeAudioContext.rejectSampleRate = false;
window.AudioContext = FakeAudioContext;
apiFetch.mockReset();
});
afterEach(() => {
stopActivePlayback();
});
// ── unit: codecs ────────────────────────────────────────────────────────────
describe('decodePcm16Base64', () => {
it('roundtrips int16 samples to normalized floats', () => {
const out = decodePcm16Base64(b64Pcm([0, 16384, -16384, 32767, -32768]));
expect(out.length).toBe(5);
expect(out[0]).toBe(0);
expect(out[1]).toBeCloseTo(0.5, 3);
expect(out[2]).toBeCloseTo(-0.5, 3);
expect(out[3]).toBeCloseTo(1, 2);
expect(out[4]).toBe(-1);
});
});
describe('pcm16BytesToFloat32', () => {
it('decodes a raw binary frame (the /ws/tts shape) identically to base64', () => {
const samples = [0, 16384, -16384, 32767, -32768];
const raw = pcm16BytesToFloat32(new Int16Array(samples).buffer);
const viaB64 = decodePcm16Base64(b64Pcm(samples));
expect(Array.from(raw)).toEqual(Array.from(viaB64));
});
it('decodes a view carved at an odd byteOffset instead of throwing', () => {
const backing = new Uint8Array(5);
backing.set(new Uint8Array(new Int16Array([-12345]).buffer), 1);
const out = pcm16BytesToFloat32(backing.subarray(1, 3));
expect(out.length).toBe(1);
expect(out[0]).toBeCloseTo(-12345 / 32768, 5);
});
it('respects a Uint8Array view with a nonzero byteOffset', () => {
const backing = new Uint8Array(8);
backing.set(new Uint8Array(new Int16Array([12345]).buffer), 2);
const out = pcm16BytesToFloat32(backing.subarray(2, 4));
expect(out.length).toBe(1);
expect(out[0]).toBeCloseTo(12345 / 32768, 5);
});
});
describe('createStreamingChunkPlayer.appendPcm16Bytes', () => {
it('schedules raw /ws/tts frames on the same gapless timeline as base64 chunks', () => {
const player = createStreamingChunkPlayer({ label: 'live', sampleRate: 24000 });
const ctx = FakeAudioContext.instances.at(-1);
const frame = new Int16Array(Array.from({ length: 2400 }, (_, i) => (i % 100) * 50));
player.appendPcm16Bytes(frame.buffer);
player.appendPcm16Bytes(frame.buffer);
expect(ctx.started.length).toBe(2);
// Both chunks keep their 80 ms scheduling lead and remain gapless.
expect(ctx.started[0].startedAt.when).toBeCloseTo(0.08, 5);
expect(ctx.started[1].startedAt.when).toBeCloseTo(0.08 + 2400 / 24000, 5);
player.fail();
});
});
describe('peaksFromChunkList', () => {
it('returns normalized peaks across chunk boundaries', () => {
const quiet = new Float32Array(1000).fill(0.1);
const loud = new Float32Array(1000).fill(0.8);
const peaks = peaksFromChunkList([quiet, loud], 10);
expect(peaks.length).toBe(10);
expect(Math.max(...peaks)).toBe(1); // normalized
expect(peaks[0]).toBeLessThan(peaks[9]); // loud tail dominates
});
it('handles empty input', () => {
expect(peaksFromChunkList([])).toBeNull();
});
});
describe('supportsStreamingPreview', () => {
it('is true with an AudioContext and false without', () => {
expect(supportsStreamingPreview()).toBe(true);
const saved = window.AudioContext;
delete window.AudioContext;
delete window.webkitAudioContext;
expect(supportsStreamingPreview()).toBe(false);
window.AudioContext = saved;
});
});
// ── streamGenerateSpeech ────────────────────────────────────────────────────
describe('streamGenerateSpeech', () => {
it('plays chunks progressively, flips the label on done, resolves metadata', async () => {
apiFetch.mockResolvedValue(
ndjsonResponse([startEvent(3), chunkEvent(0), chunkEvent(1), chunkEvent(2), doneEvent], {
'X-Seed': '7',
}),
);
const onHeaders = vi.fn();
const onProgress = vi.fn();
const meta = await streamGenerateSpeech(new FormData(), {
label: 'Streaming preview…',
finalLabel: 'Generated audio',
onHeaders,
onProgress,
});
expect(meta.id).toBe('abc12345');
expect(meta.audio_path).toBe('abc12345.wav');
expect(onHeaders).toHaveBeenCalledTimes(1);
expect(onProgress).toHaveBeenLastCalledWith(100);
// All three chunks were scheduled on one context.
const ctx = FakeAudioContext.instances[0];
expect(ctx.started.length).toBe(3);
// Tracked 'output' claim → mini-player bar; label flipped at completion;
// duration covers the buffered chunks (3 × 100 ms minus 2 × 50 ms fades).
const track = getPlaybackTrack();
expect(track.source).toBe('output');
expect(track.label).toBe('Generated audio');
expect(track.duration).toBeCloseTo(0.2, 3);
expect(track.peaks?.length).toBeGreaterThan(0);
expect(track.canSeek).toBe(true);
});
it('appends stream=true without mutating the caller FormData', async () => {
apiFetch.mockResolvedValue(ndjsonResponse([startEvent(1), chunkEvent(0), doneEvent]));
const fd = new FormData();
fd.append('text', 'hello');
await streamGenerateSpeech(fd, {});
expect(fd.get('stream')).toBeNull(); // caller's copy untouched
const sent = apiFetch.mock.calls[0][1].body;
expect(sent.get('stream')).toBe('true');
expect(sent.get('text')).toBe('hello');
});
it('turns an in-band error event into StreamingPreviewError and releases the bar', async () => {
apiFetch.mockResolvedValue(
ndjsonResponse([startEvent(3), chunkEvent(0), { type: 'error', detail: 'engine boom' }]),
);
await expect(streamGenerateSpeech(new FormData(), {})).rejects.toThrow(StreamingPreviewError);
expect(getPlaybackTrack()).toBeNull(); // playback torn down for the fallback
expect(FakeAudioContext.instances[0].state).toBe('closed');
});
it('carries the backend error class from the error frame (#1800)', async () => {
// Every unclassified engine failure renders the same floor message, so
// the auto-filed reports were byte-identical and none could be triaged.
// The class name is the only thing that separates them.
apiFetch.mockResolvedValue(
ndjsonResponse([
startEvent(3),
chunkEvent(0),
{ type: 'error', detail: 'Generation failed.', error_class: 'MemoryError' },
]),
);
await expect(streamGenerateSpeech(new FormData(), {})).rejects.toMatchObject({
errorClass: 'MemoryError',
});
});
it('leaves the error class null when the frame omits it', async () => {
apiFetch.mockResolvedValue(
ndjsonResponse([startEvent(3), chunkEvent(0), { type: 'error', detail: 'boom' }]),
);
await expect(streamGenerateSpeech(new FormData(), {})).rejects.toMatchObject({
errorClass: null,
});
});
it('carries the retryable marker from a GPU-timeout error frame (#1190)', async () => {
// A retryable failure means the backend already spent the full budget on
// this text and the abandoned job still holds the device — useTTS uses
// this flag to skip the classic re-render instead of paying the timeout
// a second time.
apiFetch.mockResolvedValue(
ndjsonResponse([
startEvent(3),
chunkEvent(0),
{ type: 'error', detail: 'ran for more than 300s', retryable: true, retry_after: 45 },
]),
);
const err = await streamGenerateSpeech(new FormData(), {}).catch((e) => e);
expect(err).toBeInstanceOf(StreamingPreviewError);
expect(err.retryable).toBe(true);
expect(err.retryAfter).toBe(45);
// A plain engine failure stays non-retryable → the classic fallback still
// applies, unchanged.
apiFetch.mockResolvedValue(
ndjsonResponse([startEvent(2), chunkEvent(0), { type: 'error', detail: 'engine boom' }]),
);
const plain = await streamGenerateSpeech(new FormData(), {}).catch((e) => e);
expect(plain.retryable).toBe(false);
});
it.each([
['GPU_ARCH_UNSUPPORTED', 'gpu_arch_unsupported'],
['WINDOWS_APP_CONTROL_BLOCKED', 'windows_app_control_blocked'],
['AUDIO_IO_FAILED', 'audio_io_failed'],
])('localizes %s while retaining retry metadata', async (topic, key) => {
const translate = vi.spyOn(i18n, 't').mockReturnValue('Localized recovery');
const terminal = topic !== 'AUDIO_IO_FAILED';
try {
apiFetch.mockResolvedValue(
ndjsonResponse([
{
type: 'error',
docs_topic: topic,
detail: 'English fallback',
terminal,
retryable: !terminal,
},
]),
);
const error = await streamGenerateSpeech(new FormData(), {}).catch((e) => e);
expect(error.message).toBe('Localized recovery');
expect(translate).toHaveBeenCalledWith(`tts_errors.${key}`);
expect(error.terminal).toBe(terminal);
expect(shouldFallbackToClassic(error)).toBe(false);
} finally {
translate.mockRestore();
}
});
it('localizes a terminal profile language refusal without retrying', async () => {
const translate = vi.spyOn(i18n, 't').mockReturnValue('Localized profile guidance');
try {
apiFetch.mockResolvedValue(
ndjsonResponse([
{
type: 'error',
code: 'profile_language_rejected',
language: 'Persian',
detail: 'English fallback',
retryable: false,
terminal: true,
},
]),
);
const error = await streamGenerateSpeech(new FormData(), {}).catch((e) => e);
expect(error.message).toBe('Localized profile guidance');
expect(shouldFallbackToClassic(error)).toBe(false);
expect(translate).toHaveBeenCalledWith('tts_errors.profile_language_rejected', {
language: 'Persian',
});
} finally {
translate.mockRestore();
}
});
it('marks actionable clone-reference errors terminal to prevent a classic retry', async () => {
apiFetch.mockResolvedValue(
ndjsonResponse([
startEvent(1),
{ type: 'error', detail: '[clone_ref_no_speech] trim the reference' },
]),
);
const err = await streamGenerateSpeech(new FormData(), {}).catch((e) => e);
expect(err).toBeInstanceOf(StreamingPreviewError);
expect(err.terminal).toBe(true);
expect(shouldFallbackToClassic(err)).toBe(false);
});
it('rejects with StreamingPreviewError when the stream ends without done', async () => {
apiFetch.mockResolvedValue(ndjsonResponse([startEvent(2), chunkEvent(0)]));
await expect(streamGenerateSpeech(new FormData(), {})).rejects.toThrow(StreamingPreviewError);
});
it('wraps a mid-body transport drop but lets pre-stream ApiError through untouched', async () => {
// Mid-body drop → StreamingPreviewError (fallback signal).
const enc = new TextEncoder();
apiFetch.mockResolvedValue({
headers: new Headers(),
body: new ReadableStream({
start(c) {
c.enqueue(enc.encode(JSON.stringify(startEvent(2)) + '\n'));
c.error(new TypeError('network dropped'));
},
}),
});
await expect(streamGenerateSpeech(new FormData(), {})).rejects.toThrow(StreamingPreviewError);
// Pre-stream HTTP failure → original error identity (NO fallback; the
// classic flow would fail identically).
const apiErr = Object.assign(new Error('400 Bad Request: nope'), { name: 'ApiError' });
apiFetch.mockRejectedValue(apiErr);
await expect(streamGenerateSpeech(new FormData(), {})).rejects.toBe(apiErr);
});
// #1330 — the take is real, but part of the text produced no audio. Driven
// through the actual NDJSON reader rather than by matching source text
// (CodeRabbit): a serialization the client cannot parse would pass the
// literal check and still leave the user with a silently short take.
it('forwards a warning frame to onWarning and still resolves with done', async () => {
const warned = [];
apiFetch.mockResolvedValue(
ndjsonResponse([
startEvent(3),
chunkEvent(0),
chunkEvent(1),
{ type: 'warning', code: 'dropped_chunks', count: 1, text: ['the tail that vanished.'] },
doneEvent,
]),
);
const meta = await streamGenerateSpeech(new FormData(), {
onWarning: (ev) => warned.push(ev),
});
// Not an error: the stream completed and the caller got its metadata.
expect(meta.id).toBe('abc12345');
expect(warned).toHaveLength(1);
expect(warned[0]).toMatchObject({ code: 'dropped_chunks', count: 1 });
expect(warned[0].text).toEqual(['the tail that vanished.']);
});
it('does not require the caller to handle warnings', async () => {
// A consumer that never passes onWarning must not crash on the new frame.
// total_chunks covers the chunk delivered plus the two that dropped —
// a payload the backend could actually emit (CodeRabbit).
apiFetch.mockResolvedValue(
ndjsonResponse([
startEvent(3),
chunkEvent(0),
{ type: 'warning', code: 'dropped_chunks', count: 2, text: [] },
doneEvent,
]),
);
await expect(streamGenerateSpeech(new FormData(), {})).resolves.toMatchObject({
id: 'abc12345',
});
});
});
// ── createStreamingChunkPlayer transport ────────────────────────────────────
describe('createStreamingChunkPlayer', () => {
it('opens at the PCM rate and falls back when that rate is unsupported', () => {
let player = createStreamingChunkPlayer({ label: 'x', sampleRate: 24000 });
expect(FakeAudioContext.constructorArgs).toEqual([{ sampleRate: 24000 }]);
expect(FakeAudioContext.instances[0].sampleRate).toBe(24000);
player.fail();
FakeAudioContext.constructorArgs = [];
FakeAudioContext.instances = [];
FakeAudioContext.rejectSampleRate = true;
player = createStreamingChunkPlayer({ label: 'x', sampleRate: 24000 });
expect(FakeAudioContext.constructorArgs).toEqual([{ sampleRate: 24000 }, undefined]);
expect(FakeAudioContext.instances[0].sampleRate).toBe(48000);
player.fail();
});
it('anchors the first chunk and waits past its scheduled end before disconnecting', () => {
vi.useFakeTimers();
try {
const onDone = vi.fn();
const player = createStreamingChunkPlayer({
label: 'x',
sampleRate: 24000,
onDone,
});
const ctx = FakeAudioContext.instances[0];
ctx.currentTime = 0.5; // the stream's first chunk arrived after context creation
player.appendPcm16Base64(b64Pcm(Array.from({ length: 24000 }, () => 1000)));
player.finalize();
const source = ctx.started[0];
expect(source.startedAt.when).toBeCloseTo(0.58, 3);
expect(getPlaybackTrack().currentTime).toBe(0);
ctx.currentTime = 1.58; // the scheduled buffer has just reached its end
vi.advanceTimersByTime(250);
expect(source.stopped).toBeUndefined();
expect(onDone).not.toHaveBeenCalled();
ctx.currentTime = 1.64; // allow the output renderer to drain the tail
vi.advanceTimersByTime(250);
expect(onDone).toHaveBeenCalledWith('ended');
expect(source.stopped).toBeUndefined();
expect(player.stopped).toBe(true);
} finally {
vi.useRealTimers();
}
});
it('preserves the crossfade when a late chunk is re-anchored', () => {
const ramp = vi.spyOn(FakeGainParam.prototype, 'linearRampToValueAtTime');
const player = createStreamingChunkPlayer({
label: 'x',
sampleRate: 24000,
crossfadeMs: 50,
});
try {
const ctx = FakeAudioContext.instances[0];
player.appendPcm16Base64(chunkEvent(0).pcm);
// The first 100 ms chunk is still playing when the next chunk arrives,
// but its planned start has passed and must be re-anchored.
ctx.currentTime = 0.15;
player.appendPcm16Base64(chunkEvent(1).pcm);
expect(ctx.started[1].startedAt.when).toBeCloseTo(0.17, 5);
expect(ramp).toHaveBeenCalledTimes(2);
expect(ramp.mock.calls.map(([target]) => target)).toEqual([0, 1]);
// Only 10 ms of the previous source remains after the new start.
expect(ramp.mock.calls[0][1]).toBeCloseTo(0.18, 5);
expect(ramp.mock.calls[1][1]).toBeCloseTo(0.18, 5);
} finally {
player.fail();
ramp.mockRestore();
}
});
it('keeps the opening audio when a late chunk follows an ended source', () => {
const ramp = vi.spyOn(FakeGainParam.prototype, 'linearRampToValueAtTime');
const player = createStreamingChunkPlayer({
label: 'x',
sampleRate: 24000,
crossfadeMs: 50,
});
try {
const ctx = FakeAudioContext.instances[0];
player.appendPcm16Base64(chunkEvent(0).pcm);
// The first 100 ms source was scheduled from 80 ms to 180 ms.
ctx.currentTime = 0.25;
player.appendPcm16Base64(chunkEvent(1).pcm);
expect(ctx.started[1].startedAt.when).toBeCloseTo(0.27, 5);
expect(ramp).not.toHaveBeenCalled();
} finally {
player.fail();
ramp.mockRestore();
}
});
it('supports seek within the buffered region (reschedules from the target)', () => {
const player = createStreamingChunkPlayer({
label: 'x',
sampleRate: 24000,
crossfadeMs: 0,
});
player.appendPcm16Base64(chunkEvent(0).pcm);
player.appendPcm16Base64(chunkEvent(1).pcm);
const ctx = FakeAudioContext.instances[0];
const before = ctx.started.length;
seekActivePlayback(0.15); // inside chunk 1 (0.1–0.2 s timeline)
expect(ctx.started.length).toBeGreaterThan(before); // rescheduled
const reseek = ctx.started[ctx.started.length - 1];
expect(reseek.startedAt.offset).toBeCloseTo(0.05, 3); // intra-chunk offset
expect(getPlaybackTrack().currentTime).toBeCloseTo(0.15, 3);
player.fail();
});
it('stop via the manager finishes the player and closes the context', () => {
const onDone = vi.fn();
const player = createStreamingChunkPlayer({
label: 'x',
sampleRate: 24000,
crossfadeMs: 50,
onDone,
});
player.appendPcm16Base64(chunkEvent(0).pcm);
stopActivePlayback();
expect(onDone).toHaveBeenCalledWith('stopped');
expect(player.stopped).toBe(true);
expect(FakeAudioContext.instances[0].state).toBe('closed');
});
});