VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
import { useState, useRef, useCallback } from 'react';
import { useAppStore } from '../store';
import { generateSpeech, TtsGenerationBusyError } from '../api/generate';
import { pickDesignSeed } from '../utils/seed';
import { generateAbortMs } from '../utils/generateBudget';
import { playBlobAudio, playPing } from '../utils/media';
import {
StreamingPreviewError,
shouldFallbackToClassic,
streamGenerateSpeech,
supportsStreamingPreview,
} from '../utils/streamingTts';
import { probeAudioDuration } from '../utils/format';
import { CLONE_MAX_SECONDS, PRESETS } from '../utils/constants';
import {
buildDesignInstruct,
designModeProfileId,
mergeDescribedAttrs,
} from '../utils/voiceInstruct';
import { toast } from 'react-hot-toast';
import { toastErrorWithReport } from '../utils/errorToast';
import { modelNotDownloadedPayload, toastModelNotDownloaded } from '../utils/modelNotDownloaded';
import { addBreadcrumb } from '../utils/breadcrumbs';
import i18next from 'i18next';
const t = i18next.t.bind(i18next);
// #21: in-memory de-dup for the synth-time routing toast — a 50-clip batch
// shouldn't fire one toast per request. Tracks the last status surfaced this
// session (module scope, no localStorage); resets on full reload.
let _lastRoutingStatus = null;
/**
* Encapsulates TTS generation logic, streaming response handling,
* audio ingestion (with trim gate), and preset/tag helpers.
*/
export default function useTTS({ selectedProfile, setSelectedProfile, loadHistory, profiles }) {
const text = useAppStore((s) => s.text);
const setText = useAppStore((s) => s.setText);
const language = useAppStore((s) => s.language);
const instruct = useAppStore((s) => s.instruct);
const refText = useAppStore((s) => s.refText);
const speed = useAppStore((s) => s.speed);
const steps = useAppStore((s) => s.steps);
const cfg = useAppStore((s) => s.cfg);
const denoise = useAppStore((s) => s.denoise);
const tShift = useAppStore((s) => s.tShift);
const posTemp = useAppStore((s) => s.posTemp);
const classTemp = useAppStore((s) => s.classTemp);
const layerPenalty = useAppStore((s) => s.layerPenalty);
const postprocess = useAppStore((s) => s.postprocess);
const duration = useAppStore((s) => s.duration);
const vdStates = useAppStore((s) => s.vdStates);
// Which "Define voice" method is active in the Voice (studio) workspace —
// 'audio' (reference clip) vs 'design' (described attributes). Replaces the
// old clone/design navigation-mode checks (voice-studio-unification P4).
const defineMethod = useAppStore((s) => s.defineMethod);
const setSidebarTab = useAppStore((s) => s.setSidebarTab);
// Voice-design seed (#526): reuse the pinned seed when "keep this seed" is on
// so tweaks stay on the same base timbre; otherwise roll a fresh one.
const keepSeed = useAppStore((s) => s.keepSeed);
const designSeed = useAppStore((s) => s.designSeed);
const setDesignSeed = useAppStore((s) => s.setDesignSeed);
const [refAudio, setRefAudio] = useState(null);
const [pendingTrimFile, setPendingTrimFile] = useState(null);
const [isGenerating, setIsGenerating] = useState(false);
const [generationTime, setGenerationTime] = useState(0);
// Real 0–100 progress when the active delivery path can measure it.
// null means the backend has not supplied a meaningful fraction yet.
const [generationProgress, setGenerationProgress] = useState(null);
const timerRef = useRef(null);
const textAreaRef = useRef(null);
const ingestRefAudio = useCallback(
async (file) => {
if (!file) {
setRefAudio(null);
return;
}
const dur = await probeAudioDuration(file);
if (dur && dur > CLONE_MAX_SECONDS) {
setPendingTrimFile(file);
setSelectedProfile(null);
toast(t('tts_errors.trim_hint', { duration: dur.toFixed(1), max: CLONE_MAX_SECONDS }));
return;
}
setRefAudio(file);
setSelectedProfile(null);
},
[setSelectedProfile],
);
const insertTag = useCallback(
(tag) => {
if (!textAreaRef.current) return;
const start = textAreaRef.current.selectionStart;
const end = textAreaRef.current.selectionEnd;
setText(text.substring(0, start) + tag + text.substring(end));
setTimeout(() => {
textAreaRef.current.focus();
textAreaRef.current.setSelectionRange(start + tag.length, start + tag.length);
}, 0);
},
[text, setText],
);
const applyPreset = useCallback(
(preset) => {
// #1771: run every "replace the whole vdStates" entry point through the
// same accent/dialect exclusivity guard the live picker uses — today's
// hardcoded PRESETS never set both, but a future one (or a hand-edited
// constants.js) shouldn't be able to rebuild the conflict the engine
// rejects.
useAppStore.getState().setVdStates(mergeDescribedAttrs(preset.attrs));
if (preset.tags && !text.includes(preset.tags.trim())) insertTag(preset.tags);
},
[text, insertTag],
);
const handleGenerate = useCallback(async () => {
if (!text.trim()) return toast.error(t('tts_errors.enter_text'));
if (defineMethod === 'audio' && !refAudio && !selectedProfile)
return toast.error(t('tts_errors.upload_or_select'));
addBreadcrumb(`generate:start (${defineMethod})`);
setIsGenerating(true);
setGenerationTime(0);
setGenerationProgress(null);
const st = Date.now();
timerRef.current = setInterval(() => {
const elapsed = ((Date.now() - st) / 1000).toFixed(1);
setGenerationTime(elapsed);
}, 100);
let abortTimer = null;
try {
const formData = new FormData();
formData.append('text', text);
if (language !== 'Auto') formData.append('language', language);
formData.append('num_step', steps);
formData.append('guidance_scale', cfg);
formData.append('speed', speed);
formData.append('denoise', denoise);
formData.append('t_shift', tShift);
formData.append('position_temperature', posTemp);
formData.append('class_temperature', classTemp);
formData.append('layer_penalty_factor', layerPenalty);
formData.append('postprocess_output', postprocess);
if (duration) formData.append('duration', parseFloat(duration));
if (defineMethod === 'audio') {
if (selectedProfile) {
formData.append('profile_id', selectedProfile);
} else if (refAudio) {
const arrBuf = await refAudio.arrayBuffer();
const safeBlob = new Blob([arrBuf], { type: refAudio.type });
formData.append('ref_audio', safeBlob, refAudio.name || 'audio.wav');
formData.append('ref_text', refText);
}
// #612: a clone's free-text style field also passes through the backend
// instruct whitelist, so raw prose (e.g. a non-EN/ZH description like the
// Vietnamese report) 400s with "Unsupported instruct items". Apply the
// SAME validator-safe guard the design path uses: keep valid style tags,
// drop the rest, and surface a localized warning toast — never round-trip
// a 400. vdStates is empty here (clone has no design sliders).
if (instruct) {
const {
instruct: safeInstruct,
unsupported,
duplicates,
conflicts,
} = buildDesignInstruct({}, instruct);
if (unsupported.length) {
toast(t('tts_errors.ignored_unsupported', { items: unsupported.join(', ') }), {
icon: '⚠️',
});
}
if (duplicates.length) {
toast(t('tts_errors.ignored_duplicate', { items: duplicates.join(', ') }), {
icon: '⚠️',
});
}
// #1771: a Chinese dialect and an English accent typed together
// (e.g. into a clone's free-text style field) — the engine rejects
// the combination outright, so drop the loser client-side instead of
// round-tripping a 400.
if (conflicts.length) {
toast(t('tts_errors.ignored_conflict', { items: conflicts.join(', ') }), {
icon: '⚠️',
});
}
if (safeInstruct) formData.append('instruct', safeInstruct);
}
} else {
// #526: reuse the pinned seed when "keep this seed" is on (stable
// tweaks), else roll a fresh one. The backend echoes the seed it used
// back via X-Seed so the UI can show + pin it.
formData.append('seed', pickDesignSeed(keepSeed, designSeed));
// plan-05 (#132): build a validator-safe instruct (one valid tag per
// category; drop unsupported free-text) so Synthesize stops failing
// with "Unsupported instruct items" (#115) / "conflicting items within
// the same category" (#114).
const {
instruct: finalInstruct,
unsupported,
duplicates,
conflicts,
} = buildDesignInstruct(vdStates, instruct);
if (unsupported.length) {
toast(t('tts_errors.ignored_unsupported', { items: unsupported.join(', ') }), {
icon: '⚠️',
});
}
if (duplicates.length) {
toast(t('tts_errors.ignored_duplicate', { items: duplicates.join(', ') }), {
icon: '⚠️',
});
}
// #1771: backstop for a dialect+accent pick that reached here anyway
// (the picker and vdStates-restore paths already prevent this) — drop
// the loser instead of letting the engine 400 on it.
if (conflicts.length) {
toast(t('tts_errors.ignored_conflict', { items: conflicts.join(', ') }), {
icon: '⚠️',
});
}
if (finalInstruct) formData.append('instruct', finalInstruct);
// #674: in design mode, never forward a CLONE profile_id — its reference
// voice would override the design attributes (e.g. "Male" has no effect).
// Design profiles still pass through (re-render a designed voice).
const designProfileId = designModeProfileId(selectedProfile, profiles);
if (designProfileId) {
formData.append('profile_id', designProfileId);
}
}
// The backend bounds every phase of a generate and returns a descriptive
// error; this abort only catches a backend gone silent, so it outlasts
// the backend's longest legitimate budget (see generateBudget.ts).
const ac = new AbortController();
abortTimer = setTimeout(() => ac.abort(), generateAbortMs(text.length));
// #1330 — one voice for both delivery paths. A dropped chunk is not an
// error (the audio is real), so it is a persistent-ish warning toast
// rather than a thrown failure, and it quotes the lost text so the user
// can tell at a glance what to re-render.
const announceDroppedText = (count, sample) => {
const preview = (sample || '').trim().slice(0, 120);
toast(
preview
? t('tts.droppedChunksWithText', { count, text: preview })
: t('tts.droppedChunks', { count }),
{ icon: '\u26a0\ufe0f', duration: 8000 },
);
};
// Header handling shared by both delivery paths (streaming + classic).
const applyResponseHeaders = (response) => {
// #526: surface the seed the backend actually used so the Design tab
// can display it and offer "keep this seed". Authoritative over the
// client guess (covers the profile-seed / materialized-seed paths too).
const xSeed = parseInt(response.headers.get('X-Seed') || '', 10);
if (Number.isInteger(xSeed)) setDesignSeed(xSeed);
// #21: one-time, non-blocking routing notice. The backend sets these
// headers only on cpu_fallback / accelerated-with-caveat (never on the
// benign cpu_only / clean-accelerated paths), so their mere presence is
// the signal. De-duped by status so a batch doesn't spam.
// #1330: some of the text rendered to no audio. The take that came
// back is clean and simply short, so nothing else would ever tell the
// user — they reported it by reading along. Classic path only; the
// streaming path learns this from a `warning` frame (headers are sent
// before the render starts).
const droppedCount = parseInt(response.headers.get('X-OmniVoice-Dropped-Chunks') || '', 10);
if (Number.isInteger(droppedCount) && droppedCount > 0) {
announceDroppedText(droppedCount, response.headers.get('X-OmniVoice-Dropped-Text'));
}
const routingStatus = response.headers.get('X-OmniVoice-Routing');
if (routingStatus && routingStatus !== _lastRoutingStatus) {
_lastRoutingStatus = routingStatus;
const reason = response.headers.get('X-OmniVoice-Routing-Reason') || '';
if (routingStatus === 'cpu_fallback') {
toast(t('tts.routingFallback', { reason }), { icon: '🐢' });
} else if (routingStatus === 'accelerated' && reason) {
// accelerated is only surfaced WITH a driver/arch caveat reason.
toast(t('tts.routingCaveat', { reason }), { icon: '⚠️' });
}
}
};
const setProgressPct = (pct) => setGenerationProgress(pct);
// Streaming preview (feat: streaming-tts-preview): playback starts from
// the FIRST synthesized chunk while the rest is still rendering, via
// /generate stream=true — instead of staring at the spinner until the
// whole render lands. Only when the preview would auto-play anyway, and
// only when Web Audio exists (all three Tauri webviews have it; if it's
// ever missing we take the classic path below — identical to today).
// Any MID-stream failure falls back to the classic whole-file flow with
// no user-visible difference beyond the old wait; pre-stream HTTP errors
// (ApiError) throw straight to the shared catch, exactly like before.
//
let streamed = false;
const wantsStreaming = useAppStore.getState().autoPlayPreview && supportsStreamingPreview();
if (wantsStreaming) {
try {
await streamGenerateSpeech(formData, {
signal: ac.signal,
label: t('player.streaming_preview'),
finalLabel: t('player.generated_audio'),
onHeaders: applyResponseHeaders,
onProgress: setProgressPct,
onWarning: (ev) => {
if (ev?.code === 'dropped_chunks' && ev.count > 0)
announceDroppedText(ev.count, (ev.text || []).join(' | '));
},
});
streamed = true;
} catch (err) {
if (!(err instanceof StreamingPreviewError)) throw err;
// #1190: a RETRYABLE mid-stream failure (GPU timeout / saturated
// pool) must not trigger the classic re-render. The backend already
// spent the full budget on this text, and the abandoned job keeps
// holding the device until it drains — re-synthesizing the whole
// text right now makes the user wait out a second timeout to see the
// same error. Surface the backend's actionable message instead.
// Non-retryable failures (transport drop, Web Audio glitch) keep the
// classic fallback: there the whole-file path genuinely can succeed.
if (!shouldFallbackToClassic(err)) {
addBreadcrumb('generate:stream-retryable-abort');
throw err;
}
console.warn(
'Streaming preview failed mid-stream; falling back to the classic generate:',
err?.message || err,
);
setGenerationProgress(null);
addBreadcrumb('generate:stream-fallback');
}
}
if (!streamed) {
const response = await generateSpeech(formData, { signal: ac.signal });
const reader = response.body.getReader();
const chunks = [];
let receivedLength = 0;
const contentLength = parseInt(response.headers.get('Content-Length') || '0', 10);
applyResponseHeaders(response);
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
receivedLength += value.length;
if (contentLength > 0) {
setProgressPct(Math.round((receivedLength / contentLength) * 100));
}
}
const blob = new Blob(chunks, { type: 'audio/wav' });
// #1032: honor the Settings → Appearance "Auto-play preview" pref here
// too. #667 added the toggle ("play the output as soon as a render
// finishes") but only wired the WaveformPlayer preview sites — the main
// generate path kept auto-playing unconditionally. getState() (not a
// subscription) so the freshest value is read at completion time.
if (useAppStore.getState().autoPlayPreview) {
try {
await playBlobAudio(blob, { label: t('player.generated_audio') });
} catch (e) {}
}
}
await loadHistory();
setSidebarTab('history');
playPing();
} catch (err) {
// Timeouts are user-recoverable (retry / shorter input) — plain toast.
// Real generation failures get the "Report this bug" action.
if (err instanceof TtsGenerationBusyError) {
toast(t('tts_errors.generation_in_progress'), { icon: '⏳' });
} else if (err?.name === 'AbortError') {
toast.error(t('tts_errors.timeout'));
} else if (modelNotDownloadedPayload(err)) {
toastModelNotDownloaded(modelNotDownloadedPayload(err));
} else {
toastErrorWithReport(t('tts_errors.error_prefix', { message: err.message }), err);
}
} finally {
if (abortTimer) clearTimeout(abortTimer);
clearInterval(timerRef.current);
setIsGenerating(false);
setGenerationProgress(null);
}
}, [
text,
defineMethod,
selectedProfile,
refAudio,
refText,
language,
instruct,
steps,
cfg,
speed,
denoise,
tShift,
posTemp,
classTemp,
layerPenalty,
postprocess,
duration,
vdStates,
keepSeed,
designSeed,
setDesignSeed,
loadHistory,
setSidebarTab,
]);
return {
refAudio,
setRefAudio,
pendingTrimFile,
setPendingTrimFile,
isGenerating,
generationTime,
generationProgress,
textAreaRef,
ingestRefAudio,
insertTag,
applyPreset,
handleGenerate,
};
}