Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio: read aloud starts speaking after the first sentence instead of after the whole reply (#16955)
Leo Borcherdingf5a7ff251962
diff --git a/studio/frontend/src/features/chat/adapters/studio-speech-synthesis-adapter.ts b/studio/frontend/src/features/chat/adapters/studio-speech-synthesis-adapter.tsindex 43c32a425ac..84815cd3342 100644--- a/studio/frontend/src/features/chat/adapters/studio-speech-synthesis-adapter.ts+++ b/studio/frontend/src/features/chat/adapters/studio-speech-synthesis-adapter.ts@@ -16,6 +16,8 @@ import { getExternalProviderApiKey } from "../external-providers";import { stripSearchImageTokens } from "../search-images/search-images";import { useExternalProvidersStore } from "../stores/external-providers-store";import { markdownToSpeechText } from "../utils/speech-text";+import { splitIntoSentences } from "../voice/speech-text";+import { planTtsChunk } from "../voice/tts-chunk-plan";/** "default" resolves to the platform's default voice; undefined lets the browser pick. */export function findTtsVoice(@@ -405,6 +407,20 @@ export function releaseTtsAudioUrl(url: string): void {if (url.startsWith("blob:")) URL.revokeObjectURL(url);}+/** Read-aloud chunks: one sentence first for a fast start, then sentences merged to a budget. */+export function readAloudChunks(text: string): string[] {+ const pending = splitIntoSentences(text);+ const chunks: string[] = [];+ while (pending.length > 0) {+ const take = Math.max(+ 1,+ planTtsChunk({ pending, first: chunks.length === 0, final: true, playing: false, waitedMs: 0 }),+ );+ chunks.push(pending.splice(0, take).join(" "));+ }+ return chunks;+}+function speakWithBackendAudio(generate: (text: string, signal?: AbortSignal) => Promise<string>,text: string,@@ -416,8 +432,10 @@ function speakWithBackendAudio(): { cancel: () => void } {const { ttsRate, ttsVolume } = useVoiceSettingsStore.getState();const controller = new AbortController();+ const chunks = readAloudChunks(text);let audio: HTMLAudioElement | null = null;let audioUrl: string | null = null;+ let next: Promise<string> | null = null;let cancelled = false;const cleanup = () => {@@ -432,13 +450,15 @@ function speakWithBackendAudio(}};- void (async () => {- try {- const url = await generate(text, controller.signal);- if (cancelled) {- releaseTtsAudioUrl(url);- return;- }+ const synth = (index: number): Promise<string> => {+ const pending = generate(chunks[index] ?? "", controller.signal);+ // Awaited later; a chunk dropped by cancel must not reject unhandled.+ pending.catch(() => {});+ return pending;+ };++ const play = (url: string) =>+ new Promise<void>((resolve, reject) => {audioUrl = url;audio = new Audio(url);audio.playbackRate = ttsRate;@@ -449,18 +469,45 @@ function speakWithBackendAudio(});audio.addEventListener("ended", () => {cleanup();- handleEnd("finished");+ resolve();});audio.addEventListener("error", () => {- if (cancelled) return;cleanup();- handleEnd("error", new Error("Audio playback failed."));+ reject(new Error("Audio playback failed."));});- markRunning();- await audio.play();+ audio.play().catch(reject);+ });++ void (async () => {+ try {+ let pendingChunk = synth(0);+ next = pendingChunk;+ for (let i = 0; i < chunks.length; i++) {+ const url = await pendingChunk;+ next = null;+ if (cancelled) {+ releaseTtsAudioUrl(url);+ return;+ }+ if (i + 1 < chunks.length) {+ pendingChunk = synth(i + 1);+ next = pendingChunk;+ }+ if (i === 0) markRunning();+ // A subscriber may cancel on the running notification.+ if (cancelled) {+ releaseTtsAudioUrl(url);+ return;+ }+ await play(url);+ if (cancelled) return;+ }+ handleEnd("finished");} catch (error) {if (cancelled || controller.signal.aborted) return;cleanup();+ controller.abort();+ next?.then(releaseTtsAudioUrl, () => {});handleEnd("error", error);}})();@@ -469,6 +516,7 @@ function speakWithBackendAudio(cancel: () => {cancelled = true;controller.abort();+ next?.then(releaseTtsAudioUrl, () => {});cleanup();handleEnd("cancelled");},diff --git a/studio/frontend/src/features/chat/voice/speech-text.ts b/studio/frontend/src/features/chat/voice/speech-text.tsnew file mode 100644index 00000000000..8a25937da00--- /dev/null+++ b/studio/frontend/src/features/chat/voice/speech-text.ts@@ -0,0 +1,62 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// A trailing period on these ends an abbreviation, not a sentence ("Dr.").+const SPEECH_ABBREVIATIONS = new Set([+ "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "mt", "vs", "etc", "eg",+ "ie", "approx", "dept", "est", "fig", "no", "vol", "inc", "ltd", "co",+ "jan", "feb", "mar", "apr", "jun", "jul", "aug", "sep", "sept", "oct",+ "nov", "dec",+]);++const SENTENCE_FINAL_ABBREVIATIONS = new Set(["etc", "inc", "ltd", "co", "jr", "sr"]);++const speakable = (s: string) => /[\p{L}\p{N}]/u.test(s);++export function splitIntoSentences(text: string): string[] {+ const norm = text+ .replace(/[^\S\n]+/g, " ")+ .replace(/\n{2,}/g, "\n")+ .trim();+ // A sentence ends at .!? (plus closing quotes) only before whitespace or the end, so "3.5",+ // ".env", "./train.py" and "D.C." stay whole; a line without one ends at the newline.+ const raw = norm.match(/\S[^\n]*?(?:[.!?]+["'”’»)\]]*(?=\s|$)|(?=\n|$))/g);+ if (!raw) return [text];+ const out: string[] = [];+ let lead = "";+ for (let part of raw) {+ if (lead) {+ part = `${lead} ${part}`;+ lead = "";+ }+ const prev = out[out.length - 1];+ // Punctuation alone ("...", "--") is no request of its own.+ if (!speakable(part)) {+ if (prev === undefined) lead = part;+ else out[out.length - 1] = `${prev} ${part}`;+ continue;+ }+ if (prev !== undefined) {+ const tail = prev.replace(/["'”’»)\]]+$/, "");+ const word = /([A-Za-z]+)\.$/.exec(tail)?.[1]?.toLowerCase();+ // "J. Smith", "John F. Kennedy" are names; "pick option A. Then" is a boundary.+ const nameInitial =+ /(?:^|(?:^|\s)(?:[A-Z][a-z]+|[A-Z]\.)\s)[A-Z]\.$/.test(tail) && /^[A-Z][a-z]/.test(part);+ const initial = /(^|\s)[A-Za-z]\.$/.test(tail) && /^[A-Za-z]\./.test(part);+ // "etc. Then I left": these also end sentences, so only a lowercase word or digit continues one.+ const abbrev =+ word !== undefined &&+ SPEECH_ABBREVIATIONS.has(word) &&+ (!SENTENCE_FINAL_ABBREVIATIONS.has(word) || /^[a-z\d]/.test(part));+ // "The U.S. is large", "e.g. the docs": a lowercase word after an initialism continues it.+ const initialism = /(?:^|\s)(?:[A-Za-z]\.){2,}$/.test(tail) && /^[a-z]/.test(part);+ if (nameInitial || initial || abbrev || initialism) {+ out[out.length - 1] = `${prev} ${part}`;+ continue;+ }+ }+ out.push(part);+ }+ if (lead) out.push(lead);+ return out;+}diff --git a/studio/frontend/src/features/chat/voice/tts-chunk-plan.ts b/studio/frontend/src/features/chat/voice/tts-chunk-plan.tsnew file mode 100644index 00000000000..c7ddbbbe939--- /dev/null+++ b/studio/frontend/src/features/chat/voice/tts-chunk-plan.ts@@ -0,0 +1,46 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// How many pending sentences the TTS player sends in one synth request. Each+// request costs a few seconds of fixed overhead on top of the audio itself, so+// one short sentence per request leaves a gap after every clip. The first chunk+// stays one sentence for a fast start; later ones merge sentences up to a+// character budget, never splitting a sentence.++export const TTS_CHUNK_TARGET_CHARS = 220;+export const TTS_CHUNK_MAX_CHARS = 320;+// A short chunk may wait for more sentences only this long while nothing plays.+export const TTS_CHUNK_WAIT_MS = 400;++export interface TtsChunkPlanInput {+ /** Sentences not yet requested, already stripped for speech, in order. */+ pending: string[];+ /** Nothing has been requested yet this stream. */+ first: boolean;+ /** The reply text is complete; no more sentences will arrive. */+ final: boolean;+ /** A clip is scheduled or audible. */+ playing: boolean;+ /** How long the head of `pending` has been held back waiting for more. */+ waitedMs: number;+}++/** Sentences to send now, from the head of `pending`; 0 means wait for more. */+export function planTtsChunk(input: TtsChunkPlanInput): number {+ const { pending, first, final, playing, waitedMs } = input;+ if (pending.length === 0) return 0;+ if (first) return 1;+ let take = 0;+ let chars = 0;+ for (const sentence of pending) {+ const add = chars === 0 ? sentence.length : sentence.length + 1;+ if (chars > 0 && (chars >= TTS_CHUNK_TARGET_CHARS || chars + add > TTS_CHUNK_MAX_CHARS)) break;+ chars += add;+ take++;+ }+ if (take < pending.length || chars >= TTS_CHUNK_TARGET_CHARS || final) return take;+ // The list ran dry under budget while the LLM is still writing: wait for+ // more, but an idle player must not sit on a sentence it could be speaking.+ if (!playing && waitedMs >= TTS_CHUNK_WAIT_MS) return take;+ return 0;+}diff --git a/studio/frontend/tests/read-aloud-sentence-chunks.test.ts b/studio/frontend/tests/read-aloud-sentence-chunks.test.tsnew file mode 100644index 00000000000..60c2aaf8a25--- /dev/null+++ b/studio/frontend/tests/read-aloud-sentence-chunks.test.ts@@ -0,0 +1,251 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// Read aloud on a backend engine speaks a reply sentence by sentence: the first sentence alone+// for a fast start, the next chunk synthesizing while the current one plays.++import assert from "node:assert/strict";+import test from "node:test";++import * as speechText from "../src/features/chat/voice/speech-text.ts";+import * as ttsChunkPlan from "../src/features/chat/voice/tts-chunk-plan.ts";+import * as modelCatalog from "../src/features/settings/api/openai-model-catalog.ts";+import * as formatError from "../src/lib/format-fastapi-error.ts";+import { loadWithStubs } from "./helpers/module-stubs.ts";++type Utterance = { status: { type: string }; cancel: () => void };+type Adapter = {+ readAloudChunks: (text: string) => string[];+ StudioSpeechSynthesisAdapter: new () => { speak: (text: string) => Utterance };+};++class FakeAudio {+ static live: FakeAudio[] = [];+ listeners: Record<string, () => void> = {};+ paused = false;+ playbackRate = 1;+ volume = 1;+ src: string;+ constructor(src: string) {+ this.src = src;+ FakeAudio.live.push(this);+ }+ addEventListener(type: string, fn: () => void) {+ this.listeners[type] = fn;+ }+ removeAttribute() {}+ pause() {+ this.paused = true;+ }+ async play() {}+ end() {+ this.listeners.ended?.();+ }+}++function load() {+ const spoken: string[] = [];+ const answers: ((v: unknown) => void)[] = [];+ const adapter = loadWithStubs<Adapter>(+ new URL("../src/features/chat/adapters/studio-speech-synthesis-adapter.ts", import.meta.url),+ {+ "@/features/auth": {+ authFetch: async (input: string, init?: { body?: string }) => {+ if (input === "/api/inference/loaded-models" || input === "/v1/models") return { ok: true, status: 200, json: async () => ({ data: [] }) };+ spoken.push(JSON.parse(init?.body ?? "{}").messages[0].content);+ await new Promise((resolve) => answers.push(resolve));+ return { ok: true, status: 200, json: async () => ({ audio: { data: String(spoken.length) } }) };+ },+ },+ "../../settings/api/openai-model-catalog": modelCatalog,+ "@/lib/format-fastapi-error": formatError,+ "../search-images/search-images": { stripSearchImageTokens: (text: string) => text },+ "../voice/speech-text": speechText,+ "../voice/tts-chunk-plan": ttsChunkPlan,+ "../utils/speech-text": { markdownToSpeechText: (text: string) => text },+ "../stores/external-providers-store": { useExternalProvidersStore: { getState: () => ({}) } },+ "../api/providers-api": { encryptProviderApiKey: async () => "" },+ "../external-providers": { getExternalProviderApiKey: () => "" },+ "@/features/settings/stores/voice-settings-store": {+ useVoiceSettingsStore: {+ getState: () => ({ ttsEngine: "studio", ttsStudioVoiceId: "", ttsRate: 1, ttsVolume: 1 }),+ },+ },+ "@/lib/toast": { toast: { error: () => {}, warning: () => {} } },+ "@/features/audio/audio-workspace-constants": { TTS_MAX_TOKENS: 1000 },+ },+ );+ return { adapter, spoken, answers };+}++const tick = async (n = 20) => {+ for (let i = 0; i < n; i++) await new Promise((resolve) => setTimeout(resolve, 0));+};++const REPLY =+ "Paris is the capital of France. It sits on the Seine. It has about two million people. " ++ "The Eiffel Tower was finished in 1889 for the World's Fair. Dr. Smith said it was 3.5 times taller than planned.";++test("the first chunk is one sentence; later ones merge up to the budget without splitting one", () => {+ const { adapter } = load();+ const chunks = adapter.readAloudChunks(REPLY);+ assert.equal(chunks[0], "Paris is the capital of France.");+ assert.ok(chunks.length >= 2);+ for (const chunk of chunks.slice(1)) assert.ok(chunk.length <= ttsChunkPlan.TTS_CHUNK_MAX_CHARS);+ assert.equal(chunks.join(" "), REPLY);+ assert.ok(chunks.some((c) => c.includes("Dr. Smith said it was 3.5 times")));+});++test("a one-sentence reply is one request, as before", () => {+ const { adapter } = load();+ assert.deepEqual(adapter.readAloudChunks("Done."), ["Done."]);+});++test("playback starts after the first chunk and the next one synthesizes while it plays", async () => {+ FakeAudio.live = [];+ (globalThis as { Audio?: unknown }).Audio = FakeAudio;+ const { adapter, spoken, answers } = load();+ const utterance = new adapter.StudioSpeechSynthesisAdapter().speak(REPLY);+ await tick();+ assert.deepEqual(spoken, ["Paris is the capital of France."]);+ answers.shift()?.(undefined);+ await tick();+ assert.equal(utterance.status.type, "running", "the first chunk plays before the rest exists");+ assert.equal(FakeAudio.live.length, 1);+ assert.equal(spoken.length, 2, "the second chunk is already synthesizing during playback");+ const total = adapter.readAloudChunks(REPLY).length;+ for (let i = 1; i < total; i++) {+ answers.shift()?.(undefined);+ await tick();+ assert.notEqual(utterance.status.type, "ended");+ FakeAudio.live[i - 1]?.end();+ await tick();+ }+ FakeAudio.live[total - 1]?.end();+ await tick();+ assert.equal(FakeAudio.live.length, total);+ assert.equal(utterance.status.type, "ended");+ assert.deepEqual(spoken, adapter.readAloudChunks(REPLY));+});++test("cancel stops playback and requests nothing further", async () => {+ FakeAudio.live = [];+ (globalThis as { Audio?: unknown }).Audio = FakeAudio;+ const { adapter, spoken, answers } = load();+ const utterance = new adapter.StudioSpeechSynthesisAdapter().speak(REPLY);+ await tick();+ answers.shift()?.(undefined);+ await tick();+ utterance.cancel();+ answers.shift()?.(undefined);+ await tick();+ assert.equal(FakeAudio.live[0]?.paused, true);+ assert.equal(FakeAudio.live.length, 1, "the chunk that landed after cancel never plays");+ assert.equal(spoken.length, 2);+ assert.equal(utterance.status.type, "ended");+});++test("an initialism before a lowercase word stays in its sentence", () => {+ assert.deepEqual(speechText.splitIntoSentences("The U.S. is large. It has fifty states."), [+ "The U.S. is large.",+ "It has fifty states.",+ ]);+ assert.deepEqual(speechText.splitIntoSentences("See e.g. the docs. Then run it."), [+ "See e.g. the docs.",+ "Then run it.",+ ]);+ assert.deepEqual(speechText.splitIntoSentences("See e.g., the docs. In the U.S., prices rose."), [+ "See e.g., the docs.",+ "In the U.S., prices rose.",+ ]);+ // A capital after one is a real boundary.+ assert.deepEqual(speechText.splitIntoSentences("Born in the U.S. He moved later."), [+ "Born in the U.S.",+ "He moved later.",+ ]);+});++test("a dot with no space after it (.75, .env) stays inside its token", () => {+ assert.deepEqual(speechText.splitIntoSentences("Set the threshold to .75. Then retry."), [+ "Set the threshold to .75.",+ "Then retry.",+ ]);+ assert.deepEqual(speechText.splitIntoSentences("It costs $.50 now. Use -.5 here."), ["It costs $.50 now.", "Use -.5 here."]);+ assert.deepEqual(speechText.splitIntoSentences("Run ./train.py to start. Then wait."), [+ "Run ./train.py to start.",+ "Then wait.",+ ]);+ assert.deepEqual(speechText.splitIntoSentences("10\n.75"), ["10", ".75"]);+ assert.deepEqual(speechText.splitIntoSentences("Use .env for the key. Then restart."), [+ "Use .env for the key.",+ "Then restart.",+ ]);+});++test("a subscriber that cancels on the running notification gets no playback", async () => {+ FakeAudio.live = [];+ (globalThis as { Audio?: unknown }).Audio = FakeAudio;+ const { adapter, answers } = load();+ const utterance = new adapter.StudioSpeechSynthesisAdapter().speak(REPLY) as Utterance & {+ subscribe: (cb: () => void) => () => void;+ };+ utterance.subscribe(() => {+ if (utterance.status.type === "running") utterance.cancel();+ });+ await tick();+ answers.shift()?.(undefined);+ await tick();+ assert.equal(FakeAudio.live.length, 0);+ assert.equal(utterance.status.type, "ended");+});++test("a closing quote ends its sentence without swallowing the next one", () => {+ assert.deepEqual(speechText.splitIntoSentences('The sign read "Stop." Then we left.'), [+ 'The sign read "Stop."',+ "Then we left.",+ ]);+ assert.deepEqual(speechText.splitIntoSentences("It ended (finally.) Good."), [+ "It ended (finally.)",+ "Good.",+ ]);+});++test("punctuation left after a sentence is never its own chunk", () => {+ assert.deepEqual(speechText.splitIntoSentences("He said \u201cHello.\u201d"), ["He said \u201cHello.\u201d"]);+ assert.deepEqual(speechText.splitIntoSentences('"Done.".'), ['"Done.".']);+ assert.deepEqual(speechText.splitIntoSentences("She said \u201cGo.\u201d Then we left."), [+ "She said \u201cGo.\u201d",+ "Then we left.",+ ]);+});++test("a dot inside a name like llama.cpp or config.json is not a boundary", () => {+ assert.deepEqual(speechText.splitIntoSentences("It runs in llama.cpp, Ollama and vLLM. Then go."), [+ "It runs in llama.cpp, Ollama and vLLM.",+ "Then go.",+ ]);+ assert.deepEqual(speechText.splitIntoSentences("Open config.json now. Visit unsloth.ai today."), [+ "Open config.json now.",+ "Visit unsloth.ai today.",+ ]);+});++test("an abbreviation that ends a sentence still ends it", () => {+ assert.deepEqual(speechText.splitIntoSentences("I brought apples, oranges, etc. Then I went home."), [+ "I brought apples, oranges, etc.",+ "Then I went home.",+ ]);+ assert.deepEqual(speechText.splitIntoSentences("Fruit, nuts, etc. are fine. Ask Dr. Smith. It is Python vs. Rust."), [+ "Fruit, nuts, etc. are fine.",+ "Ask Dr. Smith.",+ "It is Python vs. Rust.",+ ]);+});++test("an initial in a name stays with the surname", () => {+ assert.deepEqual(speechText.splitIntoSentences("J. Smith wrote it. Then he left."), ["J. Smith wrote it.", "Then he left."]);+ assert.deepEqual(speechText.splitIntoSentences("J. R. R. Tolkien wrote it."), ["J. R. R. Tolkien wrote it."]);+ assert.deepEqual(speechText.splitIntoSentences("Ask John F. Kennedy about it."), ["Ask John F. Kennedy about it."]);+ // After a lowercase word a single letter ends the sentence.+ assert.deepEqual(speechText.splitIntoSentences("Pick option A. Then run it."), ["Pick option A.", "Then run it."]);+});diff --git a/studio/frontend/tests/read-aloud-studio-voice.test.ts b/studio/frontend/tests/read-aloud-studio-voice.test.tsindex 1bbc01ee566..1238222d647 100644--- a/studio/frontend/tests/read-aloud-studio-voice.test.ts+++ b/studio/frontend/tests/read-aloud-studio-voice.test.ts@@ -9,6 +9,8 @@ import assert from "node:assert/strict";import { readdirSync } from "node:fs";import test from "node:test";+import * as speechText from "../src/features/chat/voice/speech-text.ts";+import * as ttsChunkPlan from "../src/features/chat/voice/tts-chunk-plan.ts";import { loadWithStubs } from "./helpers/module-stubs.ts";import * as modelCatalog from "../src/features/settings/api/openai-model-catalog.ts";@@ -70,6 +72,8 @@ function load("../search-images/search-images": {stripSearchImageTokens: (text: string) => text,},+ "../voice/speech-text": speechText,+ "../voice/tts-chunk-plan": ttsChunkPlan,"../utils/speech-text": {markdownToSpeechText: (text: string) => text,},diff --git a/studio/frontend/tests/tts-custom-connections-disabled.test.ts b/studio/frontend/tests/tts-custom-connections-disabled.test.tsindex 31237dcf005..317896ad837 100644--- a/studio/frontend/tests/tts-custom-connections-disabled.test.ts+++ b/studio/frontend/tests/tts-custom-connections-disabled.test.ts@@ -6,6 +6,8 @@import assert from "node:assert/strict";import test from "node:test";+import * as speechText from "../src/features/chat/voice/speech-text.ts";+import * as ttsChunkPlan from "../src/features/chat/voice/tts-chunk-plan.ts";import { loadWithStubs } from "./helpers/module-stubs.ts";import { readSrc } from "./helpers/kit.ts";@@ -100,6 +102,8 @@ function load("../search-images/search-images": {stripSearchImageTokens: (text: string) => text,},+ "../voice/speech-text": speechText,+ "../voice/tts-chunk-plan": ttsChunkPlan,"../utils/speech-text": {markdownToSpeechText: toSpeech,},@@ -177,14 +181,14 @@ test("custom TTS defaults a blank voice for strict OpenAI-compatible endpoints",test("read-aloud sends the speech text, not the markdown source (#12547)", async () => {const { adapter, posted } = load(true, {- toSpeech: (text) => `spoken(${text})`,+ toSpeech: (text) => `Spoken ${text.replaceAll("*", "")}`,});new adapter.StudioSpeechSynthesisAdapter().speak("**Done.**");// The model lookup adds a request before the speech one.for (let i = 0; i < 20 && posted.length === 0; i++) {await new Promise((resolve) => setTimeout(resolve, 0));}- assert.equal(JSON.parse(posted[0]).messages[0].content, "spoken(**Done.**)");+ assert.equal(JSON.parse(posted[0]).messages[0].content, "Spoken Done.");});test("custom TTS rejects a deleted provider even when Voice settings is unmounted", async () => {