Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: show the voice description on Clone for Breeze TTS 2 and VoxCPM2 (#15878)

Etherlee304893acd5
diff --git a/studio/backend/core/inference/audio_cpp_backend.py b/studio/backend/core/inference/audio_cpp_backend.py
index d2042a164f0..affef98b8c5 100644
--- a/studio/backend/core/inference/audio_cpp_backend.py
+++ b/studio/backend/core/inference/audio_cpp_backend.py
@@ -194,6 +194,9 @@ def _reread_after_download(model: AudioCppModel, hf_token: Optional[str]) -> Aud
# Families that read the request's top-level language, not an option (audio.cpp extract_request_language:
# VoxCPM1 keeps Cantonese text Traditional with zh-HK; Supertonic generation_options_from_request).
_READS_TOP_LEVEL_LANGUAGE = frozenset({"voxcpm1", "audio8_tts", "supertonic"})
+# Families that read a voice description only as a leading "(description)" on the text
+# (audio.cpp voxcpm2 split_leading_voice_tag); they read no instruct option.
+_DESCRIPTION_IN_TEXT = frozenset({"voxcpm2"})
# Supertonic's tokenizer refuses anything but a bare ISO code (src/models/supertonic/tokenizer_text.cpp).
_LANGUAGE_CODE_FAMILIES = frozenset({"supertonic"})
# English names of the languages Supertonic reads (model_specs/supertonic.json).
@@ -274,9 +277,30 @@ def clone_rules(model: AudioCppModel) -> Optional[dict[str, Any]]:
[name, list(values)] for name, values in clone.reference_text_waived
],
"emotion_audio": clone.emotion_audio,
+ "description": clone_description(model) is not None,
}
+def clone_description(model: AudioCppModel) -> Optional[str]:
+ """How a Clone run delivers a voice description: "text" (a leading tag), "instructions", or None."""
+ clone = model.clone
+ if clone is None or not clone.description:
+ return None
+ if model.family in _DESCRIPTION_IN_TEXT:
+ return "text"
+ # The server hands top-level instructions to the session as the instruction option.
+ keys = model.request_keys
+ return "instructions" if keys is None or "instruction" in keys else None
+
+
+def with_description(text: str, description: Optional[str]) -> str:
+ """``text`` behind a "(description)" tag; a text that opens with its own tag keeps it."""
+ tag = " ".join(str(description or "").replace("(", " ").replace(")", " ").split())
+ if not tag or (text or "").lstrip().startswith("("):
+ return text
+ return f"({tag}){text}"
+
+
def edit_rules(model: AudioCppModel) -> Optional[dict[str, Any]]:
edit = getattr(model, "edit", None)
if edit is None:
@@ -995,7 +1019,9 @@ class AudioCppBackend:
requested: dict[str, Any] = {}
# A strict contract refuses undeclared options; the retry would then drop the description too.
keys = model.request_keys
- if instructions and str(instructions).strip():
+ if instructions and str(instructions).strip() and model.family in _DESCRIPTION_IN_TEXT:
+ body["input"] = with_description(body["input"], str(instructions).strip())
+ elif instructions and str(instructions).strip():
# Breeze and Irodori v4 call it instruction, DramaBox has neither.
name = next((n for n in ("instruct", "instruction") if keys is None or n in keys), None)
if name:
@@ -1080,6 +1106,10 @@ class AudioCppBackend:
request_options = {name: _option_string(value) for name, value in options.items()}
body: dict[str, Any] = {"model": server.model_id}
spoken = speech_input(model, text)
+ description = str(instructions or "").strip()
+ delivery = clone_description(model) if description else None
+ if delivery == "text":
+ spoken = with_description(spoken, description)
if emotion:
body["text"] = spoken
else:
@@ -1100,8 +1130,8 @@ class AudioCppBackend:
body["language"] = chosen
if clone.speed and speed is not None:
body["speed"] = max(_MIN_SPEED, min(_MAX_SPEED, float(speed)))
- if clone.instructions and instructions and str(instructions).strip():
- body["instructions"] = str(instructions).strip()
+ if description and (clone.instructions or delivery == "instructions"):
+ body["instructions"] = description
if request_options:
body["options"] = request_options
if seed is not None:
diff --git a/studio/backend/core/inference/audio_cpp_models.py b/studio/backend/core/inference/audio_cpp_models.py
index c98ab946532..342cc93d246 100644
--- a/studio/backend/core/inference/audio_cpp_models.py
+++ b/studio/backend/core/inference/audio_cpp_models.py
@@ -91,6 +91,8 @@ class CloneSpec:
language_names: bool = False
speed: bool = False
instructions: bool = False
+ # Reads a free voice or style description beside the reference (Breeze, VoxCPM2).
+ description: bool = False
@dataclass(frozen = True)
@@ -556,9 +558,10 @@ _CLONE_FAMILIES: tuple[AudioCppFamily, ...] = (
instructions = True,
),
),
- AudioCppFamily("voxcpm2", "tts", clone = CloneSpec("optional")),
+ AudioCppFamily("voxcpm2", "tts", clone = CloneSpec("optional", description = True)),
AudioCppFamily("fish_audio", "tts", clone = CloneSpec("required")),
- AudioCppFamily("breeze_tts", "tts", clone = CloneSpec("required")),
+ AudioCppFamily("breeze_tts", "tts", clone = CloneSpec("required", description = True)),
+ # Its clone instruction takes only fixed attribute words; free text fails the run.
AudioCppFamily("omnivoice", "tts", clone = CloneSpec("required")),
AudioCppFamily("voxcpm1", "tts", clone = CloneSpec("required")),
AudioCppFamily("dots_tts", "tts", clone = CloneSpec("optional")),
@@ -566,6 +569,7 @@ _CLONE_FAMILIES: tuple[AudioCppFamily, ...] = (
# MOSS ignores a transcript; Irodori refuses one ("unknown Irodori-TTS request option").
AudioCppFamily("moss_tts_local", "tts", clone = CloneSpec("unused")),
AudioCppFamily("moss_tts_nano", "tts", clone = CloneSpec("unused")),
+ # No description: the 500M v3 checkpoint refuses one (no caption conditioning weights).
AudioCppFamily("irodori_tts", "tts", clone = CloneSpec("unused")),
AudioCppFamily("echo_tts", "tts", server_task = "clon", speaks = False, clone = CloneSpec("unused")),
AudioCppFamily(
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index e9530d9d508..6155d610807 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1706,6 +1706,10 @@ class _InferenceRuntimeFields(BaseModel):
audio_convert_route: Optional[str] = Field(
None, description = "Seed-VC route the running server was started with."
)
+ audio_clone: Optional[Dict[str, Any]] = Field(
+ None,
+ description = "Clone rules (transcript, emotion clip, voice description); None when it does not clone.",
+ )
@model_validator(mode = "after")
def derive_audio_workflows(self):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 3b3327c90e6..7ed08280275 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8328,6 +8328,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
audio_server_task = None,
audio_convert = None,
audio_convert_route = None,
+ audio_clone = None,
audio_music = None,
# Older/custom backend doubles predate this additive runtime field.
preserve_thinking_default = bool(getattr(llama_backend, "preserve_thinking_default", False)),
@@ -19847,6 +19848,7 @@ async def _load_model_impl(
audio_server_task = _model_info.get("audio_server_task"),
audio_convert = _model_info.get("audio_convert"),
audio_convert_route = _model_info.get("audio_convert_route"),
+ audio_clone = _model_info.get("audio_clone"),
audio_required_inputs = _model_info.get("audio_required_inputs"),
audio_request_inputs = _model_info.get("audio_request_inputs"),
audio_device_fallback = _model_info.get("audio_device_fallback"),
@@ -22411,6 +22413,7 @@ async def _slot_status(current_subject: str):
audio_server_task = model_info.get("audio_server_task"),
audio_convert = model_info.get("audio_convert"),
audio_convert_route = model_info.get("audio_convert_route"),
+ audio_clone = model_info.get("audio_clone"),
audio_reference_text = model_info.get("audio_reference_text"),
audio_required_inputs = model_info.get("audio_required_inputs"),
audio_request_inputs = model_info.get("audio_request_inputs"),
diff --git a/studio/backend/tests/test_audio_cpp_clone_requests.py b/studio/backend/tests/test_audio_cpp_clone_requests.py
index 07e078c2254..6bab05b2147 100644
--- a/studio/backend/tests/test_audio_cpp_clone_requests.py
+++ b/studio/backend/tests/test_audio_cpp_clone_requests.py
@@ -77,7 +77,13 @@ MODELS = {
"omnivoice_speed": lambda: _model("omnivoice", "OmniVoice-GGUF", options = SPEED_OPTIONS),
"dots": lambda: _model("dots_tts", "DotTTS-SOAR-GGUF"),
"irodori": lambda: _model("irodori_tts", "Irodori-TTS-v4-Small-GGUF"),
+ "breeze": lambda: replace(
+ _model("breeze_tts", "Breeze-TTS-2-GGUF"),
+ request_keys = frozenset(STRICT_KEYS["Breeze-TTS-2-GGUF"][1]),
+ ),
+ "fire": lambda: _model("fireredtts3", "FireRedTTS3-GGUF"),
}
+DESCRIBE = "Calm and slow"
class _Recorder:
@@ -155,6 +161,18 @@ SPEECH, TASKS = "/v1/audio/speech", "/v1/tasks/run"
("dots", {}, BASE),
# Irodori refuses a transcript as an unknown option.
("irodori", {"reference_text": T}, BASE),
+ # A voice description goes where the family reads it, beside the reference.
+ ("breeze", {"reference_text": T, "instructions": f" {DESCRIBE} "},
+ {**BASE, "reference_text": T, "instructions": DESCRIBE}),
+ ("vox", {"instructions": DESCRIBE}, {**BASE, "input": f"({DESCRIBE}){TEXT}"}),
+ ("vox", {"instructions": " "}, BASE),
+ # Families that read none (or read it as something else) get the body they always got.
+ ("omnivoice", {"reference_text": T, "instructions": DESCRIBE}, {**BASE, "reference_text": T}),
+ ("irodori", {"instructions": DESCRIBE}, BASE),
+ ("dots", {"instructions": DESCRIBE}, BASE),
+ ("fire", {"reference_text": T, "instructions": DESCRIBE}, {**BASE, "reference_text": T}),
+ ("qwen3", {"reference_text": T, "instructions": DESCRIBE}, {"instructions": ABSENT, "input": TEXT}),
+ ("index", {"instructions": DESCRIBE}, {"instructions": ABSENT, "input": TEXT}),
])
# fmt: on
def test_clone_request_body(key, kwargs, expect):
@@ -173,19 +191,63 @@ def test_a_top_level_speed_reaches_a_speak_model_as_its_speed_option():
assert [body["options"] for _, body in server.calls] == [{"speed": 1.5}, {"speed": 1.0}]
-def test_voxcpm2_speak_body_is_unchanged_by_clone_support():
+def test_voxcpm2_speak_sends_the_description_as_a_text_prefix():
+ # VoxCPM2 reads no instruct option, only a leading "(description)" on the text.
backend, server = _backend(MODELS["vox"]())
- backend.generate_audio_response(TEXT, seed = 7, language = "en", instructions = "warm")
- body = {"model": "studio-test", "input": TEXT, "options": {"instruct": "warm", "language": "en"}}
+ backend.generate_audio_response(TEXT, seed = 7, language = "en", instructions = " warm ")
+ body = {"model": "studio-test", "input": f"(warm){TEXT}", "options": {"language": "en"}}
assert server.calls == [(SPEECH, {**body, "seed": "7"})]
+# fmt: off
[email protected]("text, description, sent", [
+ (TEXT, "warm", f"(warm){TEXT}"),
+ (TEXT, "a (very) warm\nvoice", f"(a very warm voice){TEXT}"),
+ (TEXT, " ( ) ", TEXT),
+ (TEXT, None, TEXT),
+ # The writer's own tag wins over the field.
+ (f" (whisper){TEXT}", "warm", f" (whisper){TEXT}"),
+])
+# fmt: on
+def test_with_description_keeps_an_inline_tag(text, description, sent):
+ assert acb.with_description(text, description) == sent
+
+
[email protected]("key, sent", [("breeze", True), ("vox", True), ("omnivoice", False)])
+def test_speak_in_a_saved_voice_sends_the_description(key, sent):
+ # A saved voice sends its reference with workflow "speak", so it takes the clone path.
+ backend, server = _backend(MODELS[key]())
+ backend.generate_audio_response(TEXT, audio_inputs = {"reference": REF}, instructions = DESCRIBE)
+ ((_path, body),) = server.calls
+ assert ("instructions" in body or body["input"].startswith(f"({DESCRIBE})")) is sent
+
+
+def test_clone_description_follows_the_contract():
+ assert [acb.clone_description(MODELS[k]()) for k in ("breeze", "vox")] == ["instructions", "text"]
+ for key in ("cosy", "index", "omnivoice", "irodori", "fire", "qwen3", "dots", "chatterbox", "kokoro"):
+ assert acb.clone_description(MODELS[key]()) is None, key
+ # A lenient contract takes it; a strict one that lacks instruction gets none.
+ lenient = replace(MODELS["breeze"](), request_keys = None)
+ strict = replace(MODELS["breeze"](), request_keys = frozenset({"reference_text", "seed"}))
+ assert (acb.clone_description(lenient), acb.clone_description(strict)) == ("instructions", None)
+ backend, server = _backend(strict)
+ backend.generate_audio_response(
+ TEXT, workflow = "clone", audio_inputs = {"reference": REF}, reference_text = T, instructions = DESCRIBE
+ )
+ assert "instructions" not in server.calls[0][1]
+ # A family Unsloth Studio does not list never offers one.
+ spec = {"family": "new_tts", "tasks": ["tts", "clone"]}
+ assert acb.clone_description(_model("new_tts", spec = spec)) is None
+
+
class _StrictRecorder(_Recorder):
"""Refuses an option the model's contract does not declare, as the runtime does."""
def post_json(self, path, payload, **kwargs):
keys = self.model.request_keys
- undeclared = sorted(set(payload.get("options") or {}) - set(keys or ()))
+ # The server hands top-level instructions to the session as the instruction option.
+ sent = set(payload.get("options") or {}) | ({"instruction"} if "instructions" in payload else set())
+ undeclared = sorted(sent - set(keys or ()))
if keys is not None and undeclared:
self.calls.append((path, json.loads(json.dumps(payload))))
raise srv.AudioCppRequestError(500, f"unknown request option: {undeclared[0]}")
@@ -315,6 +377,9 @@ def test_status_fields_name_the_clone_workflow_and_transcript_rule():
assert (f["audio_workflows"], f["audio_reference_text"]) == (workflows, rt)
cosy = acb.model_info_fields(MODELS["cosy"]())["audio_clone"]
assert cosy["reference_text_waived"] == [["template_name", ["cross_lingual", "instruct"]]]
+ # CosyVoice keeps its own Instruct mode, so only Breeze and VoxCPM2 get the Clone field.
+ for key, described in (("breeze", True), ("vox", True), ("cosy", False), ("omnivoice", False), ("fire", False)):
+ assert acb.model_info_fields(MODELS[key]())["audio_clone"]["description"] is described, key
instruct = {"name": "instruct", "type": "string", "required": True}
maya = _model("maya1", "Maya1-GGUF", {"tasks": ["tts"], "options": {"request": [instruct]}})
assert acb.model_info_fields(maya)["audio_required_inputs"] == ["instruct"]
diff --git a/studio/backend/tests/test_audio_workflows.py b/studio/backend/tests/test_audio_workflows.py
index 991d0cd603a..57853041482 100644
--- a/studio/backend/tests/test_audio_workflows.py
+++ b/studio/backend/tests/test_audio_workflows.py
@@ -128,6 +128,12 @@ def test_status_carries_the_clone_fields():
clone = {"audio_workflows": ["clone"], "audio_reference_text": "required"}
status = InferenceStatusResponse(is_audio = True, audio_type = "audiocpp_tts", **clone)
assert status.model_dump().items() >= clone.items()
+ # The Clone page shows the voice description from these rules.
+ rules = {"reference_text": "required", "description": True}
+ fields = dict(is_audio = True, audio_type = "audiocpp_tts", audio_clone = rules)
+ assert InferenceStatusResponse(**fields).model_dump()["audio_clone"] == rules
+ load = LoadResponse(status = "loaded", model = "m", display_name = "m", inference = {}, **fields)
+ assert load.audio_clone == rules
def test_status_and_load_carry_the_convert_fields():
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index ead77c40aca..fc679d4cf3f 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1226,6 +1226,7 @@ class TestLoadHubDownloadExclusion:
"audio_server_task",
"audio_convert",
"audio_convert_route",
+ "audio_clone",
"audio_music",
}
unresolved = sorted(
diff --git a/studio/frontend/src/features/audio/components/instructions-fields.tsx b/studio/frontend/src/features/audio/components/instructions-fields.tsx
index 5f103b93112..c30f5566723 100644
--- a/studio/frontend/src/features/audio/components/instructions-fields.tsx
+++ b/studio/frontend/src/features/audio/components/instructions-fields.tsx
@@ -10,12 +10,17 @@ export function InstructionsField({
instructionsKind,
musicNeedsDescription,
voiceRequired = false,
+ cloning = false,
+ disabled = false,
audioInstructions,
setAudioInstructions,
}: {
instructionsKind: NativeAudioInstructionsKind;
musicNeedsDescription: boolean;
voiceRequired?: boolean;
+ /** Clone's copy: the description shapes the cloned voice's delivery. */
+ cloning?: boolean;
+ disabled?: boolean;
audioInstructions: string;
setAudioInstructions: (value: string) => void;
}) {
@@ -38,9 +43,11 @@ export function InstructionsField({
: instructionsKind === "scene"
? "Optional Higgs TTS 2 scene guidance such as room acoustics, recording conditions, or background ambience."
: instructionsKind === "voice"
- ? voiceRequired
- ? "Required by this model. Describe the voice, accent, age and delivery."
- : "Optional. Used by Qwen3-TTS VoiceDesign, Qwen3-TTS CustomVoice and VoxCPM2; other models ignore it."
+ ? cloning
+ ? "Optional. How the cloned voice should deliver the text, such as its mood, pace or tone."
+ : voiceRequired
+ ? "Required by this model. Describe the voice, accent, age and delivery."
+ : "Optional. Used by Qwen3-TTS VoiceDesign, Qwen3-TTS CustomVoice and VoxCPM2; other models ignore it."
: "Optional MOSS Local guidance such as speaking style, emotion, pace, or delivery."
}
htmlFor="audio-instructions"
@@ -48,6 +55,7 @@ export function InstructionsField({
<Textarea
id="audio-instructions"
value={audioInstructions}
+ disabled={disabled}
onChange={(event) =>
setAudioInstructions(event.target.value)
}
@@ -57,7 +65,9 @@ export function InstructionsField({
: instructionsKind === "scene"
? "Close-mic studio recording in a quiet, softly treated room…"
: instructionsKind === "voice"
- ? "A warm, low female voice, speaking slowly and calmly…"
+ ? cloning
+ ? "Calm and warm, speaking a little slowly…"
+ : "A warm, low female voice, speaking slowly and calmly…"
: "Warm, measured delivery with a calm conversational tone…"
}
className="min-h-24"
diff --git a/studio/frontend/src/features/audio/tools/clone-panels.tsx b/studio/frontend/src/features/audio/tools/clone-panels.tsx
index 5e8f30f7761..db4a5f24428 100644
--- a/studio/frontend/src/features/audio/tools/clone-panels.tsx
+++ b/studio/frontend/src/features/audio/tools/clone-panels.tsx
@@ -16,7 +16,9 @@ import type { ReactNode } from "react";
import { INDEX_TTS2_EMOTIONS } from "../clone-policy";
import { AudioSourceInput } from "../components/audio-source-input";
import { Field, FieldLabel } from "../components/field";
+import { InstructionsField } from "../components/instructions-fields";
import {
+ type CloneDescriptionValue,
type CosyVoiceMode,
type CosyVoiceModeValue,
type EmotionMode,
@@ -26,6 +28,7 @@ import {
type SpeedDialectValue,
type TimbreOnlyValue,
chatterboxExpressivenessLogic,
+ cloneDescriptionLogic,
cosyVoiceModeLogic,
emotionSourceProblem,
f5SpeedDialectLogic,
@@ -58,6 +61,22 @@ export function PanelSection({
);
}
+const cloneDescriptionPanel: AudioToolPanel<CloneDescriptionValue> = {
+ ...cloneDescriptionLogic,
+ Component: ({ value, onChange, disabled }) => (
+ <InstructionsField
+ instructionsKind="voice"
+ musicNeedsDescription={false}
+ cloning={true}
+ disabled={disabled}
+ audioInstructions={value.instructions}
+ setAudioInstructions={(instructions) =>
+ onChange({ ...value, instructions })
+ }
+ />
+ ),
+};
+
const qwen3TimbrePanel: AudioToolPanel<TimbreOnlyValue> = {
...qwen3TimbreLogic,
Component: ({ value, onChange, disabled }) => (
@@ -296,6 +315,7 @@ const f5SpeedDialectPanel: AudioToolPanel<SpeedDialectValue> = {
};
export const CLONE_TOOL_PANELS = [
+ cloneDescriptionPanel,
qwen3TimbrePanel,
indexTts2EmotionPanel,
chatterboxExpressivenessPanel,
diff --git a/studio/frontend/src/features/audio/tools/panel-logic.ts b/studio/frontend/src/features/audio/tools/panel-logic.ts
index 500b2276d33..eb04791a20f 100644
--- a/studio/frontend/src/features/audio/tools/panel-logic.ts
+++ b/studio/frontend/src/features/audio/tools/panel-logic.ts
@@ -33,6 +33,26 @@ function boolDefault(
return typeof value === "boolean" ? value : fallback;
}
+export interface CloneDescriptionValue {
+ instructions: string;
+}
+
+// Request field, not a spec option, so Advanced keeps every option.
+export const cloneDescriptionLogic: AudioToolPanelLogic<CloneDescriptionValue> =
+ {
+ id: "clone-description",
+ families: [],
+ workflows: ["clone"],
+ title: "Voice or style description",
+ claims: [],
+ appliesTo: (ctx) => ctx.cloneDescription === true,
+ initial: () => ({ instructions: "" }),
+ toRequest: (value) => {
+ const instructions = value.instructions.trim();
+ return instructions ? { instructions } : {};
+ },
+ };
+
export interface TimbreOnlyValue {
timbreOnly: boolean;
}
@@ -352,6 +372,7 @@ export const supertonicLanguageLogic: AudioToolPanelLogic<string> = {
};
export const CLONE_PANEL_LOGIC = [
+ cloneDescriptionLogic,
qwen3TimbreLogic,
indexTts2EmotionLogic,
chatterboxExpressivenessLogic,
diff --git a/studio/frontend/src/features/audio/tools/select.ts b/studio/frontend/src/features/audio/tools/select.ts
index 3d35d67f84d..c81ea08cd2d 100644
--- a/studio/frontend/src/features/audio/tools/select.ts
+++ b/studio/frontend/src/features/audio/tools/select.ts
@@ -67,6 +67,7 @@ export function audioModelContextFor(
audio_reference_text?: string | null;
audio_convert?: AudioConvertCaps | null;
audio_music?: unknown;
+ audio_clone?: unknown;
} | null,
page: {
musicGeneration: boolean;
@@ -75,6 +76,7 @@ export function audioModelContextFor(
},
): AudioModelContext {
const referenceText = status?.audio_reference_text;
+ const clone = status?.audio_clone;
return {
audioType: status?.audio_type ?? null,
audioFamily: status?.audio_family ?? null,
@@ -97,6 +99,7 @@ export function audioModelContextFor(
: null,
audioMusic: parseMusicCapabilities(status?.audio_music) !== null,
convert: convertCapsOf(status?.audio_convert),
+ cloneDescription: isPlainObject(clone) && clone.description === true,
};
}
diff --git a/studio/frontend/src/features/audio/tools/types.ts b/studio/frontend/src/features/audio/tools/types.ts
index 2ca64df2288..5960aba023e 100644
--- a/studio/frontend/src/features/audio/tools/types.ts
+++ b/studio/frontend/src/features/audio/tools/types.ts
@@ -32,6 +32,8 @@ export interface AudioModelContext {
voiceOption?: string | null;
/** Request inputs a strict model reads; null when it reads any. */
requestInputs?: readonly string[] | null;
+ /** The clone path reads a voice or style description beside the reference. */
+ cloneDescription?: boolean;
}
export interface AudioRunPatch {
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 890723d98a3..1ba75018502 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -366,6 +366,8 @@ export interface InferenceStatusResponse {
audio_device_fallback?: "cpu" | null;
/** Unknown-shaped on purpose: validated by parseMusicCapabilities. */
audio_music?: unknown;
+ /** Unknown-shaped on purpose: audioModelContextFor reads only what it needs. */
+ audio_clone?: unknown;
has_audio_input?: boolean;
has_video_input?: boolean;
/** Answers a spoken question in its own voice when asked for modalities ["text", "audio"]. */
diff --git a/studio/frontend/tests/audio-clone-panels.test.ts b/studio/frontend/tests/audio-clone-panels.test.ts
index 5c3210f0209..8f28cef9ed9 100644
--- a/studio/frontend/tests/audio-clone-panels.test.ts
+++ b/studio/frontend/tests/audio-clone-panels.test.ts
@@ -23,6 +23,7 @@ const {
SAVED_VOICE_MISSING,
SAVED_VOICE_NO_TRANSCRIPT,
chatterboxExpressivenessLogic,
+ cloneDescriptionLogic,
cosyVoiceModeLogic,
emotionSourceProblem,
f5SpeedDialectLogic,
@@ -72,6 +73,84 @@ test("each clone family gets its one panel, only on Clone", () => {
assert.deepEqual(shown("f5_tts"), ["f5-speed-dialect"]);
assert.deepEqual(shown("voxcpm2"), []);
assert.deepEqual(shown("qwen3_tts", "speak"), []);
+ assert.deepEqual(shown("voxcpm2", "clone", { cloneDescription: true }), [
+ "clone-description",
+ ]);
+ assert.deepEqual(shown("voxcpm2", "speak", { cloneDescription: true }), []);
+});
+
+test("Clone's description shows only when the model's clone path reads one", () => {
+ assert.equal(
+ panelApplies(
+ cloneDescriptionLogic,
+ "clone",
+ ctx({ cloneDescription: true }),
+ ),
+ true,
+ );
+ assert.equal(
+ panelApplies(
+ cloneDescriptionLogic,
+ "clone",
+ ctx({ cloneDescription: false }),
+ ),
+ false,
+ );
+ assert.equal(panelApplies(cloneDescriptionLogic, "clone", ctx()), false);
+ assert.equal(
+ panelApplies(
+ cloneDescriptionLogic,
+ "speak",
+ ctx({ cloneDescription: true }),
+ ),
+ false,
+ );
+});
+
+test("Clone's description is trimmed and sent only when written", () => {
+ assert.deepEqual(cloneDescriptionLogic.initial([]), { instructions: "" });
+ assert.deepEqual(
+ cloneDescriptionLogic.toRequest({ instructions: " Calm and slow \n" }),
+ { instructions: "Calm and slow" },
+ );
+ assert.deepEqual(cloneDescriptionLogic.toRequest({ instructions: "" }), {});
+ assert.deepEqual(
+ cloneDescriptionLogic.toRequest({ instructions: " \t\n " }),
+ {},
+ );
+ const { patch, error } = collectToolRequest(
+ [cloneDescriptionLogic] as never,
+ { "clone-description": { instructions: " Warm and bright " } },
+ { text: "hi" },
+ ctx({ cloneDescription: true }),
+ );
+ assert.deepEqual(patch, { instructions: "Warm and bright" });
+ assert.equal(error, null);
+ assert.deepEqual(
+ collectToolRequest(
+ [cloneDescriptionLogic] as never,
+ {},
+ { text: "hi" },
+ ctx({ cloneDescription: true }),
+ ).patch,
+ {},
+ );
+});
+
+test("Clone's description panel reuses the shared description field", () => {
+ const panels = readSrc("features/audio/tools/clone-panels.tsx");
+ assert.match(panels, /<InstructionsField\s+instructionsKind="voice"/);
+ assert.match(panels, /cloning=\{true\}\s+disabled=\{disabled\}/);
+ const field = readSrc("features/audio/components/instructions-fields.tsx");
+ // Locked while a clone runs, like the other Clone panels.
+ assert.match(field, /disabled=\{disabled\}/);
+ assert.ok(
+ field.includes(
+ "Optional. How the cloned voice should deliver the text, such as its mood, pace or tone.",
+ ),
+ );
+ assert.ok(field.includes("Calm and warm, speaking a little slowly…"));
+ assert.doesNotMatch(field, /(?<!Unsloth )\bStudio\b/);
});
test("the Voice panel shows on Speak for models that clone or declare a voice", () => {
@@ -485,6 +564,20 @@ test("the status context reads the clone fields defensively", () => {
const none = audioModelContextFor(null, page);
assert.deepEqual(none.audioWorkflows, []);
assert.equal(none.referenceTextMode, null);
+ assert.equal(none.cloneDescription, false);
+ const cloneDescription = (audio_clone: unknown) =>
+ audioModelContextFor({ audio_clone }, page).cloneDescription;
+ assert.equal(
+ cloneDescription({ reference_text: "required", description: true }),
+ true,
+ );
+ assert.equal(cloneDescription({ description: "yes" }), false);
+ assert.equal(cloneDescription({ description: false }), false);
+ assert.equal(cloneDescription({}), false);
+ assert.equal(cloneDescription(null), false);
+ assert.equal(cloneDescription(true), false);
+ assert.equal(cloneDescription([{ description: true }]), false);
+ assert.equal(audioModelContextFor({}, page).cloneDescription, false);
assert.equal(
audioModelContextFor({ audio_reference_text: "bogus" }, page)
.referenceTextMode,
diff --git a/studio/frontend/tests/audio-tool-panels.test.ts b/studio/frontend/tests/audio-tool-panels.test.ts
index 4e377f5eb58..6be4b064a78 100644
--- a/studio/frontend/tests/audio-tool-panels.test.ts
+++ b/studio/frontend/tests/audio-tool-panels.test.ts
@@ -119,6 +119,7 @@ test("instruction panels claim no spec option; clone panels claim theirs", async
CLONE_PANEL_LOGIC.map((panel) => [panel.id, [...panel.claims]]),
),
{
+ "clone-description": [],
"qwen3-timbre": ["x_vector_only_mode"],
"index-emotion": [
"emotion_vector",