Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: add a Convert audio page for voice conversion with A/B compare (#12611)

Etherl4444657aece0
diff --git a/studio/backend/core/inference/audio_cpp_backend.py b/studio/backend/core/inference/audio_cpp_backend.py
index 4d679f14469..27567b1e152 100644
--- a/studio/backend/core/inference/audio_cpp_backend.py
+++ b/studio/backend/core/inference/audio_cpp_backend.py
@@ -28,7 +28,7 @@ from dataclasses import replace
from pathlib import Path
from typing import Any, Optional, Tuple
-from core.inference import audio_cpp_files
+from core.inference import audio_cpp_convert, audio_cpp_files
from core.inference import audio_cpp_music as cm
from core.inference.audio_cpp_models import (
MUSIC_MAX_VARIATIONS,
@@ -123,6 +123,25 @@ def model_info_fields(model: AudioCppModel) -> dict[str, Any]:
"audio_clone": clone_rules(model),
"audio_edit": edit_rules(model),
"audio_music": music_rules(model),
+ "audio_options_by_workflow": (
+ {"convert": [dict(option) for option in model.convert_options]}
+ if model.convert is not None
+ else None
+ ),
+ "audio_workflow_tasks": audio_cpp_convert.workflow_tasks(model),
+ "audio_convert": audio_cpp_convert.convert_caps(model),
+ "audio_convert_rules": model.convert
+ and {"source_rate": model.convert.source_rate, "target_rate": model.convert.target_rate},
+ **server_runtime_fields(model, None),
+ }
+
+
+def server_runtime_fields(model: AudioCppModel, running: Optional[AudioCppModel]) -> dict[str, Any]:
+ """Task and Seed-VC route of the running server (``model``'s own when none runs)."""
+ served = running or model
+ return {
+ "audio_server_task": served.server_task,
+ "audio_convert_route": audio_cpp_convert.served_route(served),
}
@@ -379,9 +398,18 @@ class AudioCppBackend:
def _running_server(self, model: AudioCppModel, cancel_event) -> AudioCppServer:
with self._server_lock:
- if self._server is None or not self._server.alive() or self._server.model != model:
+ if (
+ self._server is None
+ or not self._server.alive()
+ or self._server.model != model
+ # Seed-VC's route: model equality leaves model_options out.
+ or _request_defaults(self._server.model) != _request_defaults(model)
+ ):
logger.info("audio.cpp: (re)starting the server for %s", model.id)
- self._start_server(model, cancel_event)
+ try:
+ self._start_server(model, cancel_event)
+ finally:
+ self._record_runtime()
return self._server
# Generation
@@ -405,6 +433,7 @@ class AudioCppBackend:
audio_inputs: Optional[dict] = None,
reference_text: Optional[str] = None,
speed: Optional[float] = None,
+ convert: Optional[dict] = None,
edit: Optional[dict] = None,
music: Optional[dict] = None,
output_dir: Optional[str] = None,
@@ -430,23 +459,56 @@ class AudioCppBackend:
cancel_event,
)
editing = workflow == "edit"
+ converting = workflow == "convert"
# Speak in a saved voice sends a reference with workflow "speak": a clone too.
- cloning = not editing and (workflow == "clone" or bool(audio_inputs))
+ cloning = not (editing or converting) and (workflow == "clone" or bool(audio_inputs))
if editing and model.edit is None:
raise RuntimeError(f"{model.display_name} cannot edit speech.")
- if cloning and model.clone is None:
- raise RuntimeError(f"{model.display_name} cannot clone a voice.")
- if editing:
+ if converting:
+ if model.convert is None:
+ raise RuntimeError(f"{model.display_name} cannot convert a voice.")
+ convert = convert or {}
+ mode = str(convert.get("mode") or "speech")
+ try:
+ served, options = audio_cpp_convert.served_model(
+ model, mode, validate_options(model.convert_options, audio_options)
+ )
+ request = audio_cpp_convert.convert_request(
+ model,
+ mode = mode,
+ source = (audio_inputs or {}).get("source"),
+ target = (audio_inputs or {}).get("target"),
+ voice = convert.get("voice"),
+ pitch = convert.get("pitch"),
+ pitch_auto = bool(convert.get("pitch_auto")),
+ style = str(convert.get("style") or "source"),
+ source_text = convert.get("source_text"),
+ options = {name: _option_string(value) for name, value in options.items()},
+ seed = seed,
+ )
+ except audio_cpp_convert.ConvertRequestError as exc:
+ raise RuntimeError(str(exc)) from exc
+ elif editing:
options = validate_options(edit_options(model), audio_options)
served = _served_for_edit(model)
else:
+ if cloning and model.clone is None:
+ raise RuntimeError(f"{model.display_name} cannot clone a voice.")
+ served = model
options = validate_options(
model.clone_options if cloning else model.options, audio_options
)
- served = model
server = self._running_server(served, cancel_event)
try:
- if editing:
+ if converting:
+ ctype, data = server.post_json(
+ "/v1/tasks/run",
+ {"model": server.model_id, "request": request},
+ timeout = _GENERATE_TIMEOUT_SECONDS,
+ cancel_event = cancel_event,
+ )
+ wav = _audio_from_task_response(ctype, data)
+ elif editing:
wav = self._generate_edit(
server,
model,
@@ -501,6 +563,20 @@ class AudioCppBackend:
_raise_if_cancelled(cancel_event)
return wav, _wav_sample_rate(wav)
+ def runtime_fields(self) -> dict[str, Any]:
+ model = self._model
+ if model is None:
+ return {}
+ with self._server_lock:
+ server = self._server
+ running = server.model if server is not None and server.alive() else None
+ return server_runtime_fields(model, running)
+
+ def _record_runtime(self) -> None:
+ entry = self.models.get(self.active_model_name or "")
+ if entry is not None:
+ entry.update(self.runtime_fields())
+
def separate_audio(
self,
source_path: str,
@@ -571,6 +647,7 @@ class AudioCppBackend:
# request starts a fresh one.
with self._server_lock:
self._stop_server_locked()
+ self._record_runtime()
@staticmethod
def _generate_speech(
@@ -959,6 +1036,10 @@ def _with_companions(model: AudioCppModel) -> AudioCppModel:
return replace(model, model_options = {**model.model_options, "session_options": session})
+def _request_defaults(model: AudioCppModel) -> Any:
+ return (model.model_options or {}).get("default_request_options")
+
+
_MAX_BATCH_OPTION = "stable_audio.max_batch"
_MANIFEST = "outputs.json"
diff --git a/studio/backend/core/inference/audio_cpp_convert.py b/studio/backend/core/inference/audio_cpp_convert.py
new file mode 100644
index 00000000000..91d5c613c75
--- /dev/null
+++ b/studio/backend/core/inference/audio_cpp_convert.py
@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Voice conversion on audio.cpp: the session a Convert run needs and its /v1/tasks/run request."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Any, Optional
+
+from core.inference.audio_cpp_models import AudioCppModel
+
+CONVERT_SOURCE_MAX_SECONDS = 300
+CONVERT_TARGET_MAX_SECONDS = 30
+
+CONVERT_MODES = ("speech", "singing")
+
+_STYLE_ROUTES = {"vc": "style_converted_vc"}
+
+
+class ConvertRequestError(ValueError):
+ """A user-facing reason the loaded model cannot run this conversion."""
+
+
+def convert_caps(model: AudioCppModel) -> Optional[dict[str, Any]]:
+ convert = model.convert
+ if convert is None:
+ return None
+ return {
+ "modes": [mode for mode, _task in convert.modes],
+ "target": convert.target,
+ "builtin_voices": [{"id": vid, "label": label} for vid, label in convert.builtin_voices],
+ # Seed-VC adds a manual shift on top of Auto; Vevo2's explicit steps replace its estimate.
+ "pitch": {
+ mode: {"auto": auto, "shift_with_auto": auto and convert.pitch_options == "semitone"}
+ for mode, auto in convert.pitch
+ },
+ "style": convert.style,
+ "route_reloads": bool(convert.routes),
+ "source_max_seconds": CONVERT_SOURCE_MAX_SECONDS,
+ }
+
+
+def workflow_tasks(model: AudioCppModel) -> dict[str, str]:
+ tasks = {workflow: binding.server_task for workflow, binding in model.workflows.items()}
+ if model.convert is not None:
+ for mode, task in model.convert.modes[1:]:
+ tasks[f"convert:{mode}"] = task
+ return tasks
+
+
+def served_route(model: AudioCppModel) -> Optional[str]:
+ convert = model.convert
+ if convert is None or not convert.routes:
+ return None
+ configured = ((model.model_options or {}).get("default_request_options") or {}).get("route")
+ if configured:
+ return str(configured)
+ if model.server_task == convert.modes[0][1]:
+ return convert.routes[0]
+ return "v1_svc" if model.server_task == "svc" else None
+
+
+def served_model(
+ model: AudioCppModel, mode: str, options: dict[str, Any]
+) -> tuple[AudioCppModel, dict[str, Any]]:
+ """``(model as the server must load it for this run, options left for the request)``."""
+ convert = model.convert
+ if convert is None:
+ raise ConvertRequestError(f"{model.display_name} cannot convert a voice.")
+ tasks = convert.server_tasks
+ if mode not in tasks:
+ raise ConvertRequestError(f"{model.display_name} does not convert {mode}.")
+ rest = dict(options)
+ requested = rest.pop("route", None)
+ model_options = {
+ key: value
+ for key, value in (model.model_options or {}).items()
+ if key != "default_request_options"
+ }
+ if convert.routes and mode == convert.modes[0][0]:
+ route = str(requested) if requested in convert.routes else convert.routes[0]
+ if route != convert.routes[0]:
+ model_options["default_request_options"] = {"route": route}
+ return replace(model, server_task = tasks[mode], model_options = model_options), rest
+
+
+def convert_request(
+ model: AudioCppModel,
+ *,
+ mode: str,
+ source: str,
+ target: Optional[str],
+ voice: Optional[str] = None,
+ pitch: Optional[int] = None,
+ pitch_auto: bool = False,
+ style: str = "source",
+ source_text: Optional[str] = None,
+ options: Optional[dict[str, str]] = None,
+ seed: Optional[int] = None,
+) -> dict[str, Any]:
+ convert = model.convert
+ if convert is None:
+ raise ConvertRequestError(f"{model.display_name} cannot convert a voice.")
+ if not source:
+ raise ConvertRequestError("Add the recording to convert.")
+ builtin = convert.target == "builtin"
+ if builtin and target:
+ raise ConvertRequestError(
+ f"{model.display_name} converts to its built-in voices; pick one under Built-in."
+ )
+ if not builtin and not target:
+ raise ConvertRequestError("Add the target voice.")
+ task = convert.server_tasks.get(mode)
+ take_style = convert.style and style == "target" and task in _STYLE_ROUTES
+ transcript = str(source_text or "").strip()
+ if take_style and not transcript:
+ raise ConvertRequestError("Type what's said in the recording, or press Transcribe.")
+
+ request: dict[str, Any] = {}
+ if convert.fields == "source_target":
+ request["source_audio"] = str(source)
+ request["target_voice"] = str(target)
+ else:
+ request["audio"] = str(source)
+ if target:
+ request["voice_ref"] = str(target)
+ if take_style:
+ request["route"] = _STYLE_ROUTES[task]
+ request["target_text"] = transcript
+ request["style_ref"] = str(target)
+
+ request_options = dict(options or {})
+ if builtin:
+ ids = [vid for vid, _label in convert.builtin_voices]
+ request_options["voice_id"] = voice if voice in ids else ids[0]
+ pitch_modes = dict(convert.pitch)
+ if mode in pitch_modes and not take_style:
+ auto = pitch_auto and pitch_modes[mode]
+ if convert.pitch_options == "shift_steps":
+ # Auto leaves both out: the style-preserving routes estimate the shift themselves.
+ if not auto and pitch is not None:
+ if int(pitch):
+ request_options["use_pitch_shift"] = "true"
+ request_options["source_shift_steps"] = str(int(pitch))
+ else:
+ request_options["use_pitch_shift"] = "false"
+ else:
+ if auto:
+ request_options["auto_f0_adjust"] = "true"
+ if pitch is not None:
+ request_options["semitone_shift"] = str(int(pitch))
+ if request_options:
+ request["options"] = request_options
+ if convert.seed and seed is not None:
+ request["seed"] = str(int(seed))
+ return request
diff --git a/studio/backend/core/inference/audio_cpp_models.py b/studio/backend/core/inference/audio_cpp_models.py
index 38592160756..c46c6d40d0f 100644
--- a/studio/backend/core/inference/audio_cpp_models.py
+++ b/studio/backend/core/inference/audio_cpp_models.py
@@ -119,6 +119,35 @@ class EditSpec:
claims: tuple[str, ...] = ("template_name", "source_text", "target_text", "instruction")
+@dataclass(frozen = True)
+class ConvertSpec:
+ """How a family converts a recording to another voice on /v1/tasks/run."""
+
+ modes: tuple[tuple[str, str], ...] = (("speech", "vc"),)
+ builtin_voices: tuple[tuple[str, str], ...] = ()
+ pitch: tuple[tuple[str, bool], ...] = ()
+ style: bool = False
+ # "audio" (audio + voice_ref) or "source_target" (source_audio + target_voice).
+ fields: str = "audio"
+ # RVC refuses seed as an unknown option.
+ seed: bool = True
+ source_rate: int = 16000
+ target_rate: Optional[int] = 16000
+ pitch_options: str = "semitone"
+ # Seed-VC speech routes, default first: a session keeps the route it first ran.
+ routes: tuple[str, ...] = ()
+ tool_options: tuple[dict, ...] = field(default = (), hash = False)
+ hidden_options: tuple[str, ...] = ()
+
+ @property
+ def target(self) -> str:
+ return "builtin" if self.builtin_voices else "audio"
+
+ @property
+ def server_tasks(self) -> dict[str, str]:
+ return dict(self.modes)
+
+
@dataclass(frozen = True)
class SeparationSpec:
"""How a family splits a track into stems on /v1/tasks/run (task ``sep``, 44.1 kHz input)."""
@@ -323,6 +352,11 @@ def _bindings_for(
bindings["edit"] = WorkflowBinding(
edit.server_task, "tasks", edit.route, ("source", "text", "reference_text")
)
+ convert = getattr(family, "convert", None)
+ if convert is not None:
+ bindings["convert"] = WorkflowBinding(
+ convert.modes[0][1], "tasks", None, ("source", "target", "source_text")
+ )
return bindings
if task == "music":
return {
@@ -360,6 +394,7 @@ class AudioCppFamily:
clone: Optional[CloneSpec] = None
companions: tuple[CompanionModel, ...] = ()
edit: Optional[EditSpec] = None
+ convert: Optional[ConvertSpec] = None
separation: Optional[SeparationSpec] = None
music: Optional[MusicSpec] = field(default = None, hash = False, compare = False)
@@ -480,6 +515,22 @@ _CLONE_FAMILIES: tuple[AudioCppFamily, ...] = (
),
),
),
+ # A clon session refuses a conversion ("prepare requires text input"), so Convert loads vc.
+ convert = ConvertSpec(
+ source_rate = 16000,
+ target_rate = 24000,
+ tool_options = (
+ _tool(
+ "s3gen_cfg_rate",
+ "float",
+ "Classifier-free guidance of the voice decoder.",
+ min = 0.0,
+ max = 3.0,
+ default = 0.7,
+ ),
+ _tool("num_inference_steps", "int", "Decoder steps.", min = 1, max = 100, default = 10),
+ ),
+ ),
),
AudioCppFamily("f5_tts", "tts", speaks = False, clone = CloneSpec("required", speed = True)),
AudioCppFamily(
@@ -548,6 +599,25 @@ _CLONE_FAMILIES: tuple[AudioCppFamily, ...] = (
speaks = False,
clone = CloneSpec("unused"),
edit = EditSpec("s2s", "sentence", route = "editing"),
+ convert = ConvertSpec(
+ modes = (("speech", "vc"), ("singing", "svc")),
+ pitch = (("speech", True), ("singing", True)),
+ style = True,
+ fields = "source_target",
+ pitch_options = "shift_steps",
+ source_rate = 24000,
+ target_rate = 24000,
+ tool_options = (
+ _tool(
+ "num_inference_steps",
+ "int",
+ "Flow matching steps.",
+ min = 1,
+ max = 100,
+ default = 32,
+ ),
+ ),
+ ),
),
# MioCodec's sibling-folder default never matches the umbrella layout: pass it by session option.
AudioCppFamily(
@@ -564,6 +634,52 @@ _CLONE_FAMILIES: tuple[AudioCppFamily, ...] = (
)
+_CONVERT_FAMILIES: tuple[AudioCppFamily, ...] = (
+ AudioCppFamily(
+ "rvc",
+ "tts",
+ server_task = "vc",
+ speaks = False,
+ convert = ConvertSpec(
+ builtin_voices = (
+ ("default", "Default"),
+ ("manthos", "Manthos"),
+ ("chocola", "Chocola"),
+ ("fraise", "Fraise"),
+ ),
+ pitch = (("speech", False),),
+ seed = False,
+ source_rate = 16000,
+ target_rate = None,
+ # audio_pad_duration_sec reads as a file option to the run route.
+ hidden_options = ("audio_pad_duration_sec", "speaker_id", "pitch_extractor"),
+ ),
+ ),
+ # Speech routes ignore semitone_shift; the V1 speech F0 path fails ("RMVPE is not initialized").
+ AudioCppFamily(
+ "seed_vc",
+ "tts",
+ server_task = "vc",
+ speaks = False,
+ convert = ConvertSpec(
+ modes = (("speech", "vc"), ("singing", "svc")),
+ pitch = (("singing", True),),
+ source_rate = 44100,
+ target_rate = 44100,
+ routes = ("v2_vc", "v1_whisper_bigvgan_vc", "v1_xlsr_hift_vc"),
+ hidden_options = ("f0_condition",),
+ ),
+ ),
+ AudioCppFamily(
+ "meanvc2",
+ "tts",
+ server_task = "vc",
+ speaks = False,
+ convert = ConvertSpec(source_rate = 16000, target_rate = 16000),
+ ),
+)
+
+
_SEPARATION_FAMILIES: tuple[AudioCppFamily, ...] = (
AudioCppFamily("htdemucs", "sep", separation = SeparationSpec()),
AudioCppFamily("htdemucs_6stems", "sep", separation = SeparationSpec()),
@@ -583,6 +699,7 @@ _FAMILY_LIST: tuple[AudioCppFamily, ...] = (
AudioCppFamily("supertonic", "tts", request_defaults = {"voice": "F1"}),
AudioCppFamily("qwen3_tts", "tts"),
*_CLONE_FAMILIES,
+ *_CONVERT_FAMILIES,
*(
AudioCppFamily(name, "tts")
for name in (
@@ -812,7 +929,6 @@ _FAMILY_LIST: tuple[AudioCppFamily, ...] = (
(("sortformer_diar", "sortformer_diar_v2", "nemotron_3_diar"), "Speaker diarization"),
(("pulsevad", "silero_vad", "marblenet_vad"), "Voice activity detection"),
(("muscriptor", "sheetsage2"), "Music transcription"),
- (("meanvc2", "rvc", "seed_vc"), "Voice conversion"),
(("apollo", "audiosr", "universr", "personaplex"), "Speech-to-speech"),
)
for name in names
@@ -1641,6 +1757,18 @@ _MUSIC_DRIVEN_OPTIONS = frozenset(
"semantic_min_tokens",
}
)
+CONVERT_DRIVEN_OPTIONS = frozenset(
+ {
+ "voice_id",
+ "semitone_shift",
+ "auto_f0_adjust",
+ "use_pitch_shift",
+ "source_shift_steps",
+ "prosody_shift_steps",
+ "target_text",
+ "style_ref_text",
+ }
+)
_RENDERABLE_TYPES = frozenset({"bool", "int", "float", "string", "enum"})
_MAX_STRING_OPTION = 4000
@@ -1763,6 +1891,8 @@ def option_schema(
source = embedded
raw = ((source or {}).get("options") or {}).get("request") or []
driven = _MUSIC_DRIVEN_OPTIONS if policy.task == "music" else frozenset()
+ if policy.convert is not None:
+ driven = driven | frozenset(policy.convert.hidden_options)
out = []
names = set()
for item in raw:
@@ -1864,6 +1994,7 @@ class AudioCppModel:
companions: tuple[CompanionModel, ...] = ()
required_inputs: tuple[str, ...] = ()
edit: Optional[EditSpec] = None
+ convert: Optional[ConvertSpec] = None
separation: Optional[SeparationSpec] = None
music: Optional[MusicSpec] = field(default = None, hash = False, compare = False)
# A strict spec (``schema_version``) makes the runtime refuse any undeclared option.
@@ -1911,6 +2042,14 @@ class AudioCppModel:
names = {o["name"] for o in self.options}
return (*self.options, *(o for o in self.clone.tool_options if o["name"] not in names))
+ @property
+ def convert_options(self) -> tuple[dict, ...]:
+ if self.convert is None:
+ return ()
+ own = tuple(o for o in self.options if o["name"] not in CONVERT_DRIVEN_OPTIONS)
+ names = {o["name"] for o in own}
+ return (*own, *(o for o in self.convert.tool_options if o["name"] not in names))
+
@property
def key(self) -> str:
"""A short name, unique per model and variant, for the link farm."""
@@ -2238,6 +2377,7 @@ def _resolve_uncached(
companions = policy.companions,
required_inputs = _required_inputs(spec, embedded, policy),
edit = policy.edit,
+ convert = policy.convert,
separation = policy.separation,
music = music_with_spec_bounds(policy.music, _raw_request_options(spec, embedded)),
request_keys = _request_keys(spec, embedded),
diff --git a/studio/backend/core/inference/audio_cpp_server.py b/studio/backend/core/inference/audio_cpp_server.py
index d8c7dd3d78d..a643fe1ed81 100644
--- a/studio/backend/core/inference/audio_cpp_server.py
+++ b/studio/backend/core/inference/audio_cpp_server.py
@@ -673,12 +673,16 @@ class AudioCppServer:
process.wait(timeout = 10)
except subprocess.TimeoutExpired:
process.kill()
+ # A killed server can sit in GPU context teardown well past 10 s on a busy card (Seed-VC
+ # took ~54 s); the music reload restarts it mid-session and must not fail on that.
try:
- # A killed server can sit in GPU context teardown well past 10 s on a busy
- # card; the music reload restarts it mid-session and must not fail on that.
process.wait(timeout = 120)
except subprocess.TimeoutExpired:
- logger.warning("audiocpp_server %s did not exit after SIGKILL", process.pid)
+ logger.warning(
+ "audio.cpp: server pid %s still exiting after SIGKILL", process.pid
+ )
+ shutil.rmtree(self._config_dir, ignore_errors = True)
+ return
forget_pid(process.pid)
shutil.rmtree(self._config_dir, ignore_errors = True)
diff --git a/studio/backend/core/inference/audio_gallery.py b/studio/backend/core/inference/audio_gallery.py
index fd84a8f230c..3186f311bdb 100644
--- a/studio/backend/core/inference/audio_gallery.py
+++ b/studio/backend/core/inference/audio_gallery.py
@@ -5,7 +5,8 @@
Each clip is a pair under ``workspace_root()/audio``: ``{id}.wav`` holds the bytes and
``{id}.json`` the recipe (a WAV has no portable text chunk). A lone file is not a
-valid record. Dumb storage: the route owns the schema, this reads, writes and sorts.
+valid record. A converted upload is kept as ``{id}.source.wav`` and goes with its clip.
+Dumb storage: the route owns the schema, this reads, writes and sorts.
"""
from __future__ import annotations
@@ -41,7 +42,11 @@ def gallery_dir() -> Path:
return ensure_account_dir(account_path("audio"))
-def save(wav_bytes: bytes, meta: dict[str, Any]) -> dict[str, Any]:
+def save(
+ wav_bytes: bytes,
+ meta: dict[str, Any],
+ source_wav: Optional[Path] = None,
+) -> dict[str, Any]:
"""Persist WAV bytes plus their recipe sidecar; return the record.
Staged then renamed in, wav first: the sidecar is the pair's commit marker. On any
@@ -53,13 +58,21 @@ def save(wav_bytes: bytes, meta: dict[str, Any]) -> dict[str, Any]:
wav_tmp = directory / f".{audio_id}.wav.tmp"
sidecar = directory / f"{audio_id}.json"
sidecar_tmp = directory / f".{audio_id}.json.tmp"
+ source_path = _source_path(audio_id)
+ source_tmp = directory / f".{audio_id}.source.wav.tmp"
+ if source_wav is not None:
+ meta = {**meta, "source_saved": True}
try:
wav_tmp.write_bytes(wav_bytes)
+ if source_wav is not None:
+ shutil.copyfile(source_wav, source_tmp)
sidecar_tmp.write_text(json.dumps(meta), encoding = "utf-8")
os.replace(wav_tmp, wav_path)
+ if source_wav is not None:
+ os.replace(source_tmp, source_path)
os.replace(sidecar_tmp, sidecar)
except BaseException:
- for path in (wav_tmp, sidecar_tmp, wav_path, sidecar):
+ for path in (wav_tmp, sidecar_tmp, source_tmp, wav_path, source_path, sidecar):
try:
path.unlink(missing_ok = True)
except OSError:
@@ -143,10 +156,13 @@ def _env_limit(name: str, default: int) -> int:
def _clip_bytes(audio_id: str) -> int:
- try:
- return (gallery_dir() / f"{audio_id}.wav").stat().st_size
- except OSError:
- return 0
+ total = 0
+ for path in (gallery_dir() / f"{audio_id}.wav", _source_path(audio_id)):
+ try:
+ total += path.stat().st_size
+ except OSError:
+ pass
+ return total
def _prune_to_cap() -> int:
@@ -210,6 +226,7 @@ def _prune_to_cap() -> int:
continue
removed += 1
pruned.append(audio_id)
+ _remove_source(audio_id)
try:
_sidecar_path(audio_id).unlink()
except OSError:
@@ -254,7 +271,7 @@ def _record(
def _workflow(meta: dict[str, Any]) -> str:
workflow = meta.get("workflow")
- if workflow in ("speak", "clone", "edit", "music", "separate"):
+ if workflow in ("speak", "clone", "edit", "convert", "music", "separate"):
return workflow
return workflow_for_audio_type(meta.get("audio_type"))
@@ -275,6 +292,21 @@ def _sidecar_path(audio_id: str) -> Path:
return gallery_dir() / f"{audio_id}.json"
+def _source_path(audio_id: str) -> Path:
+ return gallery_dir() / f"{audio_id}.source.wav"
+
+
+def _remove_source(audio_id: str) -> None:
+ try:
+ _source_path(audio_id).unlink(missing_ok = True)
+ except OSError:
+ pass
+
+
+def _clip_wavs(directory: Path) -> list[Path]:
+ return [p for p in directory.glob("*.wav") if _ID_RE.match(p.stem)]
+
+
def _sources_in_use(directory: Path, leaving: set[str]) -> set[str]:
"""Hidden Edit sources that a clip staying in the gallery still plays as its Original."""
try:
@@ -331,6 +363,17 @@ def owned_audio_path(audio_id: str) -> Optional[Path]:
return path
+def owned_source_path(audio_id: str) -> Optional[Path]:
+ if owned_audio_path(audio_id) is None:
+ return None
+ path = _source_path(audio_id)
+ try:
+ path.resolve().relative_to(gallery_dir().resolve())
+ except ValueError:
+ return None
+ return path if path.is_file() else None
+
+
def _mtime(path: Path) -> float:
try:
return path.stat().st_mtime
@@ -364,7 +407,7 @@ def _list_audio_entries(
archived: bool = False,
) -> list[tuple[dict[str, Any], GalleryCursor]]:
try:
- paths = list(gallery_dir().glob("*.wav"))
+ paths = _clip_wavs(gallery_dir())
except OSError:
return []
flags = gallery_flags.read(gallery_dir())
@@ -452,9 +495,7 @@ def move(audio_id: str, after_id: Optional[str]) -> Optional[dict[str, Any]]:
# The whole shelf in listing order, so neighbours past the client's loaded window are known.
try:
paths = [
- p
- for p in gallery_dir().glob("*.wav")
- if not gallery_flags.is_archived(flags, p.stem)
+ p for p in _clip_wavs(gallery_dir()) if not gallery_flags.is_archived(flags, p.stem)
]
except OSError:
paths = []
@@ -487,6 +528,7 @@ def delete(audio_id: str) -> bool:
_sidecar_path(audio_id).unlink()
except OSError:
pass
+ _remove_source(audio_id)
gallery_flags.forget(gallery_dir(), [audio_id])
return True
@@ -496,14 +538,14 @@ def clear(include_archived: bool = False, workflow: Optional[str] = None) -> int
orphan WAVs are preserved, since list_audio already hides them.
Archived clips are spared unless ``include_archived``, and sparing them raises
- FlagsUnavailable when the flag store cannot be read. A ``workflow`` (speak, clone, edit, music
- or separate) spares the other workflows' clips."""
+ FlagsUnavailable when the flag store cannot be read. A ``workflow`` (speak, clone, edit,
+ convert, music or separate) spares the other workflows' clips."""
removed = 0
directory = gallery_dir()
with gallery_flags.exclusive(directory, require_file_lock = not include_archived):
flags = {} if include_archived else gallery_flags.read_trusted(directory)
try:
- paths = list(directory.glob("*.wav"))
+ paths = _clip_wavs(directory)
except OSError:
return 0
doomed = []
@@ -527,6 +569,7 @@ def clear(include_archived: bool = False, workflow: Optional[str] = None) -> int
continue
removed += 1
cleared.append(path.stem)
+ _remove_source(path.stem)
try:
_sidecar_path(path.stem).unlink()
except OSError:
diff --git a/studio/backend/core/inference/audio_workflows.py b/studio/backend/core/inference/audio_workflows.py
index 35b45372a6f..c5dbfdae97f 100644
--- a/studio/backend/core/inference/audio_workflows.py
+++ b/studio/backend/core/inference/audio_workflows.py
@@ -1,8 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Which Audio page workflows (Speak, Clone, Edit, Music, Separate, Transcribe) a model or a
-clip belongs to.
+"""Which Audio page workflows (Speak, Clone, Edit, Convert, Music, Separate, Transcribe) a model
+or a clip belongs to.
Mirrors ``studio/frontend/src/features/audio/workflows.ts``. This module reads the audio.cpp
constants; ``audio_cpp_models`` never imports it.
@@ -18,7 +18,7 @@ from core.inference.audio_cpp_models import (
HUB_TASKS,
)
-AUDIO_WORKFLOW_IDS = ("speak", "clone", "edit", "music", "separate", "transcribe")
+AUDIO_WORKFLOW_IDS = ("speak", "clone", "edit", "convert", "music", "separate", "transcribe")
# Generation audio types that make music; every other generation audio type speaks.
MUSIC_AUDIO_TYPES = frozenset(("minimax_music3", AUDIO_CPP_MUSIC_AUDIO_TYPE))
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 63683156da5..c82f2e3de8e 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -391,6 +391,12 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"audio_reference_text": model_info.get("audio_reference_text"),
"audio_required_inputs": model_info.get("audio_required_inputs"),
"audio_clone": model_info.get("audio_clone"),
+ "audio_options_by_workflow": model_info.get("audio_options_by_workflow"),
+ "audio_workflow_tasks": model_info.get("audio_workflow_tasks"),
+ "audio_server_task": model_info.get("audio_server_task"),
+ "audio_convert": model_info.get("audio_convert"),
+ "audio_convert_route": model_info.get("audio_convert_route"),
+ "audio_convert_rules": model_info.get("audio_convert_rules"),
"audio_edit": model_info.get("audio_edit"),
"audio_music": model_info.get("audio_music"),
"audio_cpp_backend": model_info.get("audio_cpp_backend"),
@@ -3374,6 +3380,7 @@ class InferenceOrchestrator:
audio_inputs: Optional[dict[str, str]] = None,
reference_text: Optional[str] = None,
speed: Optional[float] = None,
+ convert: Optional[dict] = None,
edit: Optional[dict] = None,
music: Optional[dict] = None,
output_dir: Optional[str] = None,
@@ -3381,7 +3388,7 @@ class InferenceOrchestrator:
) -> Tuple[bytes, int]:
"""Generate TTS audio. Returns (wav_bytes, sample_rate). Blocking: sends the command and
waits for the full audio response. ``audio_inputs`` maps a role (reference, emotion,
- source) to a server-local WAV path; audio bytes never cross the queue. A separation
+ source, target) to a server-local WAV path; audio bytes never cross the queue. A separation
(``output_dir`` set) returns (outputs, sample_rate) instead, see ``separate_audio_response``."""
if not self._ensure_subprocess_alive():
raise RuntimeError("Inference subprocess is not running")
@@ -3459,6 +3466,8 @@ class InferenceOrchestrator:
cmd["reference_text"] = reference_text
if speed is not None:
cmd["speed"] = float(speed)
+ if convert is not None:
+ cmd["convert"] = dict(convert)
if edit is not None:
cmd["edit"] = dict(edit)
if music is not None:
@@ -3527,6 +3536,13 @@ class InferenceOrchestrator:
worker_started = True
continue
+ if rtype in ("audio_done", "audio_error") and isinstance(
+ resp.get("audio_runtime"), dict
+ ):
+ entry = self.models.get(expected_model)
+ if entry is not None:
+ entry.update(resp["audio_runtime"])
+
if rtype == "audio_done":
if cancel_event is not None and cancel_event.is_set():
raise AudioGenerationCancelledError("Audio generation cancelled")
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 249f531702a..38e56ead37b 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -795,6 +795,12 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"audio_reference_text",
"audio_required_inputs",
"audio_clone",
+ "audio_options_by_workflow",
+ "audio_workflow_tasks",
+ "audio_server_task",
+ "audio_convert",
+ "audio_convert_route",
+ "audio_convert_rules",
"audio_edit",
"audio_music",
"audio_cpp_backend",
@@ -1598,6 +1604,17 @@ def _handle_share_object(backend, cmd: dict, resp_queue: Any) -> None:
)
+def _audio_runtime(backend) -> dict:
+ fields = getattr(backend, "runtime_fields", None)
+ if not callable(fields):
+ return {}
+ try:
+ return {"audio_runtime": dict(fields())}
+ except Exception as exc: # noqa: BLE001 - status detail, never fails the request
+ logger.debug("audio runtime fields unavailable: %s", exc)
+ return {}
+
+
def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
"""Handle TTS audio generation — returns WAV bytes + sample_rate.
@@ -1633,7 +1650,7 @@ def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) ->
# workflow its backend has no keyword for.
params = inspect.signature(backend.generate_audio_response).parameters
takes_any = any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
- for key in ("workflow", "audio_inputs", "reference_text", "speed", "edit"):
+ for key in ("workflow", "audio_inputs", "reference_text", "speed", "convert", "edit"):
if cmd.get(key) is None:
continue
if takes_any or key in params:
@@ -1665,6 +1682,7 @@ def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) ->
"request_id": request_id,
"wav_base64": base64.b64encode(wav_bytes).decode("ascii"),
"sample_rate": sample_rate,
+ **_audio_runtime(backend),
"stats": getattr(backend, "last_generation_stats", None),
}
take_status_patch = getattr(backend, "take_status_patch", None)
@@ -1683,6 +1701,7 @@ def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) ->
# Flag a shared-event cancel (unload, training, arbiter) so the orchestrator does not report HTTP 500.
"cancelled": bool(cancel_event is not None and cancel_event.is_set()),
"stack": traceback.format_exc(limit = 20),
+ **_audio_runtime(backend),
}
if isinstance(exc, AudioRuntimeError):
response["code"] = AUDIO_RUNTIME_ERROR_CODE
diff --git a/studio/backend/core/library.py b/studio/backend/core/library.py
index 6fdb0573b55..8b56e017079 100644
--- a/studio/backend/core/library.py
+++ b/studio/backend/core/library.py
@@ -526,6 +526,12 @@ def _gallery_items(kind: str) -> list[dict]:
sidecar = path.with_suffix(".json").stat().st_size
except OSError:
pass
+ # A conversion keeps the recording it converted as {id}.source.wav beside its clip.
+ if sidecar is not None and kind == "audio":
+ try:
+ sidecar += path.with_name(f"{path.stem}.source.wav").stat().st_size
+ except OSError:
+ pass
items.append(
_item(
f"{kind}:{record['id']}",
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 62c5ab01395..9eac231a5dc 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1396,6 +1396,21 @@ class _InferenceRuntimeFields(BaseModel):
"None for a model the Music studio does not drive (and native MiniMax)."
),
)
+ audio_options_by_workflow: Optional[Dict[str, List[Dict[str, Any]]]] = Field(
+ None, description = "Options per Audio workflow where they differ from audio_options."
+ )
+ audio_workflow_tasks: Optional[Dict[str, str]] = Field(
+ None, description = "audio.cpp server task per workflow, e.g. {'convert:singing': 'svc'}."
+ )
+ audio_server_task: Optional[str] = Field(
+ None, description = "audio.cpp server task the running server was started with."
+ )
+ audio_convert: Optional[Dict[str, Any]] = Field(
+ None, description = "What the model offers on Convert; None when it does not convert."
+ )
+ audio_convert_route: Optional[str] = Field(
+ None, description = "Seed-VC route the running server was started with."
+ )
@model_validator(mode = "after")
def derive_audio_workflows(self):
@@ -5150,7 +5165,7 @@ class AudioGalleryItem(BaseModel):
archived: bool = Field(False, description = "Moved to the archived shelf, hidden from history")
workflow: Optional[str] = Field(
None,
- description = "Audio page workflow that made the clip: speak, clone, edit, music or separate",
+ description = "Audio page workflow that made the clip: speak, clone, edit, convert, music or separate",
)
order_at: Optional[float] = Field(
None,
@@ -5166,6 +5181,20 @@ class AudioGalleryItem(BaseModel):
None, description = "Language, options, speed and whether a transcript was used"
)
reference_name: Optional[str] = Field(None, description = "Name of the reference clip or voice")
+ source_input_id: Optional[str] = Field(
+ None, description = "Uploaded input a conversion took its recording from"
+ )
+ source_name: Optional[str] = Field(None, description = "Name of the recording a run converted")
+ target_builtin: Optional[str] = Field(
+ None, description = "Built-in voice a conversion converted to"
+ )
+ target_clip_id: Optional[str] = Field(
+ None, description = "History clip a conversion took its target voice from"
+ )
+ target_input_id: Optional[str] = Field(
+ None, description = "Uploaded input a conversion took its target voice from"
+ )
+ source_saved: bool = Field(False, description = "Served at /audio/gallery/{id}/source/file")
_AUDIO_ID_PATTERN = r"^[A-Za-z0-9_-]{1,128}$"
@@ -5179,7 +5208,8 @@ _AUDIO_CLIP_OPTION_ENDINGS = frozenset({"audio", "wav", "ref", "video", "image",
def _names_a_file(name: str, value: Any) -> bool:
words = re.split(r"[^a-z0-9]+", name.lower())
- if any(word in _AUDIO_FILE_OPTION_WORDS for word in words):
+ # Vevo2's target_voice is a file, but neither of its words gives that away.
+ if name.lower() == "target_voice" or any(word in _AUDIO_FILE_OPTION_WORDS for word in words):
return True
return isinstance(value, str) and words[-1] in _AUDIO_CLIP_OPTION_ENDINGS
@@ -5208,6 +5238,19 @@ class AudioRunInputs(BaseModel):
source: Optional[AudioSourceRef] = None
reference_text: Optional[str] = Field(None, max_length = 4000)
emotion: Optional[AudioSourceRef] = None
+ target: Optional[AudioSourceRef] = None
+ # A 5 minute source (CONVERT_SOURCE_MAX_SECONDS) runs well past 4000 characters of speech.
+ source_text: Optional[str] = Field(None, max_length = 16000)
+
+
+class AudioConvertParams(BaseModel):
+ model_config = ConfigDict(extra = "forbid")
+
+ mode: Literal["speech", "singing"] = "speech"
+ pitch: Optional[int] = Field(None, ge = -24, le = 24)
+ pitch_auto: bool = False
+ style: Literal["source", "target"] = "source"
+ voice: Optional[Literal["default", "manthos", "chocola", "fraise"]] = None
class AudioRunEdit(BaseModel):
@@ -5252,8 +5295,8 @@ class AudioRunRequest(BaseModel):
model_config = ConfigDict(extra = "forbid")
- workflow: Literal["clone", "speak", "edit", "music", "separate"]
- # Required except for a separation (the route answers a separation given text with a 400).
+ workflow: Literal["clone", "speak", "edit", "convert", "music", "separate"]
+ # Required except for a conversion or a separation (the routes answer those given text with a 400).
text: Optional[str] = None
language: Optional[str] = Field(None, max_length = 64)
instructions: Optional[str] = Field(None, max_length = 4000)
@@ -5271,6 +5314,27 @@ class AudioRunRequest(BaseModel):
speed: Optional[float] = Field(None, ge = 0.25, le = 4.0)
seed: Optional[int] = Field(None, ge = -(2**63), le = 2**64 - 1)
max_tokens: Optional[int] = Field(None, ge = 1)
+ convert: Optional[AudioConvertParams] = None
+
+ @model_validator(mode = "after")
+ def _fields_of_the_workflow(self):
+ inputs = self.inputs
+ if self.workflow == "convert":
+ if self.text is not None:
+ raise ValueError("A conversion takes no text.")
+ if inputs.reference is not None or inputs.emotion is not None:
+ raise ValueError("A conversion takes inputs.source and inputs.target.")
+ if self.convert is None:
+ self.convert = AudioConvertParams()
+ return self
+ if self.text is None and self.workflow != "separate":
+ raise ValueError("text is required.")
+ # inputs.source also carries a Music edit's clip; the route refuses it elsewhere.
+ if self.convert is not None or any(
+ v is not None for v in (inputs.target, inputs.source_text)
+ ):
+ raise ValueError("convert and the target inputs are for workflow convert.")
+ return self
@model_validator(mode = "after")
def _edit_fields(self):
@@ -5303,7 +5367,7 @@ class AudioRunRequest(BaseModel):
raise ValueError("Pick a music mode: song, sfx or edit.")
if self.text is None:
raise ValueError("text is required for music.")
- elif self.workflow != "separate" and not self.text:
+ elif self.workflow not in ("convert", "separate") and not self.text:
raise ValueError("text must not be empty.")
return self
@@ -5350,6 +5414,7 @@ class AudioInputTranscribeRequest(BaseModel):
engine: Optional[str] = Field(None, max_length = 64)
device: Optional[Literal["auto", "cpu", "gpu"]] = None
language: Optional[str] = Field(None, max_length = 64)
+ purpose: Literal["reference", "convert"] = "reference"
class AudioInputTranscript(BaseModel):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 9360f0b99ca..c309091b71e 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8255,6 +8255,11 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
audio_workflows = None,
audio_reference_text = None,
audio_required_inputs = None,
+ audio_options_by_workflow = None,
+ audio_workflow_tasks = None,
+ audio_server_task = None,
+ audio_convert = None,
+ audio_convert_route = None,
audio_music = None,
# Older/custom backend doubles predate this additive runtime field.
preserve_thinking_default = bool(getattr(llama_backend, "preserve_thinking_default", False)),
@@ -19169,6 +19174,11 @@ async def _load_model_impl(
audio_options = _model_info.get("audio_options"),
audio_workflows = _model_info.get("audio_workflows"),
audio_reference_text = _model_info.get("audio_reference_text"),
+ audio_options_by_workflow = _model_info.get("audio_options_by_workflow"),
+ audio_workflow_tasks = _model_info.get("audio_workflow_tasks"),
+ audio_server_task = _model_info.get("audio_server_task"),
+ audio_convert = _model_info.get("audio_convert"),
+ audio_convert_route = _model_info.get("audio_convert_route"),
audio_required_inputs = _model_info.get("audio_required_inputs"),
audio_music = _model_info.get("audio_music"),
is_mlx = bool(_model_info.get("is_mlx", False)),
@@ -21851,6 +21861,11 @@ async def _slot_status(current_subject: str):
audio_family = model_info.get("audio_family"),
audio_options = model_info.get("audio_options"),
audio_workflows = model_info.get("audio_workflows"),
+ audio_options_by_workflow = model_info.get("audio_options_by_workflow"),
+ audio_workflow_tasks = model_info.get("audio_workflow_tasks"),
+ audio_server_task = model_info.get("audio_server_task"),
+ audio_convert = model_info.get("audio_convert"),
+ audio_convert_route = model_info.get("audio_convert_route"),
audio_reference_text = model_info.get("audio_reference_text"),
audio_required_inputs = model_info.get("audio_required_inputs"),
audio_music = model_info.get("audio_music"),
@@ -22047,8 +22062,17 @@ def _audio_request_problem(
if not model_info.get("audio_music"):
return _audio_music_unavailable(label)
return None
- cloning = bool(run_inputs) and (run_inputs.get("workflow") == "clone" or bool(inputs))
- if cloning:
+ converting = bool(run_inputs) and run_inputs.get("workflow") == "convert"
+ cloning = (
+ bool(run_inputs)
+ and not converting
+ and (run_inputs.get("workflow") == "clone" or bool(inputs))
+ )
+ if converting:
+ problem = _audio_convert_problem(model_info, label, run_inputs)
+ if problem:
+ return problem
+ elif cloning:
if not workflows or "clone" not in workflows:
return "Load a model that can clone a voice."
if not inputs.get("reference"):
@@ -22070,6 +22094,13 @@ def _audio_request_problem(
and "clone" in workflows
):
return f"{label} speaks in a voice cloned from a recording. Open Clone and add one."
+ elif (
+ workflows is not None
+ and audio_type == "audiocpp_tts"
+ and "speak" not in workflows
+ and "convert" in workflows
+ ):
+ return f"{label} converts recordings. Open Convert and add one."
if (
"instruct" in (model_info.get("audio_required_inputs") or [])
and not str(payload.audio_instructions or "").strip()
@@ -22078,6 +22109,32 @@ def _audio_request_problem(
return None
+def _audio_convert_problem(model_info: dict, label: str, run_inputs: dict) -> Optional[str]:
+ caps = model_info.get("audio_convert")
+ if "convert" not in (model_info.get("audio_workflows") or []) or not caps:
+ return "Load a model that can convert a voice."
+ given = run_inputs.get("convert_inputs") or ()
+ if "source" not in given:
+ return "Add the recording to convert."
+ builtin = caps.get("target") == "builtin"
+ if builtin and "target" in given:
+ return f"{label} converts to its built-in voices; pick one under Built-in."
+ if not builtin and "target" not in given:
+ return "Add the target voice."
+ params = run_inputs.get("convert") or {}
+ mode = params.get("mode") or "speech"
+ if mode not in (caps.get("modes") or []):
+ return f"{label} does not convert {mode}."
+ if (
+ caps.get("style")
+ and params.get("style") == "target"
+ and mode == "speech"
+ and not str(params.get("source_text") or "").strip()
+ ):
+ return "Type what's said in the recording, or press Transcribe."
+ return None
+
+
async def _generate_tts_wav(
text: str,
payload: ChatCompletionRequest,
@@ -22091,7 +22148,8 @@ async def _generate_tts_wav(
) -> tuple[bytes, int, str, Optional[str]]:
"""Shared core of /audio/generate, /audio/speech and /audio/run. Returns
(wav_bytes, sample_rate, model_name, audio_type). ``run_inputs`` is /audio/run's ``workflow``,
- ``audio_inputs`` (role -> server-local WAV path), ``reference_text`` and ``speed``."""
+ ``audio_inputs`` (role -> server-local WAV path), ``reference_text``, ``speed`` and ``convert``;
+ a conversion fills ``audio_inputs`` via ``prepare_audio_inputs(model_info)``."""
# A named target must be budgeted against its own context after preflight.
if requested_model == _RELOAD_ONLY_MODEL:
_raise_if_prompt_leaves_no_speech_budget(text)
@@ -22193,6 +22251,7 @@ async def _generate_tts_wav(
"audio_inputs",
"reference_text",
"speed",
+ "convert",
"edit",
"music",
"output_dir",
@@ -22217,6 +22276,10 @@ async def _generate_tts_wav(
)
if _request_problem:
raise HTTPException(status_code = 400, detail = _request_problem)
+ _prepare_audio_inputs = (run_inputs or {}).get("prepare_audio_inputs")
+ if _prepare_audio_inputs is not None:
+ # The gen lambda reads run_inputs when it runs, so the prepared paths reach the worker.
+ run_inputs["audio_inputs"] = await asyncio.to_thread(_prepare_audio_inputs, model_info)
if audio_type == "minimax_music3" and not str(payload.audio_instructions or "").strip():
raise HTTPException(status_code = 400, detail = _MINIMAX_NEEDS_DESCRIPTION)
if audio_type == "audiocpp_music" and (run_inputs or {}).get("workflow") != "music":
@@ -22306,6 +22369,7 @@ def _persist_tts_clip(
audio_type: Optional[str],
extra_meta: Optional[dict] = None,
workflow: Optional[str] = None,
+ source_wav: Optional[str] = None,
) -> Optional[dict[str, Any]]:
"""Best-effort gallery save: persistence never fails the request that produced
the audio. Blocking, so callers run it off the event loop."""
@@ -22323,6 +22387,8 @@ def _persist_tts_clip(
}
meta.update({k: v for k, v in (extra_meta or {}).items() if v is not None})
try:
+ if source_wav is not None:
+ return audio_gallery.save(wav_bytes, meta, Path(source_wav))
return audio_gallery.save(wav_bytes, meta)
except Exception as exc: # noqa: BLE001 - log and serve the audio anyway
logger.warning("audio_gallery.persist_failed: %s", exc)
@@ -22904,21 +22970,135 @@ async def _run_music_workflow(
return AudioRunResponse(clips = clips, group_id = group_id, model = model_name)
+async def _run_audio_convert(
+ body: AudioRunRequest, request: Request, current_subject: str
+) -> AudioRunResponse:
+ from core.inference import audio_inputs
+ from core.inference.audio_cpp_convert import (
+ CONVERT_SOURCE_MAX_SECONDS,
+ CONVERT_TARGET_MAX_SECONDS,
+ )
+
+ if any(
+ name in body.model_fields_set and getattr(body, name) not in (None, False, 1)
+ for name in _MUSIC_ONLY_FIELDS
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = "A music mode, lyrics, variations and edits apply to Music only.",
+ )
+ params = body.convert.model_dump()
+ refs = {"source": body.inputs.source, "target": body.inputs.target}
+ if refs["source"] is None:
+ raise HTTPException(status_code = 400, detail = "Add the recording to convert.")
+ resolved: dict[str, Any] = {}
+ try:
+ for role, ref in refs.items():
+ if ref is not None:
+ resolved[role] = await asyncio.to_thread(
+ audio_inputs.resolve_source, ref.model_dump(exclude_none = True)
+ )
+ except audio_inputs.AudioInputError as exc:
+ raise _audio_source_error(exc) from None
+ source, target = resolved["source"], resolved.get("target")
+ max_seconds = {"source": CONVERT_SOURCE_MAX_SECONDS, "target": CONVERT_TARGET_MAX_SECONDS}
+ seen: dict[str, Any] = {}
+
+ def _prepare(model_info: dict) -> dict[str, str]:
+ seen["model_info"] = model_info
+ rules = model_info.get("audio_convert_rules") or {}
+ try:
+ seen["paths"] = {
+ role: str(
+ audio_inputs.prepared_path(
+ item,
+ int(rules.get(f"{role}_rate") or 16000),
+ max_seconds = max_seconds[role],
+ )
+ )
+ for role, item in resolved.items()
+ }
+ except audio_inputs.AudioInputError as exc:
+ raise _audio_source_error(exc) from None
+ return seen["paths"]
+
+ source_text = (body.inputs.source_text or "").strip() or None
+ payload = ChatCompletionRequest(
+ messages = [{"role": "user", "content": source.name}],
+ audio_options = body.options,
+ seed = body.seed,
+ )
+ wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(
+ source.name,
+ payload,
+ request,
+ current_subject,
+ run_inputs = {
+ "workflow": "convert",
+ "audio_inputs": None,
+ "convert": {**params, "source_text": source_text},
+ "convert_inputs": tuple(resolved),
+ "prepare_audio_inputs": _prepare,
+ },
+ )
+ builtin = None
+ if target is not None:
+ target_name = target.name
+ else:
+ voices = ((seen.get("model_info") or {}).get("audio_convert") or {}).get("builtin_voices")
+ voices = voices or [{"id": params.get("voice"), "label": "Built-in"}]
+ voice = next((v for v in voices if v.get("id") == params.get("voice")), voices[0])
+ builtin, target_name = voice.get("id"), voice.get("label") or voice.get("id")
+ extra_meta: dict[str, Any] = {
+ "role": "output",
+ "source_name": source.name,
+ "reference_name": target_name,
+ "target_builtin": builtin,
+ "settings": {
+ **{key: params.get(key) for key in ("mode", "pitch", "pitch_auto", "style")},
+ "options": _audio_run_settings(body, False)["options"],
+ },
+ }
+ if source.kind == "clip":
+ extra_meta["source_clip_id"] = source.id
+ elif source.kind == "input":
+ extra_meta["source_input_id"] = source.id
+ if target is not None:
+ key = {"voice": "voice_id", "clip": "target_clip_id", "input": "target_input_id"}
+ if target.kind in key:
+ extra_meta[key[target.kind]] = target.id
+ record = await asyncio.to_thread(
+ _persist_tts_clip,
+ wav_bytes,
+ sample_rate,
+ f"{source.name} → {target_name}",
+ model_name,
+ audio_type,
+ extra_meta,
+ "convert",
+ # An upload expires within a day; a history clip does not.
+ (seen.get("paths") or {}).get("source") if source.kind == "input" else None,
+ )
+ return _audio_run_response(record, wav_bytes, sample_rate, model_name)
+
+
@router.post("/audio/run", response_model = AudioRunResponse)
async def run_audio_workflow(
body: AudioRunRequest,
request: Request,
current_subject: str = Depends(get_current_subject),
):
- """Run one Audio page workflow (Clone, Speak in a saved voice, Edit, Music or Separate); clips
- go to history. Audio is named by id and resolved here in the caller's account; the worker gets
- a prepared copy's path, never bytes."""
+ """Run one Audio page workflow (Clone, Speak in a saved voice, Edit, Convert, Music or
+ Separate); clips go to history. Audio is named by id and resolved here in the caller's account;
+ the worker gets a prepared copy's path, never bytes."""
if body.workflow == "separate":
return await _run_separation(body, request, current_subject)
import base64
from core.inference import audio_inputs
+ if body.workflow == "convert":
+ return await _run_audio_convert(body, request, current_subject)
if body.workflow == "edit":
return await _run_audio_edit(body, request, current_subject)
if body.workflow == "music":
@@ -45496,6 +45676,25 @@ async def get_gallery_audio_file(
)
+@studio_router.get("/audio/gallery/{audio_id}/source/file")
+async def get_gallery_audio_source_file(
+ audio_id: str, current_subject: str = Depends(get_current_subject)
+):
+ """The upload a conversion clip converted, when it kept a copy."""
+ from core.inference import audio_gallery
+
+ path = await asyncio.to_thread(audio_gallery.owned_source_path, audio_id)
+ if path is None:
+ raise HTTPException(status_code = 404, detail = "Audio not found.")
+ from fastapi.responses import FileResponse
+
+ return FileResponse(
+ path,
+ media_type = "audio/wav",
+ headers = {"Cache-Control": "private, max-age=31536000, immutable"},
+ )
+
+
@studio_router.patch("/audio/gallery/{audio_id}", response_model = AudioGalleryItem)
async def update_gallery_audio_flags(
audio_id: str,
@@ -45573,7 +45772,7 @@ async def delete_gallery_audio(audio_id: str, current_subject: str = Depends(get
@studio_router.delete("/audio/gallery")
async def clear_gallery_audio(
- workflow: Optional[Literal["speak", "clone", "edit", "music", "separate"]] = None,
+ workflow: Optional[Literal["speak", "clone", "edit", "convert", "music", "separate"]] = None,
current_subject: str = Depends(get_current_subject),
):
from core.inference import audio_gallery
@@ -45651,6 +45850,7 @@ async def transcribe_audio_input(
"""What is said in an input (or, with the path id ``source``, a history clip or saved voice),
over the first 30 s a clone uses. Saves nothing to the transcript history."""
from core.inference import audio_inputs
+ from core.inference.audio_cpp_convert import CONVERT_SOURCE_MAX_SECONDS
ref = {
# ``source`` (or ``-``) stands for "the clip_id or voice_id in the query". A real input id
@@ -45662,7 +45862,10 @@ async def transcribe_audio_input(
def _prepare() -> bytes:
source = audio_inputs.resolve_source(ref)
- cap = None if source.kind == "voice" else audio_inputs.REFERENCE_MAX_SECONDS
+ if body.purpose == "convert":
+ cap = CONVERT_SOURCE_MAX_SECONDS
+ else:
+ cap = None if source.kind == "voice" else audio_inputs.REFERENCE_MAX_SECONDS
path = audio_inputs.prepared_path(source, 16000, max_seconds = cap)
return path.read_bytes()
diff --git a/studio/backend/tests/multi_account/factories_media.py b/studio/backend/tests/multi_account/factories_media.py
index 9f6bc8e441d..110d5ed9af1 100644
--- a/studio/backend/tests/multi_account/factories_media.py
+++ b/studio/backend/tests/multi_account/factories_media.py
@@ -110,6 +110,31 @@ def seed_audio(account) -> dict[str, str]:
return {"audio_id": record["id"]}
+@seeder("media-audio-converted")
+def seed_converted_audio(account) -> dict[str, str]:
+ import tempfile
+ from pathlib import Path
+
+ from core.inference import audio_gallery
+ from utils.account_context import run_as
+
+ meta = {
+ "prompt": SENTINEL,
+ "model": "media/none",
+ "audio_type": "speech",
+ "workflow": "convert",
+ "sample_rate": 8000,
+ "duration_s": 0.008,
+ "created_at": "2026-01-01T00:00:00+00:00",
+ }
+ # A conversion of an upload keeps the recording it converted beside the clip.
+ with tempfile.TemporaryDirectory() as scratch:
+ source = Path(scratch) / "source.wav"
+ source.write_bytes(_wav_bytes())
+ record = run_as(account, audio_gallery.save, _wav_bytes(), meta, source)
+ return {"audio_id": record["id"]}
+
+
@seeder("media-audio-project")
def seed_audio_and_project(account, actor: str = "right") -> dict[str, str]:
params = seed_audio(account)
@@ -330,6 +355,7 @@ FACTORIES = {
"media-image-project", {"project_id": MEDIA_PROJECT_ID}, fragment = "sandbox"
),
"routes.inference:GET:/audio/gallery/{audio_id}/file": Factory("media-audio"),
+ "routes.inference:GET:/audio/gallery/{audio_id}/source/file": Factory("media-audio-converted"),
"routes.inference:PATCH:/audio/gallery/{audio_id}": Factory(
"media-audio", {"archived": True}, fragment = SENTINEL
),
diff --git a/studio/backend/tests/test_account_media_isolation.py b/studio/backend/tests/test_account_media_isolation.py
index bbc24c00bfb..df2ec1802d8 100644
--- a/studio/backend/tests/test_account_media_isolation.py
+++ b/studio/backend/tests/test_account_media_isolation.py
@@ -259,6 +259,15 @@ def _save_input(account):
return run_as(account, lambda: asyncio.run(audio_inputs.save_stream(chunks(), "me.wav"))[0])
+def _save_voice(account):
+ from core.inference import audio_inputs, audio_voices
+ record = _save_input(account)
+ return run_as(
+ account,
+ lambda: audio_voices.create(audio_inputs.input_path(record["id"]), {"name": "private"}),
+ )
+
+
@pytest.mark.parametrize("account,other", [(ALICE, BOB), (BOB, ALICE)])
def test_audio_inputs_voices_and_clips_do_not_resolve_another_accounts_ids(
account, other, monkeypatch
@@ -306,6 +315,40 @@ def test_inputs_and_voices_live_in_the_accounts_audio_folder(tmp_path):
assert run_as(OWNER, audio_inputs.inputs_dir) == tmp_path / "audio" / "inputs"
[email protected]("role", ["source", "target"])
[email protected]("kind", ["input", "clip", "voice"])
+def test_a_conversion_does_not_resolve_another_accounts_audio(role, kind, monkeypatch):
+ async def fail(*_args, **_kwargs):
+ raise AssertionError("a foreign id must stop before generation")
+
+ monkeypatch.setattr(inference, "_generate_tts_wav", fail)
+ if kind == "input":
+ foreign = {"input_id": _save_input(ALICE)["id"]}
+ elif kind == "clip":
+ foreign = {"clip_id": run_as(ALICE, _save, "audio")["id"]}
+ else:
+ foreign = {"voice_id": _save_voice(ALICE)["id"]}
+ own = {"input_id": _save_input(BOB)["id"]}
+ inputs = {"source": own, "target": own, role: foreign}
+ app = FastAPI()
+
+ async def subject():
+ token = bind_account(BOB)
+ try:
+ yield BOB.username
+ finally:
+ reset_account(token)
+
+ app.dependency_overrides[get_current_subject] = subject
+ app.dependency_overrides[authenticated_via_api_key] = lambda: False
+ app.include_router(inference.router, prefix = "/api/inference")
+ with TestClient(app) as client:
+ response = client.post(
+ "/api/inference/audio/run", json = {"workflow": "convert", "inputs": inputs}
+ )
+ assert response.status_code == 404, response.text
+
+
@pytest.mark.parametrize("account,other", [(ALICE, BOB), (BOB, ALICE)])
def test_transcribe_sources_and_transcripts_do_not_resolve_another_accounts_ids(account, other):
from core.inference import transcript_gallery
diff --git a/studio/backend/tests/test_audio_cpp_convert_requests.py b/studio/backend/tests/test_audio_cpp_convert_requests.py
new file mode 100644
index 00000000000..eed20b96742
--- /dev/null
+++ b/studio/backend/tests/test_audio_cpp_convert_requests.py
@@ -0,0 +1,452 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The exact requests audio.cpp receives for a voice conversion, per family, on a fake server."""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import wave
+
+import pytest
+
+from core.inference import audio_cpp_backend as acb
+from core.inference import audio_cpp_convert as acc
+from core.inference import audio_cpp_files
+from core.inference import audio_cpp_models as acm
+from core.inference import audio_cpp_server as srv
+from core.inference.audio_cpp_models import (
+ AUDIO_CPP_REPO,
+ AudioCppModel,
+ AudioCppVariant,
+ RepoFile,
+)
+from core.inference.audio_errors import AudioRuntimeError
+
+SOURCE = "/srv/accounts/a/audio/inputs/0123.16000.mono.m300.wav"
+TARGET = "/srv/accounts/a/audio/inputs/4567.16000.mono.m30.wav"
+TRANSCRIPT = "Okay, I'm Cemo and what you just heard wasn't a human voice."
+
+# What the runtime specs declare (model_specs/*.json at the pin), as option_schema cleans them.
+_RVC_OPTIONS = (
+ {"name": "voice_id", "type": "enum", "values": ["default", "manthos", "chocola", "fraise"]},
+ {"name": "retrieval_blend", "type": "float", "min": 0.0, "max": 1.0, "default": 0.0},
+ {"name": "semitone_shift", "type": "int", "default": 0},
+ {"name": "rms_mix_rate", "type": "float", "default": 0.25},
+ {"name": "unvoiced_protection", "type": "float", "min": 0.0, "max": 1.0, "default": 0.33},
+)
+_SEED_VC_OPTIONS = (
+ {
+ "name": "route",
+ "type": "enum",
+ "values": ["v2_vc", "v1_svc", "v1_whisper_bigvgan_vc", "v1_xlsr_hift_vc"],
+ },
+ {"name": "length_adjust", "type": "float", "min": 0.0, "default": 1.0},
+ {"name": "similarity_guidance_scale", "type": "float", "min": 0.0, "default": 0.7},
+ {"name": "voice_anonymization", "type": "bool", "default": False},
+ {"name": "auto_f0_adjust", "type": "bool", "default": False},
+ {"name": "semitone_shift", "type": "int", "default": 0},
+)
+
+
+_BLANK_OPTION = {
+ "description": "",
+ "required": False,
+ "default": None,
+ "min": None,
+ "max": None,
+ "values": None,
+}
+
+
+def _wav(rate) -> bytes:
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as w:
+ w.setnchannels(1)
+ w.setsampwidth(2)
+ w.setframerate(rate)
+ w.writeframes(b"\x00\x00" * 240)
+ return buf.getvalue()
+
+
+def _model(family, options = ()) -> AudioCppModel:
+ policy = acm.FAMILIES[family]
+ variant = AudioCppVariant("Q8_0", (RepoFile("m-q8_0.gguf", 1),), "m-q8_0.gguf")
+ return AudioCppModel(
+ id = f"{AUDIO_CPP_REPO}/{family}",
+ repo_id = AUDIO_CPP_REPO,
+ folder = family,
+ display_name = family,
+ family = family,
+ task = policy.task,
+ server_task = policy.default_server_task,
+ variant = variant,
+ variants = (variant,),
+ default_variant = "Q8_0",
+ options = tuple({**_BLANK_OPTION, **o} for o in options),
+ speaks = policy.speaks,
+ clone = policy.clone,
+ convert = policy.convert,
+ )
+
+
+class _Server:
+ model_id = "studio-test"
+ backend = "cpu"
+
+ def __init__(
+ self,
+ model,
+ fail = None,
+ ):
+ self.model = model
+ self.calls: list[tuple[str, dict]] = []
+ self.fail = fail
+
+ def alive(self):
+ return True
+
+ def stop(self):
+ pass
+
+ def post_json(self, path, payload, **_kwargs):
+ self.calls.append((path, json.loads(json.dumps(payload))))
+ if self.fail is not None:
+ raise self.fail
+ if path == "/v1/tasks/run":
+ body = {"audio": base64.b64encode(_wav(16000)).decode(), "sample_rate": 16000}
+ return "application/json", json.dumps(body).encode()
+ return "audio/wav", _wav(24000)
+
+
+def started(monkeypatch):
+ servers: list[_Server] = []
+
+ def start(served, path, **_kwargs):
+ servers.append(_Server(served))
+ return servers[-1]
+
+ monkeypatch.setattr(acb.AudioCppServer, "start", start)
+ monkeypatch.setattr(audio_cpp_files, "materialize", lambda model: "/models/m.gguf")
+ return servers
+
+
+def _backend(model, started):
+ backend = acb.AudioCppBackend()
+ backend._model = model
+ backend.models = {model.id: {"is_audio": True, **acb.model_info_fields(model)}}
+ backend.active_model_name = model.id
+ backend._start_server(model)
+ started.clear()
+ return backend
+
+
+def _convert(
+ backend,
+ inputs = None,
+ options = None,
+ seed = 7,
+ **convert,
+):
+ wav, rate = backend.generate_audio_response(
+ "ignored",
+ workflow = "convert",
+ audio_inputs = {"source": SOURCE, "target": TARGET} if inputs is None else inputs,
+ audio_options = options,
+ seed = seed,
+ convert = convert,
+ )
+ assert wav[:4] == b"RIFF" and rate == 16000
+ path, body = backend._server.calls[-1]
+ assert path == "/v1/tasks/run"
+ assert body["model"] == "studio-test"
+ return body["request"]
+
+
+def test_rvc_converts_to_a_builtin_voice_by_options_voice_id(started):
+ backend = _backend(_model("rvc", _RVC_OPTIONS), started)
+ request = _convert(
+ backend,
+ inputs = {"source": SOURCE},
+ options = {"retrieval_blend": 0.6, "voice_id": "fraise", "semitone_shift": 9},
+ voice = "manthos",
+ pitch = 3,
+ )
+ # Strict: no seed, no voice_ref, no language; the typed fields win over driven options.
+ assert request == {
+ "audio": SOURCE,
+ "options": {"retrieval_blend": "0.6", "voice_id": "manthos", "semitone_shift": "3"},
+ }
+ assert _convert(backend, inputs = {"source": SOURCE}) == {
+ "audio": SOURCE,
+ "options": {"voice_id": "default"},
+ }
+ # Auto is not offered on RVC, so it never becomes auto_f0_adjust.
+ request = _convert(backend, inputs = {"source": SOURCE}, pitch = -5, pitch_auto = True)
+ assert request["options"] == {"voice_id": "default", "semitone_shift": "-5"}
+ assert started == []
+
+
+ "family, target, match",
+ [("rvc", TARGET, "built-in voices"), ("meanvc2", None, "Add the target voice.")],
+)
+def test_a_target_the_family_cannot_take_is_refused(family, target, match):
+ with pytest.raises(acc.ConvertRequestError, match = match):
+ acc.convert_request(_model(family), mode = "speech", source = SOURCE, target = target)
+
+
+def test_meanvc2_sends_audio_voice_ref_and_seed():
+ request = acc.convert_request(
+ _model("meanvc2"), mode = "speech", source = SOURCE, target = TARGET, pitch = 4, seed = 7
+ )
+ assert request == {"audio": SOURCE, "voice_ref": TARGET, "seed": "7"}
+
+
+def test_seed_vc_speech_sends_no_pitch_and_its_route_by_model_entry(started):
+ model = _model("seed_vc", _SEED_VC_OPTIONS)
+ backend = _backend(model, started)
+ request = _convert(
+ backend,
+ options = {
+ "length_adjust": 1.25,
+ "similarity_guidance_scale": 1.2,
+ "semitone_shift": 5,
+ "auto_f0_adjust": True,
+ "f0_condition": True,
+ },
+ pitch = 5,
+ pitch_auto = True,
+ )
+ assert request == {
+ "audio": SOURCE,
+ "voice_ref": TARGET,
+ "options": {"length_adjust": "1.25", "similarity_guidance_scale": "1.2"},
+ "seed": "7",
+ }
+ # The default engine (v2_vc) runs on the loaded session: no restart, no route sent.
+ assert started == []
+ # Another engine reloads the model with that route as its default, never in the request.
+ request = _convert(backend, options = {"route": "v1_xlsr_hift_vc"})
+ assert "route" not in request and "route" not in request.get("options", {})
+ (server,) = started
+ assert server.model.server_task == "vc"
+ assert server.model.model_options["default_request_options"] == {"route": "v1_xlsr_hift_vc"}
+ assert backend.models[model.id]["audio_convert_route"] == "v1_xlsr_hift_vc"
+ _convert(backend, options = {"route": "v1_xlsr_hift_vc"})
+ assert len(started) == 1
+ _convert(backend)
+ assert len(started) == 2 and "default_request_options" not in started[-1].model.model_options
+ assert backend.models[model.id]["audio_convert_route"] == "v2_vc"
+
+
+def test_seed_vc_singing_reloads_under_svc_and_sends_its_pitch(started):
+ model = _model("seed_vc", _SEED_VC_OPTIONS)
+ backend = _backend(model, started)
+ # The runtime applies the manual shift on top of the matched pitch, as seed-vc does.
+ request = _convert(backend, mode = "singing", pitch = 12, pitch_auto = True)
+ assert request["options"] == {"auto_f0_adjust": "true", "semitone_shift": "12"}
+ request = _convert(backend, mode = "singing", pitch_auto = True)
+ assert request["options"] == {"auto_f0_adjust": "true"}
+ (server,) = started
+ assert server.model.server_task == "svc"
+ assert backend.models[model.id]["audio_server_task"] == "svc"
+ assert backend.models[model.id]["audio_convert_route"] == "v1_svc"
+ # A speech route sent in singing is dropped: v1_svc is the only singing route.
+ request = _convert(backend, mode = "singing", pitch = -3, options = {"route": "v1_xlsr_hift_vc"})
+ assert request["options"] == {"semitone_shift": "-3"}
+ assert len(started) == 1
+ _convert(backend)
+ assert [s.model.server_task for s in started] == ["svc", "vc"]
+
+
+def test_chatterbox_converts_under_vc_and_clones_back_under_clon(started):
+ model = _model("chatterbox")
+ backend = _backend(model, started)
+ request = _convert(
+ backend,
+ options = {"s3gen_cfg_rate": 1.0, "num_inference_steps": 6, "exaggeration": 0.9},
+ )
+ # Exaggeration is clone-only and never reaches a conversion.
+ assert request == {
+ "audio": SOURCE,
+ "voice_ref": TARGET,
+ "options": {"s3gen_cfg_rate": "1", "num_inference_steps": "6"},
+ "seed": "7",
+ }
+ assert [s.model.server_task for s in started] == ["vc"]
+ assert backend.models[model.id]["audio_server_task"] == "vc"
+ _convert(backend)
+ assert len(started) == 1
+ backend.generate_audio_response(
+ "Hello there.", workflow = "clone", audio_inputs = {"reference": TARGET}
+ )
+ assert [s.model.server_task for s in started] == ["vc", "clon"]
+ assert backend.models[model.id]["audio_server_task"] == "clon"
+ assert started[-1].calls[-1][0] == "/v1/audio/speech"
+
+
+def test_vevo2_keeps_the_source_style_with_source_audio_and_target_voice(started):
+ model = _model("vevo2")
+ backend = _backend(model, started)
+ assert _convert(backend, pitch_auto = True, pitch = 4) == {
+ "source_audio": SOURCE,
+ "target_voice": TARGET,
+ "seed": "7",
+ }
+ assert _convert(backend, pitch = 4)["options"] == {
+ "use_pitch_shift": "true",
+ "source_shift_steps": "4",
+ }
+ assert _convert(backend, pitch = 0)["options"] == {"use_pitch_shift": "false"}
+ assert _convert(backend, options = {"num_inference_steps": 20})["options"] == {
+ "num_inference_steps": "20"
+ }
+ _convert(backend, mode = "singing", pitch = -2)
+ assert [s.model.server_task for s in started] == ["vc", "svc"]
+ assert started[-1].calls[-1][1]["request"]["options"] == {
+ "use_pitch_shift": "true",
+ "source_shift_steps": "-2",
+ }
+ assert backend.models[model.id]["audio_convert_route"] is None
+
+
+def test_vevo2_takes_the_target_style_through_the_source_transcript():
+ model = _model("vevo2")
+ request = acc.convert_request(
+ model,
+ mode = "speech",
+ source = SOURCE,
+ target = TARGET,
+ style = "target",
+ source_text = f" {TRANSCRIPT} ",
+ pitch = 5,
+ seed = 7,
+ )
+ # No pitch under the target's style: its prosody is the target's.
+ assert request == {
+ "source_audio": SOURCE,
+ "target_voice": TARGET,
+ "route": "style_converted_vc",
+ "target_text": TRANSCRIPT,
+ "style_ref": TARGET,
+ "seed": "7",
+ }
+ with pytest.raises(acc.ConvertRequestError, match = "Type what's said in the recording"):
+ acc.convert_request(model, mode = "speech", source = SOURCE, target = TARGET, style = "target")
+ singing = acc.convert_request(
+ model, mode = "singing", source = SOURCE, target = TARGET, style = "target"
+ )
+ assert "route" not in singing and "target_text" not in singing
+
+
+def test_a_mode_the_model_lacks_is_refused(started):
+ backend = _backend(_model("meanvc2"), started)
+ with pytest.raises(RuntimeError, match = "does not convert singing"):
+ _convert(backend, mode = "singing")
+ with pytest.raises(acc.ConvertRequestError):
+ acc.served_model(_model("kokoro_tts"), "speech", {})
+
+
+def test_a_runtime_refusal_is_an_audio_runtime_error(started):
+ backend = _backend(_model("meanvc2"), started)
+ backend._server.fail = srv.AudioCppRequestError(500, "MeanVC2 requires --voice-ref target")
+ with pytest.raises(AudioRuntimeError, match = "MeanVC2 requires --voice-ref target") as info:
+ _convert(backend)
+ assert info.value.status == 500
+
+
+def test_status_fields_describe_convert_per_family():
+ rvc = acb.model_info_fields(_model("rvc", _RVC_OPTIONS))
+ assert rvc["audio_workflows"] == ["convert"]
+ assert rvc["audio_convert"] == {
+ "modes": ["speech"],
+ "target": "builtin",
+ "builtin_voices": [
+ {"id": "default", "label": "Default"},
+ {"id": "manthos", "label": "Manthos"},
+ {"id": "chocola", "label": "Chocola"},
+ {"id": "fraise", "label": "Fraise"},
+ ],
+ "pitch": {"speech": {"auto": False, "shift_with_auto": False}},
+ "style": False,
+ "route_reloads": False,
+ "source_max_seconds": 300,
+ }
+ assert rvc["audio_workflow_tasks"] == {"convert": "vc"}
+ assert (rvc["audio_server_task"], rvc["audio_convert_route"]) == ("vc", None)
+ assert [o["name"] for o in rvc["audio_options_by_workflow"]["convert"]] == [
+ "retrieval_blend",
+ "rms_mix_rate",
+ "unvoiced_protection",
+ ]
+ assert rvc["audio_convert_rules"] == {"source_rate": 16000, "target_rate": None}
+
+ seed_vc = acb.model_info_fields(_model("seed_vc", _SEED_VC_OPTIONS))
+ assert seed_vc["audio_convert"]["modes"] == ["speech", "singing"]
+ assert seed_vc["audio_convert"]["pitch"] == {"singing": {"auto": True, "shift_with_auto": True}}
+ assert seed_vc["audio_convert"]["route_reloads"] is True
+ assert seed_vc["audio_workflow_tasks"] == {"convert": "vc", "convert:singing": "svc"}
+ assert seed_vc["audio_convert_route"] == "v2_vc"
+ assert seed_vc["audio_convert_rules"] == {"source_rate": 44100, "target_rate": 44100}
+
+ vevo2 = acb.model_info_fields(_model("vevo2"))
+ assert vevo2["audio_workflows"] == ["clone", "convert"]
+ assert vevo2["audio_convert"]["pitch"] == {
+ "speech": {"auto": True, "shift_with_auto": False},
+ "singing": {"auto": True, "shift_with_auto": False},
+ }
+ assert vevo2["audio_convert"]["style"] is True
+ assert vevo2["audio_workflow_tasks"] == {
+ "clone": "tts",
+ "convert": "vc",
+ "convert:singing": "svc",
+ }
+ assert vevo2["audio_server_task"] == "tts"
+
+ chatterbox = acb.model_info_fields(_model("chatterbox"))
+ assert chatterbox["audio_workflow_tasks"] == {"clone": "clon", "convert": "vc"}
+ assert chatterbox["audio_convert"]["pitch"] == {}
+ assert chatterbox["audio_convert_rules"] == {"source_rate": 16000, "target_rate": 24000}
+ assert acb.model_info_fields(_model("meanvc2"))["audio_convert"]["pitch"] == {}
+
+ kokoro = acb.model_info_fields(_model("kokoro_tts"))
+ assert kokoro["audio_convert"] is None and kokoro["audio_options_by_workflow"] is None
+ assert kokoro["audio_workflow_tasks"] == {"speak": "tts"}
+
+
+def test_the_seed_vc_server_entry_carries_its_route(monkeypatch, tmp_path):
+ seen: dict = {}
+ binary = tmp_path / srv.BINARY_NAME
+ binary.write_bytes(b"")
+
+ def spy(command, *_args, **_kwargs):
+ with open(command[command.index("--config") + 1], encoding = "utf-8") as f:
+ seen.update(json.load(f))
+ raise OSError("not launched in tests")
+
+ monkeypatch.setattr(srv, "ensure_binary", lambda: str(binary))
+ monkeypatch.setattr(srv, "model_runtime_problem", lambda model, binary = None: None)
+ monkeypatch.setattr(srv, "select_backend", lambda binary, force_cpu: "cpu")
+ monkeypatch.setattr(srv.subprocess, "Popen", spy)
+ served, rest = acc.served_model(
+ _model("seed_vc", _SEED_VC_OPTIONS), "speech", {"route": "v1_whisper_bigvgan_vc", "x": 1}
+ )
+ assert rest == {"x": 1}
+ with pytest.raises(srv.AudioCppUnavailableError):
+ srv.AudioCppServer.start(served, "/models/m.gguf")
+ (entry,) = seen["models"]
+ assert entry["task"] == "vc"
+ assert entry["default_request_options"] == {"route": "v1_whisper_bigvgan_vc"}
+
+
+def test_a_five_minute_source_transcript_fits_the_request():
+ from models.inference import AudioRunInputs
+
+ # ~150 words a minute for the 300 s Convert source cap is ~5000 characters.
+ text = "word " * 1100
+ inputs = AudioRunInputs(source = {"input_id": "a" * 32}, source_text = text)
+ assert inputs.source_text == text
diff --git a/studio/backend/tests/test_audio_cpp_models.py b/studio/backend/tests/test_audio_cpp_models.py
index da2bd272244..4bf2fe8755c 100644
--- a/studio/backend/tests/test_audio_cpp_models.py
+++ b/studio/backend/tests/test_audio_cpp_models.py
@@ -526,6 +526,16 @@ _CLONE_TABLE = {
}
+# family -> (load task, mode -> server task, target, source rate)
+_CONVERT_TABLE = {
+ "rvc": ("vc", {"speech": "vc"}, "builtin", 16000),
+ "seed_vc": ("vc", {"speech": "vc", "singing": "svc"}, "audio", 44100),
+ "meanvc2": ("vc", {"speech": "vc"}, "audio", 16000),
+ "chatterbox": ("clon", {"speech": "vc"}, "audio", 16000),
+ "vevo2": ("tts", {"speech": "vc", "singing": "svc"}, "audio", 24000),
+}
+
+
def test_every_task_family_binds_its_workflows():
workflow_for = {"tts": "speak", "music": "music", "asr": "transcribe", "sep": "separate"}
endpoint_for = {
@@ -540,9 +550,26 @@ def test_every_task_family_binds_its_workflows():
if not family.task:
assert bindings == {}, family.family
continue
+ if family.convert is not None:
+ load_task, modes, target, source_rate = _CONVERT_TABLE[family.family]
+ assert family.default_server_task == load_task, family.family
+ assert family.convert.server_tasks == modes, family.family
+ assert family.convert.target == target, family.family
+ convert = bindings["convert"]
+ assert (convert.endpoint, convert.server_task, family.convert.source_rate) == (
+ "tasks",
+ modes["speech"],
+ source_rate,
+ ), family.family
+ assert convert.inputs == ("source", "target", "source_text")
+ if family.clone is None:
+ assert list(bindings) == ["convert"], family.family
+ continue
if family.clone is not None:
server_task, speaks, reference_text = _CLONE_TABLE[family.family]
expected = (["speak"] if speaks else []) + ["clone"]
+ if family.convert is not None:
+ expected.append("convert")
assert list(bindings) == expected, family.family
assert family.default_server_task == server_task, family.family
assert family.clone.reference_text == reference_text, family.family
@@ -555,6 +582,10 @@ def test_every_task_family_binds_its_workflows():
assert binding.endpoint == endpoint_for[workflow]
assert binding.server_task == family.default_server_task
assert set(_CLONE_TABLE) == {f.family for f in acm.FAMILIES.values() if f.clone is not None}
+ assert set(_CONVERT_TABLE) == {f.family for f in acm.FAMILIES.values() if f.convert is not None}
+ # Chatterbox-Turbo is its own family and still speaks.
+ assert list(acm.FAMILIES["chatterbox_turbo"].workflows) == ["speak"]
+ assert acm.FAMILIES["chatterbox_turbo"].default_server_task == "tts"
assert acm.family_from_names(["Chatterbox-Turbo-GGUF", "chatterbox-turbo-q8_0.gguf"]) == (
"chatterbox_turbo"
)
@@ -586,7 +617,9 @@ def test_edit_families_bind_the_edit_workflow():
for names in (["DotTTS-MF-GGUF"], ["DotTTS-SOAR-GGUF"], []):
assert list(acm.family_policy("dots_tts", names = names).workflows) == ["speak"], names
vevo2, firered = acm.FAMILIES["vevo2"], acm.FAMILIES["firered_audio"]
- assert list(vevo2.workflows) == list(firered.workflows) == ["clone", "edit"]
+ assert list(firered.workflows) == ["clone", "edit"]
+ # Vevo2 also converts a voice.
+ assert list(vevo2.workflows) == ["clone", "edit", "convert"]
# Vevo2 edits in an s2s session though it loads as tts.
assert vevo2.default_server_task == "tts"
assert (vevo2.workflows["edit"].server_task, vevo2.workflows["edit"].route) == (
@@ -2159,6 +2192,115 @@ def test_required_inputs_come_from_the_raw_spec(hub):
assert acm.resolve(f"{AUDIO_CPP_REPO}/{folder}", network = False).required_inputs == ()
+def _opt_spec(name, type_, **kw):
+ return {"name": name, "type": type_, "description": name, **kw}
+
+
+# The request options of the runtime's rvc and seed_vc specs (model_specs/*.json at the pin).
+RVC_SPEC = {
+ "family": "rvc",
+ "tasks": ["vc"],
+ "options": {
+ "request": [
+ _opt_spec("voice_id", "enum", values = ["default", "manthos", "chocola", "fraise"]),
+ _opt_spec("voice_model_path", "path"),
+ _opt_spec("pitch_extractor", "enum", values = ["rmvpe"], default = "rmvpe"),
+ _opt_spec("pitch_path", "path"),
+ _opt_spec("retrieval_index_path", "path"),
+ _opt_spec("retrieval_blend", "float", default = 0.0, min = 0.0, max = 1.0),
+ _opt_spec("semitone_shift", "int", default = 0),
+ _opt_spec("pitch_filter_radius", "int", default = 3, min = 0),
+ _opt_spec("output_sample_rate", "int", default = 0, min = 0),
+ _opt_spec("rms_mix_rate", "float", default = 0.25),
+ _opt_spec("unvoiced_protection", "float", default = 0.33, min = 0.0, max = 1.0),
+ _opt_spec("speaker_id", "int", default = 0, min = 0),
+ _opt_spec("audio_pad_duration_sec", "int", default = 1, min = 1),
+ _opt_spec("split_query_sec", "int", default = 5, min = 1),
+ ]
+ },
+}
+SEED_VC_SPEC = {
+ "family": "seed_vc",
+ "tasks": ["vc", "svc"],
+ "options": {
+ "request": [
+ _opt_spec(
+ "route",
+ "enum",
+ values = ["v2_vc", "v1_svc", "v1_whisper_bigvgan_vc", "v1_xlsr_hift_vc"],
+ ),
+ _opt_spec("length_adjust", "float", default = 1.0, min = 0.0),
+ _opt_spec("num_inference_steps", "int", default = 30, min = 1),
+ _opt_spec("inference_guidance_scale", "float", default = 0.7, min = 0.0),
+ _opt_spec("intelligibility_guidance_scale", "float", default = 0.7, min = 0.0),
+ _opt_spec("similarity_guidance_scale", "float", default = 0.7, min = 0.0),
+ _opt_spec("voice_anonymization", "bool", default = False),
+ _opt_spec("seed", "int", min = 0),
+ _opt_spec("noise_path", "path"),
+ _opt_spec("f0_condition", "bool"),
+ _opt_spec("auto_f0_adjust", "bool", default = False),
+ _opt_spec("semitone_shift", "int", default = 0),
+ ]
+ },
+}
+MEANVC2_SPEC = {
+ "family": "meanvc2",
+ "tasks": ["vc"],
+ "options": {"request": [_opt_spec("seed", "int", default = 42, min = 0)]},
+}
+_NEVER_IN_CONVERT = {
+ "voice_model_path",
+ "retrieval_index_path",
+ "pitch_path",
+ "noise_path",
+ "audio_pad_duration_sec",
+ "seed",
+}
+
+
+def test_voice_conversion_families_resolve_and_offer_convert_alone(hub):
+ snap = _snapshot(hub)
+ _put(snap, "RVC-GGUF/rvc-f16.gguf", _gguf_bytes(family = "rvc", spec = RVC_SPEC))
+ for name in ("seed-vc-mlx-q8_0", "seed-vc-mlx-f16", "seed-vc-mlx-q4_k"):
+ _put(
+ snap,
+ f"SeedVC-MLX-GGUF/{name}.gguf",
+ _gguf_bytes(family = "seed_vc", spec = SEED_VC_SPEC),
+ )
+ for name in ("meanvc2-120ms-40ms-fp32", "meanvc2-120ms-40ms-q4_k"):
+ _put(
+ snap,
+ f"MeanVC2-GGUF/{name}.gguf",
+ _gguf_bytes(family = "meanvc2", spec = MEANVC2_SPEC),
+ )
+ rvc = acm.resolve(f"{AUDIO_CPP_REPO}/RVC-GGUF", network = False)
+ seed_vc = acm.resolve(f"{AUDIO_CPP_REPO}/SeedVC-MLX-GGUF", network = False)
+ meanvc2 = acm.resolve(f"{AUDIO_CPP_REPO}/MeanVC2-GGUF", network = False)
+ for model in (rvc, seed_vc, meanvc2):
+ assert model.unsupported is None, model.family
+ assert (model.task, model.server_task, model.speaks) == ("tts", "vc", False)
+ assert list(model.workflows) == ["convert"]
+ acm.require_runnable(model, "tts")
+ names = {o["name"] for o in model.convert_options}
+ assert not names & _NEVER_IN_CONVERT, model.family
+ assert not names & acm.CONVERT_DRIVEN_OPTIONS, model.family
+ assert seed_vc.variant.key == "Q8_0"
+ assert meanvc2.variant.key == "FP32"
+ assert [o["name"] for o in rvc.convert_options] == [
+ "retrieval_blend",
+ "pitch_filter_radius",
+ "output_sample_rate",
+ "rms_mix_rate",
+ "unvoiced_protection",
+ "split_query_sec",
+ ]
+ assert {"route", "length_adjust", "similarity_guidance_scale"} <= {
+ o["name"] for o in seed_vc.convert_options
+ }
+ assert "f0_condition" not in {o["name"] for o in seed_vc.options}
+ assert meanvc2.convert_options == ()
+
+
def test_samsone_is_refused_for_transcription(hub):
from core.inference import stt_audiocpp_sidecar as s
from core.inference.stt_sidecar import SttModelIdError
diff --git a/studio/backend/tests/test_audio_cpp_server_http.py b/studio/backend/tests/test_audio_cpp_server_http.py
index 9ed3562ed0d..d8040cd49d3 100644
--- a/studio/backend/tests/test_audio_cpp_server_http.py
+++ b/studio/backend/tests/test_audio_cpp_server_http.py
@@ -308,3 +308,33 @@ def test_stop_survives_a_server_slow_to_exit_after_sigkill(tmp_path):
server._config_dir = tmp_path / "cfg"
server.stop()
assert Stuck.waits == [10, 120]
+
+
+def test_a_server_slow_to_exit_after_kill_does_not_fail_the_restart(tmp_path, monkeypatch):
+ """Seed-VC exited ~54 s after SIGTERM following a run; a reload must not raise on it."""
+ import subprocess as sp
+
+ class SlowProcess:
+ pid = 424242
+ killed = False
+
+ def poll(self):
+ return None
+
+ def terminate(self):
+ pass
+
+ def kill(self):
+ self.killed = True
+
+ def wait(self, timeout = None):
+ raise sp.TimeoutExpired(["audiocpp_server"], timeout)
+
+ forgotten = []
+ monkeypatch.setattr(srv, "forget_pid", forgotten.append)
+ config_dir = tmp_path / "cfg"
+ config_dir.mkdir()
+ process = SlowProcess()
+ server = srv.AudioCppServer(process, 1, KOKORO, "kokoro", "cuda", config_dir)
+ server.stop()
+ assert process.killed and not config_dir.exists() and forgotten == []
diff --git a/studio/backend/tests/test_audio_gallery.py b/studio/backend/tests/test_audio_gallery.py
index 0a049ad5f16..3c2bbf7a280 100644
--- a/studio/backend/tests/test_audio_gallery.py
+++ b/studio/backend/tests/test_audio_gallery.py
@@ -924,3 +924,59 @@ def test_a_separate_scoped_clear_spares_speak_and_clone():
assert gallery.clear(workflow = "separate") == 1
assert gallery.audio_path(stem["id"]) is None
assert {r["id"] for r in gallery.list_audio()} == {speech["id"], clone["id"]}
+
+
+def test_a_convert_clip_keeps_its_workflow_meta_and_scoped_clear():
+ from models.inference import AudioGalleryItem
+ from routes.inference import clear_gallery_audio, _persist_tts_clip
+
+ speech = gallery.save(_wav(), _meta())
+ convert = _persist_tts_clip(
+ _wav(),
+ 16000,
+ "take.wav → Manthos",
+ "rvc",
+ "audiocpp_tts",
+ {
+ "role": "output",
+ "source_input_id": "i" * 32,
+ "source_name": "take.wav",
+ "reference_name": "Manthos",
+ "target_builtin": "manthos",
+ "settings": {"mode": "speech", "pitch": 3, "pitch_auto": False, "options": {}},
+ },
+ "convert",
+ )
+ listed = {r["id"]: r for r in gallery.list_audio()}
+ item = AudioGalleryItem(**listed[convert["id"]])
+ assert item.workflow == "convert" and item.prompt == "take.wav → Manthos"
+ assert (item.source_input_id, item.source_name) == ("i" * 32, "take.wav")
+ assert (item.reference_name, item.target_builtin) == ("Manthos", "manthos")
+ assert gallery.set_flags(convert["id"], pinned = True)["workflow"] == "convert"
+ assert asyncio.run(clear_gallery_audio(workflow = "convert", current_subject = "tester")) == {
+ "removed": 1
+ }
+ assert [r["id"] for r in gallery.list_audio()] == [speech["id"]]
+
+
+def test_a_kept_source_goes_with_its_clip(tmp_path):
+ source = tmp_path / "source.wav"
+ source.write_bytes(_wav())
+ kept = gallery.save(_wav(), _meta(), source)
+ plain = gallery.save(_wav(), _meta())
+ assert kept["source_saved"] is True and "source_saved" not in plain
+ copy = gallery.gallery_dir() / f"{kept['id']}.source.wav"
+ assert copy.read_bytes() == source.read_bytes()
+ assert gallery.owned_source_path(kept["id"]) == copy
+ assert gallery.owned_source_path(plain["id"]) is None
+ # Never a clip of its own, in the listing or a move.
+ assert sorted(r["id"] for r in gallery.list_audio()) == sorted([kept["id"], plain["id"]])
+ assert gallery.move(plain["id"], None)["id"] == plain["id"]
+ # Archiving keeps it with the clip; a clear that spares archived clips leaves it.
+ gallery.set_flags(kept["id"], archived = True)
+ assert gallery.clear() == 1 and copy.is_file()
+ assert gallery.clear(include_archived = True) == 1
+ assert not copy.exists()
+ again = gallery.save(_wav(), _meta(), source)
+ assert gallery.delete(again["id"])
+ assert not (gallery.gallery_dir() / f"{again['id']}.source.wav").exists()
diff --git a/studio/backend/tests/test_audio_inputs.py b/studio/backend/tests/test_audio_inputs.py
index 7112feeadf6..6e55f30c140 100644
--- a/studio/backend/tests/test_audio_inputs.py
+++ b/studio/backend/tests/test_audio_inputs.py
@@ -284,6 +284,25 @@ def test_transcribe_sends_a_16k_mono_copy_and_saves_nothing(client, monkeypatch)
assert [r["id"] for r in gallery.list_audio()] == [clip_q["clip_id"]]
+def test_a_convert_source_is_transcribed_past_the_clone_reference_cap(client, monkeypatch):
+ from routes import inference
+
+ seconds = []
+
+ async def fake_transcribe(raw, *args, **kwargs):
+ with wave.open(io.BytesIO(raw)) as w:
+ seconds.append(round(w.getnframes() / w.getframerate()))
+ return {"text": "words"}
+
+ monkeypatch.setattr(inference, "_transcribe_audio_result", fake_transcribe)
+ record = _upload(client, encode("wav", "pcm_s16le", 16000, "mono", 40.0), "long.wav").json()
+ url = f"{INPUTS}/{record['id']}/transcribe"
+ for purpose in ("reference", "convert"):
+ response = client.post(url, json = {"model": "m", "purpose": purpose})
+ assert response.status_code == 200, response.text
+ assert seconds == [30, 40]
+
+
def test_the_byte_cap_counts_only_kept_inputs(client, monkeypatch):
clock = [1_000_000.0]
monkeypatch.setattr(audio_inputs, "_now", lambda: clock[0])
diff --git a/studio/backend/tests/test_audio_run_route.py b/studio/backend/tests/test_audio_run_route.py
index a4507f10159..071358632db 100644
--- a/studio/backend/tests/test_audio_run_route.py
+++ b/studio/backend/tests/test_audio_run_route.py
@@ -1391,6 +1391,370 @@ def test_the_worker_keeps_run_fields_off_a_backend_without_them():
assert seen == ["hi"]
+def test_speak_in_a_saved_voice_on_a_speak_and_clone_model(stub, tmp_path):
+ backend = stub["use"](
+ "audio-cpp/audio.cpp-gguf/VoxCPM2-GGUF",
+ _clone_info(workflows = ("speak", "clone"), reference_text = "optional"),
+ )
+ input_id = _input(ALICE)
+ voice = run_as(
+ ALICE, lambda: audio_voices.create(audio_inputs.input_path(input_id), {"name": "Mine"})
+ )
+ with _client(ALICE) as client:
+ response = _run(client, workflow = "speak", inputs = {"reference": {"voice_id": voice["id"]}})
+ assert response.status_code == 200, response.text
+ (call,) = backend.calls
+ assert call["workflow"] == "speak"
+ reference = Path(call["audio_inputs"]["reference"])
+ assert reference.parent == tmp_path / "accounts" / ALICE.account_id / "audio" / "inputs"
+ clip = response.json()["clips"][0]
+ assert clip["workflow"] == "speak"
+ meta = json.loads(
+ (tmp_path / "accounts" / ALICE.account_id / "audio" / f"{clip['id']}.json").read_text(
+ encoding = "utf-8"
+ )
+ )
+ assert meta["voice_id"] == voice["id"] and meta["workflow"] == "speak"
+
+
+def _wav_params(path):
+ import wave
+ with wave.open(str(path)) as w:
+ return {"sample_rate": w.getframerate(), "channels": w.getnchannels()}
+
+
+def _convert_info(family, folder):
+ from core.inference import audio_cpp_backend as acb
+ from core.inference import audio_cpp_models as acm
+
+ policy = acm.FAMILIES[family]
+ variant = acm.AudioCppVariant("Q8_0", (acm.RepoFile("m.gguf", 1),), "m.gguf")
+ model = acm.AudioCppModel(
+ id = f"audio-cpp/audio.cpp-gguf/{folder}",
+ repo_id = acm.AUDIO_CPP_REPO,
+ folder = folder,
+ display_name = folder,
+ family = family,
+ task = policy.task,
+ server_task = policy.default_server_task,
+ variant = variant,
+ variants = (variant,),
+ default_variant = "Q8_0",
+ speaks = policy.speaks,
+ clone = policy.clone,
+ convert = policy.convert,
+ )
+ return f"audio-cpp/audio.cpp-gguf/{folder}", {
+ "is_audio": True,
+ "audio_type": "audiocpp_tts",
+ **acb.model_info_fields(model),
+ }
+
+
+def _use(stub, family, folder):
+ return stub["use"](*_convert_info(family, folder))
+
+
+def _convert(client, **body):
+ return client.post("/api/inference/audio/run", json = {"workflow": "convert", **body})
+
+
+def _meta(tmp_path, account, clip_id):
+ return json.loads(
+ (tmp_path / "accounts" / account.account_id / "audio" / f"{clip_id}.json").read_text(
+ encoding = "utf-8"
+ )
+ )
+
+
+def test_an_rvc_run_converts_an_upload_to_a_builtin_voice(stub, tmp_path):
+ backend = _use(stub, "rvc", "RVC-GGUF")
+ input_id = _input(ALICE, seconds = 2.0)
+ with _client(ALICE) as client:
+ response = _convert(
+ client,
+ inputs = {"source": {"input_id": input_id}},
+ convert = {"voice": "manthos", "pitch": 3},
+ options = {"retrieval_blend": 0.5},
+ seed = 7,
+ )
+ assert response.status_code == 200, response.text
+ (call,) = backend.calls
+ inputs_root = tmp_path / "accounts" / ALICE.account_id / "audio" / "inputs"
+ source = Path(call["audio_inputs"]["source"])
+ assert set(call["audio_inputs"]) == {"source"}
+ assert source.parent == inputs_root and source.name == f"{input_id}.16000.mono.m300.wav"
+ info = _wav_params(source)
+ assert (info["sample_rate"], info["channels"]) == (16000, 1)
+ assert call["workflow"] == "convert" and call["seed"] == 7
+ assert call["convert"] == {
+ "mode": "speech",
+ "pitch": 3,
+ "pitch_auto": False,
+ "style": "source",
+ "voice": "manthos",
+ "source_text": None,
+ }
+ assert call["audio_options"] == {"retrieval_blend": 0.5}
+ assert "reference_text" not in call and "speed" not in call
+ (clip,) = response.json()["clips"]
+ assert clip["workflow"] == "convert" and clip["role"] == "output"
+ meta = _meta(tmp_path, ALICE, clip["id"])
+ assert meta["workflow"] == "convert" and meta["prompt"] == "me.webm → Manthos"
+ assert (meta["source_input_id"], meta["source_name"]) == (input_id, "me.webm")
+ assert (meta["reference_name"], meta["target_builtin"]) == ("Manthos", "manthos")
+ assert meta["settings"] == {
+ "mode": "speech",
+ "pitch": 3,
+ "pitch_auto": False,
+ "style": "source",
+ "options": {"retrieval_blend": 0.5},
+ }
+ assert "voice_id" not in meta and "source_clip_id" not in meta
+ sidecar = json.dumps(meta)
+ assert str(tmp_path) not in sidecar and "inputs" not in sidecar
+ # The upload expires, so the clip keeps the 16 kHz copy the model heard beside it.
+ assert meta["source_saved"] is True
+ kept = tmp_path / "accounts" / ALICE.account_id / "audio" / f"{clip['id']}.source.wav"
+ assert kept.read_bytes() == source.read_bytes()
+ source_url = f"/api/inference/audio/gallery/{clip['id']}/source/file"
+ with _client(BOB) as client:
+ assert client.get(source_url).status_code == 404
+ with _client(ALICE) as client:
+ served = client.get(source_url)
+ assert served.status_code == 200 and served.content == kept.read_bytes()
+ listed = client.get("/api/inference/audio/gallery").json()["audio"]
+ assert [c["id"] for c in listed] == [clip["id"]] and listed[0]["source_saved"] is True
+ assert client.delete(f"/api/inference/audio/gallery/{clip['id']}").status_code == 200
+ assert client.get(source_url).status_code == 404
+ assert not kept.exists()
+
+
+def test_a_seed_vc_run_takes_a_history_clip_and_a_saved_voice_at_its_rates(stub, tmp_path):
+ backend = _use(stub, "seed_vc", "SeedVC-MLX-GGUF")
+ sources = _alice_sources()
+ with _client(ALICE) as client:
+ response = _convert(
+ client,
+ inputs = {
+ "source": {"clip_id": sources["clip_id"]},
+ "target": {"voice_id": sources["voice_id"]},
+ },
+ convert = {"mode": "singing", "pitch_auto": True},
+ )
+ assert response.status_code == 200, response.text
+ (call,) = backend.calls
+ source, target = Path(call["audio_inputs"]["source"]), Path(call["audio_inputs"]["target"])
+ assert source.name == f"c-{sources['clip_id']}.44100.mono.m300.wav"
+ assert target.name == f"v-{sources['voice_id']}.44100.mono.m30.wav"
+ assert _wav_params(target)["sample_rate"] == 44100
+ assert call["convert"]["mode"] == "singing" and call["convert"]["pitch_auto"] is True
+ meta = _meta(tmp_path, ALICE, response.json()["clips"][0]["id"])
+ assert meta["prompt"] == "alice said → Alice"
+ assert (meta["source_clip_id"], meta["voice_id"]) == (sources["clip_id"], sources["voice_id"])
+ assert meta["reference_name"] == "Alice" and "target_builtin" not in meta
+ # A history clip stays in history: no copy, and no source file to serve.
+ assert "source_saved" not in meta
+ clip_id = response.json()["clips"][0]["id"]
+ audio_dir = tmp_path / "accounts" / ALICE.account_id / "audio"
+ assert not (audio_dir / f"{clip_id}.source.wav").exists()
+ with _client(ALICE) as client:
+ assert client.get(f"/api/inference/audio/gallery/{clip_id}/source/file").status_code == 404
+
+
+ "body",
+ [
+ {"inputs": {"source": {"path": "/etc/passwd"}}},
+ {"inputs": {"source": {"input_id": "a" * 32}, "target": {"url": "http://x"}}},
+ {"options": {"voice_model_path": "/x.pth"}},
+ {"options": {"target_voice": "/etc/passwd"}},
+ {"text": "hello"},
+ {"convert": {"engine": "v2"}},
+ {"convert": {"pitch": 25}},
+ {"convert": {"mode": "rap"}},
+ {"convert": {"voice": "nope"}},
+ {"inputs": {"source": {"input_id": "a" * 32}, "reference": {"input_id": "a" * 32}}},
+ {"source_audio": "/etc/passwd"},
+ ],
+)
+def test_convert_client_paths_and_unknown_fields_are_422(stub, body):
+ _use(stub, "seed_vc", "SeedVC-MLX-GGUF")
+ with _client(ALICE) as client:
+ response = _convert(client, **body)
+ assert response.status_code == 422, response.text
+ assert stub["backend"].calls == []
+
+
+def test_convert_fields_on_clone_and_speak_are_422(stub):
+ with _client(ALICE) as client:
+ assert _run(client, convert = {"mode": "speech"}).status_code == 422
+ # A source on clone or speak reaches the route, which refuses it as Music-only.
+ assert _run(client, inputs = {"source": {"input_id": "a" * 32}}).status_code == 400
+ assert _run(client, workflow = "speak", text = None).status_code == 422
+ assert stub["backend"].calls == []
+
+
+ "family, folder, body, detail",
+ [
+ (None, None, {"target": True}, "Load a model that can convert a voice."),
+ ("seed_vc", "SeedVC-MLX-GGUF", {"source": False}, "Add the recording to convert."),
+ ("meanvc2", "MeanVC2-GGUF", {}, "Add the target voice."),
+ (
+ "rvc",
+ "RVC-GGUF",
+ {"target": True},
+ "RVC converts to its built-in voices; pick one under Built-in.",
+ ),
+ (
+ "meanvc2",
+ "MeanVC2-GGUF",
+ {"target": True, "convert": {"mode": "singing"}},
+ "MeanVC2 does not convert singing.",
+ ),
+ (
+ "vevo2",
+ "Vevo2-GGUF",
+ {"target": True, "convert": {"style": "target"}},
+ "Type what's said in the recording, or press Transcribe.",
+ ),
+ ],
+)
+def test_convert_problems_are_400s_in_words(stub, family, folder, body, detail):
+ if family is not None:
+ _use(stub, family, folder)
+ input_id = _input(ALICE)
+ inputs = {} if body.get("source") is False else {"source": {"input_id": input_id}}
+ if body.get("target"):
+ inputs["target"] = {"input_id": input_id}
+ with _client(ALICE) as client:
+ response = _convert(client, inputs = inputs, convert = body.get("convert") or {})
+ assert response.status_code == 400, response.text
+ assert response.json()["detail"] == detail
+ assert stub["backend"].calls == []
+
+
+def test_vevo2_takes_the_target_style_with_a_transcript_and_clone_still_works(stub, tmp_path):
+ backend = _use(stub, "vevo2", "Vevo2-GGUF")
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ converted = _convert(
+ client,
+ inputs = {
+ "source": {"input_id": input_id},
+ "target": {"input_id": input_id},
+ "source_text": " Okay, I'm Cemo. ",
+ },
+ convert = {"style": "target"},
+ )
+ cloned = _run(client, inputs = {"reference": {"input_id": input_id}})
+ assert converted.status_code == 200 and cloned.status_code == 200, cloned.text
+ convert_call, clone_call = backend.calls
+ assert convert_call["convert"]["source_text"] == "Okay, I'm Cemo."
+ assert {Path(p).name.split(".")[1] for p in convert_call["audio_inputs"].values()} == {"24000"}
+ assert clone_call["workflow"] == "clone" and "convert" not in clone_call
+
+
+def test_a_convert_only_model_refuses_plain_speech(stub):
+ _use(stub, "rvc", "RVC-GGUF")
+ with _client(ALICE) as client:
+ generate = client.post(
+ "/api/inference/audio/generate", json = {"messages": [{"role": "user", "content": "hi"}]}
+ )
+ speak = _run(client, workflow = "speak")
+ for response in (generate, speak):
+ assert response.status_code == 400
+ assert response.json()["detail"] == "RVC converts recordings. Open Convert and add one."
+ assert stub["backend"].calls == []
+
+
[email protected]("role", ["source", "target"])
[email protected]("kind", ["input_id", "clip_id", "voice_id"])
+def test_another_accounts_ids_are_404_on_convert(stub, role, kind):
+ _use(stub, "seed_vc", "SeedVC-MLX-GGUF")
+ sources = _alice_sources()
+ own = _input(BOB)
+ inputs = {"source": {"input_id": own}, "target": {"input_id": own}}
+ inputs[role] = {kind: sources[kind]}
+ with _client(BOB) as client:
+ response = _convert(client, inputs = inputs)
+ assert response.status_code == 404, response.text
+ assert stub["backend"].calls == []
+
+
+def test_the_worker_and_orchestrator_carry_convert_and_the_running_task(monkeypatch):
+ from core.inference.orchestrator import InferenceOrchestrator
+ from core.inference.worker import _handle_generate_audio
+
+ seen = []
+
+ class _Backend:
+ def generate_audio_response(self, **kwargs):
+ seen.append(kwargs)
+ return _wav(), 24000
+
+ def runtime_fields(self):
+ return {"audio_server_task": "vc", "audio_convert_route": None}
+
+ responses: queue.Queue = queue.Queue()
+ convert = {"mode": "speech", "pitch": None, "pitch_auto": False, "style": "source"}
+ _handle_generate_audio(
+ _Backend(),
+ {
+ "request_id": "r1",
+ "text": "x",
+ "workflow": "convert",
+ "audio_inputs": {"source": "/abs/s.wav", "target": "/abs/t.wav"},
+ "convert": convert,
+ },
+ responses,
+ threading.Event(),
+ )
+ assert seen[0]["convert"] == convert and seen[0]["workflow"] == "convert"
+ done = responses.get_nowait()
+ assert done["audio_runtime"] == {"audio_server_task": "vc", "audio_convert_route": None}
+
+ orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ sent = []
+
+ def read_one(*, timeout):
+ return {**done, "request_id": sent[0]["request_id"]}
+
+ class _Null:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_exc):
+ return False
+
+ for name, value in {
+ "_ensure_subprocess_alive": lambda: True,
+ "_send_cmd": lambda cmd: sent.append(cmd),
+ "_direct_reader": lambda _rid, _cancel = None: (read_one, lambda **_k: None, lambda: None),
+ "_reserve_worker": lambda _why: _Null(),
+ "_wait_worker_idle": lambda **_k: True,
+ "_claim_worker": lambda _cancel: None,
+ "_release_worker": lambda *_a, **_k: None,
+ }.items():
+ monkeypatch.setattr(orchestrator, name, value, raising = False)
+ orchestrator.active_model_name = "m"
+ orchestrator.models = {"m": {"audio_server_task": "clon"}}
+ orchestrator._gen_lock = threading.Lock()
+ orchestrator._send_order_lock = threading.Lock()
+ orchestrator._unload_pending = False
+ orchestrator.generate_audio_response(
+ "x",
+ workflow = "convert",
+ audio_inputs = {"source": "/abs/s.wav"},
+ convert = convert,
+ )
+ (cmd,) = sent
+ assert cmd["convert"] == convert and cmd["audio_inputs"] == {"source": "/abs/s.wav"}
+ # The status mirror follows the server the run left running.
+ assert orchestrator.models["m"]["audio_server_task"] == "vc"
+
+
def test_a_float_wav_separation_source_is_probed_through_ffmpeg(tmp_path):
import struct
diff --git a/studio/backend/tests/test_audio_workflows.py b/studio/backend/tests/test_audio_workflows.py
index 3c549b6b35d..55acc35c7ec 100644
--- a/studio/backend/tests/test_audio_workflows.py
+++ b/studio/backend/tests/test_audio_workflows.py
@@ -105,7 +105,15 @@ def test_workflow_for_audio_type_is_speak_or_music():
assert aw.workflow_for_audio_type("snac") == "speak"
assert aw.workflow_for_audio_type("unknown") == "speak"
assert aw.workflow_for_audio_type(None) == "speak"
- assert aw.AUDIO_WORKFLOW_IDS == ("speak", "clone", "edit", "music", "separate", "transcribe")
+ assert aw.AUDIO_WORKFLOW_IDS == (
+ "speak",
+ "clone",
+ "edit",
+ "convert",
+ "music",
+ "separate",
+ "transcribe",
+ )
def test_workflow_ids_match_the_frontend_order():
@@ -122,17 +130,51 @@ def test_status_carries_the_clone_fields():
assert status.model_dump().items() >= clone.items()
+def test_status_and_load_carry_the_convert_fields():
+ caps = {
+ "modes": ["speech", "singing"],
+ "target": "audio",
+ "builtin_voices": [],
+ "pitch": {"singing": {"auto": True}},
+ "style": False,
+ "route_reloads": True,
+ "source_max_seconds": 300,
+ }
+ fields = dict(
+ is_audio = True,
+ audio_type = AUDIO_CPP_TTS_AUDIO_TYPE,
+ audio_workflows = ["convert"],
+ audio_options_by_workflow = {"convert": [{"name": "length_adjust", "type": "float"}]},
+ audio_workflow_tasks = {"convert": "vc", "convert:singing": "svc"},
+ audio_server_task = "svc",
+ audio_convert = caps,
+ audio_convert_route = "v1_svc",
+ )
+ dumped = InferenceStatusResponse(**fields).model_dump()
+ assert dumped["audio_convert"] == caps
+ assert dumped["audio_workflow_tasks"]["convert:singing"] == "svc"
+ assert (dumped["audio_server_task"], dumped["audio_convert_route"]) == ("svc", "v1_svc")
+ assert dumped["audio_options_by_workflow"]["convert"][0]["name"] == "length_adjust"
+ load = LoadResponse(status = "loaded", model = "m", display_name = "m", inference = {}, **fields)
+ assert load.audio_convert == caps and load.audio_server_task == "svc"
+ plain = InferenceStatusResponse(is_audio = True, audio_type = AUDIO_CPP_TTS_AUDIO_TYPE)
+ assert plain.audio_convert is None and plain.audio_workflow_tasks is None
+
+
@pytest.mark.parametrize(
"folder, family, workflows",
[
- ("Chatterbox-GGUF", "chatterbox", ["clone"]),
+ ("Chatterbox-GGUF", "chatterbox", ["clone", "convert"]),
("Chatterbox-Turbo-GGUF", "chatterbox_turbo", ["speak"]),
("VoxCPM2-GGUF", "voxcpm2", ["speak", "clone"]),
("Qwen3-TTS-12Hz-0.6B-Base-GGUF", "qwen3_tts", ["clone"]),
("Kokoro-82M-GGUF", "kokoro_tts", ["speak"]),
+ ("RVC-GGUF", "rvc", ["convert"]),
+ ("SeedVC-MLX-GGUF", "seed_vc", ["convert"]),
+ ("MeanVC2-GGUF", "meanvc2", ["convert"]),
("DotTTS-Edit-GGUF", "dots_tts", ["speak", "edit"]),
("DotTTS-MF-GGUF", "dots_tts", ["speak"]),
- ("Vevo2-GGUF", "vevo2", ["clone", "edit"]),
+ ("Vevo2-GGUF", "vevo2", ["clone", "edit", "convert"]),
("FireRedAudio-GGUF", "firered_audio", ["clone", "edit"]),
],
)
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 2b2d03db36f..76c13bf7b64 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1080,6 +1080,12 @@ class TestLoadHubDownloadExclusion:
"audio_workflows",
"audio_reference_text",
"audio_required_inputs",
+ # Constant None: nor converts one.
+ "audio_options_by_workflow",
+ "audio_workflow_tasks",
+ "audio_server_task",
+ "audio_convert",
+ "audio_convert_route",
"audio_music",
}
unresolved = sorted(
diff --git a/studio/backend/tests/test_library_routes.py b/studio/backend/tests/test_library_routes.py
index 63b4dc525c5..e43e0b80919 100644
--- a/studio/backend/tests/test_library_routes.py
+++ b/studio/backend/tests/test_library_routes.py
@@ -461,6 +461,26 @@ def test_a_clip_counts_its_recipe_toward_what_it_takes_on_disk(client, monkeypat
assert "storageBytes" not in items[f"image:{image}"]
+def test_a_conversion_counts_the_recording_it_kept_toward_what_it_takes_on_disk(
+ client, monkeypatch, tmp_path
+):
+ from core.inference import audio_gallery
+
+ monkeypatch.setattr(library, "_SOURCES", (library._audio_items,))
+ source = tmp_path / "upload.wav"
+ source.write_bytes(b"S" * 1000)
+ meta = {"model": "sample-vc", "audio_type": "speech", "sample_rate": 24000, "duration_s": 1.0}
+ audio = audio_gallery.save(
+ b"RIFF0000WAVE", {**meta, "prompt": "Converted", "created_at": 1_700_000_000}, source
+ )["id"]
+ wav = audio_gallery.audio_path(audio)
+ item = _items(client)[0][f"audio:{audio}"]
+ assert item["sizeBytes"] == wav.stat().st_size
+ assert item["storageBytes"] == (
+ wav.stat().st_size + wav.with_suffix(".json").stat().st_size + 1000
+ )
+
+
def test_listed_images_are_sized_without_resolving_each_but_links_are_still_checked(
client, monkeypatch, tmp_path
):
diff --git a/studio/backend/tests/test_pr7699_worker_audio_mirror.py b/studio/backend/tests/test_pr7699_worker_audio_mirror.py
index 56e294eb739..15d708f991b 100644
--- a/studio/backend/tests/test_pr7699_worker_audio_mirror.py
+++ b/studio/backend/tests/test_pr7699_worker_audio_mirror.py
@@ -127,6 +127,12 @@ def test_audio_cpp_workflow_fields_cross_the_worker_and_the_parent(monkeypatch):
"audio_reference_text": "required",
"audio_required_inputs": ["instruct"],
"audio_clone": {"reference_text": "required", "emotion_audio": False},
+ "audio_options_by_workflow": {"convert": [{"name": "length_adjust", "type": "float"}]},
+ "audio_workflow_tasks": {"clone": "clon", "convert": "vc", "convert:singing": "svc"},
+ "audio_server_task": "clon",
+ "audio_convert": {"modes": ["speech"], "target": "audio", "source_max_seconds": 300},
+ "audio_convert_route": "v2_vc",
+ "audio_convert_rules": {"source_rate": 16000, "target_rate": 24000},
"audio_edit": {"style": "instructions", "delivery": True, "max_changes": 5},
"audio_music": {"modes": [{"id": "song", "variations": None}]},
# The /audio/run route sizes CPU waits from it.
diff --git a/studio/frontend/src/features/audio/api.ts b/studio/frontend/src/features/audio/api.ts
index 84571a7bf93..22b05a615fb 100644
--- a/studio/frontend/src/features/audio/api.ts
+++ b/studio/frontend/src/features/audio/api.ts
@@ -108,12 +108,20 @@ export interface AudioGalleryClip {
/** The Audio workflow that made the clip. Older servers omit it; read it through clipWorkflow. */
workflow?: string | null;
reference_name?: string | null;
+ /** The history clip an edit changed or a conversion started from. */
+ source_clip_id?: string | null;
+ source_input_id?: string | null;
+ voice_id?: string | null;
+ /** Served at /audio/gallery/{id}/source/file. */
+ source_saved?: boolean;
+ source_name?: string | null;
+ target_builtin?: string | null;
+ target_clip_id?: string | null;
+ target_input_id?: string | null;
/** Clips one run made together share it (a separation's stems, music takes); null for one clip. */
group_id?: string | null;
/** A stem's name (vocals, drums, ...) or an edit's run part ("output" or "source"). */
role?: string | null;
- /** The original an edit changed. */
- source_clip_id?: string | null;
/** The run's settings, e.g. a separation's stem list. */
settings?: Record<string, unknown> | null;
}
@@ -220,7 +228,7 @@ export async function deleteAudioClip(id: string): Promise<void> {
}
export async function clearAudioGallery(
- workflow?: "speak" | "clone" | "edit" | "music" | "separate",
+ workflow?: "speak" | "clone" | "edit" | "convert" | "music" | "separate",
): Promise<number> {
const query = workflow ? `?workflow=${workflow}` : "";
const response = await authFetch(`/api/inference/audio/gallery${query}`, {
@@ -407,7 +415,13 @@ interface TranscribeInputResponse {
export async function transcribeAudioInput(
ref: AudioSourceRef,
- body: { model: string; engine?: string; device?: string; language?: string },
+ body: {
+ model: string;
+ engine?: string;
+ device?: string;
+ language?: string;
+ purpose?: "reference" | "convert";
+ },
signal?: AbortSignal,
): Promise<TranscribeInputResponse> {
const response = await authFetch(transcribeUrl(ref), {
diff --git a/studio/frontend/src/features/audio/audio-cpp-catalog.ts b/studio/frontend/src/features/audio/audio-cpp-catalog.ts
index bc6952c7634..16bcce7063e 100644
--- a/studio/frontend/src/features/audio/audio-cpp-catalog.ts
+++ b/studio/frontend/src/features/audio/audio-cpp-catalog.ts
@@ -23,6 +23,7 @@ export type AudioCppWorkflow =
| "speak"
| "clone"
| "edit"
+ | "convert"
| "music"
| "separate"
| "transcribe";
@@ -63,14 +64,17 @@ export const AUDIO_CPP_MODELS: readonly AudioCppModel[] = [
{ id: folder("Chatterbox-Turbo-GGUF"), task: "tts" },
{ id: folder("VoxCPM2-GGUF"), task: "tts", workflows: ["speak", "clone"] },
{ id: folder("Qwen3-TTS-12Hz-0.6B-Base-GGUF"), task: "tts", workflows: ["clone"] },
- { id: folder("Chatterbox-GGUF"), task: "tts", workflows: ["clone"] },
+ { id: folder("Chatterbox-GGUF"), task: "tts", workflows: ["clone", "convert"] },
{ id: folder("IndexTTS2-GGUF"), task: "tts", workflows: ["clone"] },
{ id: folder("CosyVoice3-GGUF"), task: "tts", workflows: ["clone"] },
{ id: folder("Qwen3-TTS-12Hz-1.7B-CustomVoice-GGUF"), task: "tts" },
{ id: folder("Qwen3-TTS-12Hz-1.7B-VoiceDesign-GGUF"), task: "tts" },
{ id: folder("DotTTS-Edit-GGUF"), task: "tts", workflows: ["speak", "edit"] },
- { id: folder("Vevo2-GGUF"), task: "tts", workflows: ["clone", "edit"] },
+ { id: folder("Vevo2-GGUF"), task: "tts", workflows: ["clone", "edit", "convert"] },
{ id: folder("FireRedAudio-GGUF"), task: "tts", workflows: ["clone", "edit"] },
+ { id: folder("SeedVC-MLX-GGUF"), task: "tts", workflows: ["convert"] },
+ { id: folder("RVC-GGUF"), task: "tts", workflows: ["convert"] },
+ { id: folder("MeanVC2-GGUF"), task: "tts", workflows: ["convert"] },
{ id: "audio-cpp/MiniMax-Music3-GGUF", task: "music" },
{ id: "audio-cpp/Yue2-3B-GGUF", task: "music" },
{ id: folder("ACE-Step1.5-GGUF"), task: "music" },
diff --git a/studio/frontend/src/features/audio/audio-page-policy.ts b/studio/frontend/src/features/audio/audio-page-policy.ts
index d11dd1e8015..126b845b6e2 100644
--- a/studio/frontend/src/features/audio/audio-page-policy.ts
+++ b/studio/frontend/src/features/audio/audio-page-policy.ts
@@ -69,6 +69,7 @@ export type AudioBusy =
export type AudioGenerationPhase =
| "preparing"
+ | "switching"
| "generating"
| "stopping"
| "finishing"
@@ -84,6 +85,7 @@ export type AudioGenerationPresentation = {
* browser-visible numeric progress, so these labels never imply a fraction or ETA. */
export function audioGenerationPresentation(
phase: AudioGenerationPhase,
+ detail?: string | null,
): AudioGenerationPresentation | null {
switch (phase) {
case "preparing":
@@ -92,6 +94,12 @@ export function audioGenerationPresentation(
actionLabel: "Preparing…",
canStop: false,
};
+ case "switching":
+ return {
+ status: detail || "Switching model…",
+ actionLabel: "Stop",
+ canStop: true,
+ };
case "generating":
return {
status: "Generating audio…",
diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsx
index fba5c945b52..d7d3e30b371 100644
--- a/studio/frontend/src/features/audio/audio-page.tsx
+++ b/studio/frontend/src/features/audio/audio-page.tsx
@@ -31,7 +31,11 @@ import { cn } from "@/lib/utils";
import { useIsMobileShell } from "@/hooks/use-mobile";
import { useShallow } from "zustand/react/shallow";
-import type { AudioGalleryClip } from "./api";
+import {
+ type AudioGalleryClip,
+ fetchAudioBlob,
+ uploadAudioInput,
+} from "./api";
import {
type AudioBusy,
type AudioGenerationPhase,
@@ -55,6 +59,7 @@ import { galleryCache, useAudioGallery, useWorkflowHistory } from "./hooks/use-a
import { useAudioHandoff } from "./hooks/use-audio-handoff";
import { useAudioModelSlot } from "./hooks/use-audio-model-slot";
import { useCloneGeneration } from "./hooks/use-clone-generation";
+import { useConvertGeneration } from "./hooks/use-convert-generation";
import { useEditGeneration } from "./hooks/use-edit-generation";
import { useSeparateGeneration } from "./hooks/use-separate-generation";
import { useSpeechGeneration } from "./hooks/use-speech-generation";
@@ -67,6 +72,12 @@ import {
adoptReference,
clonePageModels,
} from "./pages/clone-page";
+import {
+ ConvertFooter,
+ ConvertOutput,
+ ConvertRail,
+ convertPageModels,
+} from "./pages/convert-page";
import {
EditFooter,
EditOutput,
@@ -98,6 +109,7 @@ import {
} from "./transcribe-languages";
import { type AudioPickerRow, audioRowMatchesWorkflow } from "./picker-filter";
import { useAudioCloneStore } from "./stores/audio-clone-store";
+import { useAudioConvertStore } from "./stores/audio-convert-store";
import { useAudioEditStore } from "./stores/audio-edit-store";
import { useAudioWorkspaceStore } from "./stores/audio-workspace-store";
import { AudioToolPanels } from "./tools/tool-panel-host";
@@ -124,6 +136,54 @@ const HUB_TASKS_BY_MODE = {
const RECOMMENDED_MUSIC_MODELS = ["ACE-Step1.5-GGUF", "Stable-Audio-3-Small-Music-GGUF"];
+function reuseConvertInputs(clip: AudioGalleryClip) {
+ const store = useAudioConvertStore.getState();
+ const sourceId = clip.source_clip_id ?? clip.source_input_id ?? null;
+ const name = clip.source_name ?? "Recording";
+ if (sourceId) {
+ store.setSource({
+ kind: clip.source_clip_id ? "clip" : "input",
+ id: sourceId,
+ name,
+ durationS: null,
+ });
+ }
+ // An upload expires within a day; the clip kept what it converted, so upload that copy again.
+ if (!clip.source_clip_id && clip.source_saved) {
+ void fetchAudioBlob(
+ `/api/inference/audio/gallery/${encodeURIComponent(clip.id)}/source/file`,
+ )
+ .then((blob) => uploadAudioInput(blob, name))
+ .then((record) => {
+ if (useAudioConvertStore.getState().source?.id !== sourceId) return;
+ store.setSource({
+ kind: "input",
+ id: record.id,
+ name,
+ durationS: record.duration_s,
+ expiresAt: record.expires_at,
+ });
+ })
+ .catch(() => undefined);
+ }
+ const target = clip.voice_id
+ ? { kind: "voice" as const, id: clip.voice_id }
+ : clip.target_clip_id
+ ? { kind: "clip" as const, id: clip.target_clip_id }
+ : clip.target_input_id
+ ? { kind: "input" as const, id: clip.target_input_id }
+ : null;
+ if (target) {
+ store.setTarget({
+ ...target,
+ name: clip.reference_name ?? "Target voice",
+ durationS: null,
+ });
+ } else if (clip.target_builtin) {
+ store.setBuiltinVoice(clip.target_builtin);
+ }
+}
+
export function AudioPage({
active = true,
onInitialReady,
@@ -169,7 +229,6 @@ export function AudioPage({
},
[],
);
- const generationPresentation = audioGenerationPresentation(generationPhase);
const [status, setStatus] = useState<InferenceStatusResponse | null>(null);
const generateAbort = useRef<AbortController | null>(null);
@@ -299,6 +358,13 @@ export function AudioPage({
handleCopyPrompt,
} = useAudioGallery({ active });
+ // A Clone run, even a stopped one, can restart audio.cpp under another task: re-read it on arriving.
+ useEffect(() => {
+ if (active && ttsWorkflow === "convert" && initialReadySent.current) {
+ void refreshStatus();
+ }
+ }, [active, ttsWorkflow, refreshStatus]);
+
// Resync on activation: another tab may have loaded/unloaded models meanwhile.
useEffect(() => {
if (!active) return;
@@ -518,6 +584,27 @@ export function AudioPage({
audioOptionValues,
sttRepo: cloneSttRepo,
});
+ const convert = useConvertGeneration({
+ status,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ generateAbort,
+ setMode,
+ setAdvancedOpen,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ refreshGallery,
+ selectClip: selectGeneratedClip,
+ setFallbackClip,
+ setSelectedId,
+ pendingTranscribeRelease,
+ replayQueuedTtsPick,
+ audioOptionSpecs,
+ audioOptionValues,
+ sttRepo: cloneSttRepo,
+ });
const edit = useEditGeneration({
status,
busyRef,
@@ -557,6 +644,10 @@ export function AudioPage({
pendingTranscribeRelease,
replayQueuedTtsPick,
});
+ const generationPresentation = audioGenerationPresentation(
+ generationPhase,
+ convert.runningNotice ?? undefined,
+ );
const { navigateSelf } = useAudioHandoff({
active,
@@ -622,11 +713,25 @@ export function AudioPage({
() => handleClearGallery(ttsWorkflow),
[handleClearGallery, ttsWorkflow],
);
+ const handleSendToConvert = useCallback(
+ (clip: AudioGalleryClip) => {
+ if (!transitionWorkflow("convert")) return;
+ useAudioConvertStore.getState().setSource({
+ kind: "clip",
+ id: clip.id,
+ name: clip.prompt,
+ durationS: clip.duration_s ?? null,
+ });
+ },
+ [transitionWorkflow],
+ );
const sendHandlersFor = useCallback(
(clip: AudioGalleryClip): ClipSendHandlers => ({
clone: () => {
- if (transitionWorkflow("clone")) adoptReference(clipReference(clip));
+ if (transitionWorkflow("clone"))
+ adoptReference(clipReference({ ...clip, workflow: clipWorkflow(clip) }));
},
+ convert: () => handleSendToConvert(clip),
transcribe: () => {
// Switching mid-run would stop the run in progress; wait for it instead.
if (busyRef.current !== null) {
@@ -647,13 +752,14 @@ export function AudioPage({
});
},
}),
- [transitionWorkflow, busyRef],
+ [transitionWorkflow, busyRef, handleSendToConvert],
);
const handleUseTextAgain = useCallback(
(clip: AudioGalleryClip) => {
const target = clipWorkflow(clip);
if (!transitionWorkflow(target)) return;
if (target === "clone") useAudioCloneStore.getState().setText(clip.prompt);
+ else if (target === "convert") reuseConvertInputs(clip);
else if (target === "edit") useAudioEditStore.getState().setEdited(clip.prompt);
else setPrompt(clip.prompt);
},
@@ -730,6 +836,15 @@ export function AudioPage({
label: `use ${model.name}`,
onClick: () => void pickRecommendedModel(model.id),
}));
+ const recommendedConvertActions = convertPageModels(
+ MODELS_BY_MODE.speak,
+ isMac,
+ )
+ .slice(0, 2)
+ .map((model) => ({
+ label: `use ${model.name}`,
+ onClick: () => void pickRecommendedModel(model.id),
+ }));
const recommendedEditActions = editPageModels(MODELS_BY_MODE.speak, isMac)
.slice(0, 2)
.map((model) => ({
@@ -745,6 +860,10 @@ export function AudioPage({
label: `use ${model.name}`,
onClick: () => void pickRecommendedModel(model.id),
}));
+ // RVC, Seed-VC and MeanVC2 only convert; Speak must not send their users to Clone.
+ const loadedConvertsOnly =
+ !!status?.audio_workflows?.includes("convert") &&
+ !status.audio_workflows.includes("clone");
const loadedSeparates =
status?.audio_workflows?.includes("separate") === true;
const generateBlocker: GenerateBlocker | null =
@@ -761,6 +880,8 @@ export function AudioPage({
? "Load a model that can clone a voice."
: ttsWorkflow === "edit"
? "Load a model that can edit speech."
+ : ttsWorkflow === "convert"
+ ? "Load a model that can convert a voice."
: ttsWorkflow === "separate"
? "Load a model that can separate audio."
: "Load a speech model to generate.",
@@ -769,6 +890,8 @@ export function AudioPage({
? [chooseModelAction, ...recommendedCloneActions]
: ttsWorkflow === "edit"
? [chooseModelAction, ...recommendedEditActions]
+ : ttsWorkflow === "convert"
+ ? [chooseModelAction, ...recommendedConvertActions]
: ttsWorkflow === "separate"
? [chooseModelAction, ...recommendedSeparateActions]
: ttsWorkflow === "music"
@@ -818,6 +941,17 @@ export function AudioPage({
},
],
}
+ : !pageModelLoaded && ttsWorkflow === "convert"
+ ? {
+ reason: "The loaded model cannot convert a voice.",
+ actions: [
+ {
+ label: "Choose a model that can convert",
+ onClick: openSelector,
+ },
+ ...recommendedConvertActions,
+ ],
+ }
: !pageModelLoaded
? musicGeneration
? {
@@ -830,6 +964,17 @@ export function AudioPage({
},
],
}
+ : ttsWorkflow === "speak" && loadedConvertsOnly
+ ? {
+ reason: "The loaded model converts recordings.",
+ actions: [
+ { label: "Choose a speech model", onClick: openSelector },
+ {
+ label: "open Convert",
+ onClick: () => transitionWorkflow("convert"),
+ },
+ ],
+ }
: ttsWorkflow === "speak"
? {
reason: "The loaded model needs a voice to clone.",
@@ -858,6 +1003,8 @@ export function AudioPage({
? clone.blocker
: ttsWorkflow === "edit"
? edit.blocker
+ : ttsWorkflow === "convert"
+ ? convert.blocker
: musicStudio
? music.blocker
: !prompt.trim() && !lyricsOptional
@@ -882,6 +1029,8 @@ export function AudioPage({
? clone.generationError
: ttsWorkflow === "edit"
? edit.generationError
+ : ttsWorkflow === "convert"
+ ? convert.generationError
: musicStudio
? music.generationError
: generationError;
@@ -895,12 +1044,14 @@ export function AudioPage({
: null;
const setCloneGenerationError = clone.setGenerationError;
const setEditGenerationError = edit.setGenerationError;
+ const setConvertGenerationError = convert.setGenerationError;
const setSeparateGenerationError = separate.setGenerationError;
const setMusicGenerationError = music.setGenerationError;
useEffect(() => {
setGenerationError(null);
setCloneGenerationError(null);
setEditGenerationError(null);
+ setConvertGenerationError(null);
setSeparateGenerationError(null);
setMusicGenerationError(null);
}, [
@@ -908,6 +1059,7 @@ export function AudioPage({
setGenerationError,
setCloneGenerationError,
setEditGenerationError,
+ setConvertGenerationError,
setSeparateGenerationError,
setMusicGenerationError,
]);
@@ -915,6 +1067,7 @@ export function AudioPage({
if (busy === "generating") setGenerationError(null);
if (busy === "generating") setCloneGenerationError(null);
if (busy === "generating") setEditGenerationError(null);
+ if (busy === "generating") setConvertGenerationError(null);
if (busy === "generating") setSeparateGenerationError(null);
if (busy === "generating") setMusicGenerationError(null);
}, [
@@ -922,6 +1075,7 @@ export function AudioPage({
setGenerationError,
setCloneGenerationError,
setEditGenerationError,
+ setConvertGenerationError,
setSeparateGenerationError,
setMusicGenerationError,
]);
@@ -1097,7 +1251,8 @@ export function AudioPage({
const handlePageGenerate =
ttsWorkflow === "separate" ? separate.handleGenerate :
ttsWorkflow === "clone" ? clone.handleGenerate :
- ttsWorkflow === "edit" ? edit.handleGenerate : handleGenerate;
+ ttsWorkflow === "edit" ? edit.handleGenerate :
+ ttsWorkflow === "convert" ? convert.handleGenerate : handleGenerate;
generateShortcut.current = () => {
if (canTranscribe) handleTranscribe();
else if (canGenerate) void handlePageGenerate();
@@ -1127,6 +1282,8 @@ export function AudioPage({
? clonePageModels(MODELS_BY_MODE.speak, isMac)
: ttsWorkflow === "edit"
? editPageModels(MODELS_BY_MODE.speak, isMac)
+ : ttsWorkflow === "convert"
+ ? convertPageModels(MODELS_BY_MODE.speak, isMac)
: ttsWorkflow === "separate"
? separatePageModels(MODELS_BY_MODE.speak, isMac)
: speakPageModels(MODELS_BY_MODE.speak, isMac)
@@ -1182,7 +1339,7 @@ export function AudioPage({
? "music"
: ttsWorkflow === "separate"
? "separate"
- : ttsWorkflow === "clone" || ttsWorkflow === "edit"
+ : ttsWorkflow === "clone" || ttsWorkflow === "edit" || ttsWorkflow === "convert"
? ttsWorkflow
: "tts",
status?.audio_type,
@@ -1195,8 +1352,12 @@ export function AudioPage({
? "The loaded model cannot clone a voice."
: ttsWorkflow === "edit"
? "The loaded model cannot edit speech."
+ : ttsWorkflow === "convert"
+ ? "The loaded model cannot convert a voice."
: musicGeneration
? "The loaded model makes music. Pick a speech model."
+ : ttsWorkflow === "speak" && loadedConvertsOnly
+ ? "The loaded model converts recordings. Pick a speech model."
: ttsWorkflow === "speak"
? "The loaded model needs a voice to clone. Pick a speech model."
: "The loaded model is not a music model."
@@ -1210,7 +1371,9 @@ export function AudioPage({
? "No voice cloning model loaded."
: ttsWorkflow === "edit"
? "No speech editing model loaded."
- : "No TTS model loaded."
+ : ttsWorkflow === "convert"
+ ? "No voice conversion model loaded."
+ : "No TTS model loaded."
: sttSelected
? audioCapabilityLine("stt", sttReady ? "ready" : "loading")
: transcribeRepo
@@ -1454,6 +1617,14 @@ export function AudioPage({
<AudioActiveProvider value={active}>
<EditRail {...railProps} edit={edit} historyClips={clips} />
</AudioActiveProvider>
+ ) : ttsWorkflow === "convert" ? (
+ <AudioActiveProvider value={active}>
+ <ConvertRail
+ {...railProps}
+ convert={convert}
+ historyClips={clips}
+ />
+ </AudioActiveProvider>
) : (
<SpeakRail {...railProps} />
);
@@ -1519,6 +1690,18 @@ export function AudioPage({
elapsedSeconds={elapsedSeconds}
clone={clone}
/>
+ ) : ttsWorkflow === "convert" ? (
+ <ConvertFooter
+ busy={busy}
+ generationPresentation={generationPresentation}
+ handleStopGeneration={handleStopGeneration}
+ ttsLoaded={pageModelLoaded}
+ blocker={generateBlocker}
+ shortcutLabel={shortcutLabel}
+ error={generateFailure}
+ elapsedSeconds={elapsedSeconds}
+ convert={convert}
+ />
) : (
<TtsFooter
busy={busy}
@@ -1670,6 +1853,14 @@ export function AudioPage({
recommendedModels={selectorModels}
onPickModel={pickRecommendedModel}
/>
+ ) : ttsWorkflow === "convert" ? (
+ <ConvertOutput
+ {...outputProps}
+ useAgainLabel="Use these inputs again"
+ modelReady={pageModelLoaded}
+ recommendedModels={selectorModels}
+ onPickModel={pickRecommendedModel}
+ />
) : (
<SpeakOutput {...outputProps} modelReady={pageModelLoaded} />
);
diff --git a/studio/frontend/src/features/audio/audio-run-request.ts b/studio/frontend/src/features/audio/audio-run-request.ts
index b01801cf95f..c73cc7b1b9c 100644
--- a/studio/frontend/src/features/audio/audio-run-request.ts
+++ b/studio/frontend/src/features/audio/audio-run-request.ts
@@ -36,9 +36,12 @@ export function clipReference(clip: {
workflow?: string | null;
reference_name?: string | null;
}): AudioSourceSelection {
- // Only speech is a transcript: not a Music description, nor the file name an untranscribed edit is titled by.
+ // Only speech is a transcript: not a Music description, a Convert label, nor the file name an
+ // untranscribed edit is titled by.
const spoken =
- clip.workflow !== "music" && clip.prompt !== clip.reference_name;
+ clip.workflow !== "music" &&
+ clip.workflow !== "convert" &&
+ clip.prompt !== clip.reference_name;
return {
kind: "clip",
id: clip.id,
@@ -118,7 +121,33 @@ export interface AudioMusicRunFields {
source?: AudioSourceRef | null;
}
-export interface AudioRunRequest {
+export type ConvertMode = "speech" | "singing";
+
+export type ConvertStyle = "source" | "target";
+
+export interface AudioConvertParams {
+ mode: ConvertMode;
+ pitch: number | null;
+ pitch_auto: boolean;
+ style?: ConvertStyle;
+ voice?: string | null;
+}
+
+export interface AudioConvertRunRequest {
+ workflow: "convert";
+ inputs: {
+ source: AudioSourceRef;
+ target?: AudioSourceRef | null;
+ source_text?: string | null;
+ };
+ convert: AudioConvertParams;
+ options?: Record<string, AudioOptionScalar>;
+ seed?: number | null;
+}
+
+export type AudioRunRequest = AudioTextRunRequest | AudioConvertRunRequest;
+
+export interface AudioTextRunRequest {
workflow: "clone" | "speak" | "edit" | "music" | "separate";
music?: AudioMusicRunFields;
/** Required for clone, speak and music; separate takes none. */
@@ -150,10 +179,64 @@ function cleanRef(
return Object.keys(out).length === 1 ? out : null;
}
+function cleanOptions(
+ options: Record<string, AudioOptionScalar> | undefined,
+): Record<string, AudioOptionScalar> {
+ return Object.fromEntries(
+ Object.entries(options ?? {}).filter(
+ ([name, value]) =>
+ name &&
+ (typeof value === "boolean" ||
+ typeof value === "string" ||
+ (typeof value === "number" && Number.isFinite(value))),
+ ),
+ );
+}
+
+const CONVERT_MODES: ReadonlySet<string> = new Set(["speech", "singing"]);
+
+/** Ids only, never a path: the route resolves them in the caller's account. */
+function buildConvertRunBody(
+ request: AudioConvertRunRequest,
+): Record<string, unknown> {
+ const inputs: Record<string, unknown> = {};
+ const source = cleanRef(request.inputs?.source);
+ if (source) inputs.source = source;
+ const target = cleanRef(request.inputs?.target);
+ if (target) inputs.target = target;
+ const sourceText = request.inputs?.source_text?.trim();
+ if (sourceText) inputs.source_text = sourceText;
+ const params = request.convert;
+ const convert: Record<string, unknown> = {
+ mode: CONVERT_MODES.has(params?.mode) ? params.mode : "speech",
+ pitch_auto: params?.pitch_auto === true,
+ pitch:
+ typeof params?.pitch === "number" && Number.isFinite(params.pitch)
+ ? Math.min(24, Math.max(-24, Math.round(params.pitch)))
+ : null,
+ };
+ if (params?.style === "source" || params?.style === "target") {
+ convert.style = params.style;
+ }
+ const voice = params?.voice?.trim();
+ if (voice) convert.voice = voice;
+ const body: Record<string, unknown> = {
+ workflow: "convert",
+ inputs,
+ convert,
+ };
+ const options = cleanOptions(request.options);
+ if (Object.keys(options).length > 0) body.options = options;
+ if (typeof request.seed === "number" && Number.isInteger(request.seed))
+ body.seed = request.seed;
+ return body;
+}
+
/** Allowed keys only, empty values dropped: the route 422s on extra keys. */
export function buildAudioRunBody(
request: AudioRunRequest,
): Record<string, unknown> {
+ if (request.workflow === "convert") return buildConvertRunBody(request);
const body: Record<string, unknown> = { workflow: request.workflow };
if (request.workflow !== "separate") body.text = request.text ?? "";
const language = request.language?.trim();
@@ -175,15 +258,7 @@ export function buildAudioRunBody(
if (Object.keys(inputs).length > 0) body.inputs = inputs;
const edit = cleanEdit(request.edit);
if (edit) body.edit = edit;
- const options = Object.fromEntries(
- Object.entries(request.options ?? {}).filter(
- ([name, value]) =>
- name &&
- (typeof value === "boolean" ||
- typeof value === "string" ||
- (typeof value === "number" && Number.isFinite(value))),
- ),
- );
+ const options = cleanOptions(request.options);
if (Object.keys(options).length > 0) body.options = options;
if (typeof request.speed === "number" && Number.isFinite(request.speed))
body.speed = request.speed;
diff --git a/studio/frontend/src/features/audio/catalog.ts b/studio/frontend/src/features/audio/catalog.ts
index d696967aea7..9c99bfeecac 100644
--- a/studio/frontend/src/features/audio/catalog.ts
+++ b/studio/frontend/src/features/audio/catalog.ts
@@ -148,19 +148,20 @@ export function audioModelsForTask(task: AudioTask): ModelOption[] {
}
const CAPABILITY_LABEL: Record<
- AudioTask | "music" | "clone" | "edit" | "separate",
+ AudioTask | "music" | "clone" | "edit" | "convert" | "separate",
string
> = {
music: "Music generation",
separate: "Source separation",
clone: "Voice cloning",
edit: "Speech editing",
+ convert: "Voice conversion",
tts: "Text-to-speech",
stt: "Speech-to-text",
};
export function audioCapabilityLine(
- task: AudioTask | "music" | "clone" | "edit" | "separate",
+ task: AudioTask | "music" | "clone" | "edit" | "convert" | "separate",
detail?: string | null,
): string {
const base = CAPABILITY_LABEL[task];
diff --git a/studio/frontend/src/features/audio/components/audio-source-input.tsx b/studio/frontend/src/features/audio/components/audio-source-input.tsx
index b89e5358a59..b5335a7049b 100644
--- a/studio/frontend/src/features/audio/components/audio-source-input.tsx
+++ b/studio/frontend/src/features/audio/components/audio-source-input.tsx
@@ -40,10 +40,12 @@ import {
REFERENCE_EXPIRED_MESSAGE,
} from "../hooks/audio-source-state";
import { recordingSupported, useAudioSource } from "../hooks/use-audio-source";
+import { clipWorkflow } from "../workflows";
import { VoicePicker } from "./voice-picker";
import { Waveform } from "./waveform";
import { formatSeconds } from "./waveform-peaks";
+/** The gallery clips an input card offers under From history. Provided by the page. */
const AudioHistoryContext = createContext<readonly AudioGalleryClip[]>([]);
export const AudioHistoryProvider = AudioHistoryContext.Provider;
// False while the persistently mounted Audio page is hidden, so no card keeps the mic.
@@ -324,7 +326,9 @@ export function AudioSourceInput({
durationS !== null &&
durationS > usesFirstSeconds ? (
<p className="text-ui-11p5 leading-snug text-muted-foreground">
- Uses the first {usesFirstSeconds} s.
+ {usesFirstSeconds % 60 === 0 && usesFirstSeconds >= 60
+ ? `Uses the first ${usesFirstSeconds / 60} min.`
+ : `Uses the first ${usesFirstSeconds} s.`}
</p>
) : null}
</div>
@@ -427,7 +431,9 @@ export function AudioSourceInput({
<button
type="button"
disabled={disabled}
- onClick={() => onChange(clipReference(clip))}
+ onClick={() =>
+ onChange(clipReference({ ...clip, workflow: clipWorkflow(clip) }))
+ }
className="flex w-full min-w-0 items-center gap-2 rounded-full px-3 py-1.5 text-left text-ui-13 transition-colors hover:bg-accent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
>
<span className="min-w-0 flex-1 truncate">
diff --git a/studio/frontend/src/features/audio/components/clip-card.tsx b/studio/frontend/src/features/audio/components/clip-card.tsx
index f76cb214ed7..c8597c27643 100644
--- a/studio/frontend/src/features/audio/components/clip-card.tsx
+++ b/studio/frontend/src/features/audio/components/clip-card.tsx
@@ -84,7 +84,7 @@ export function ClipCard({
status?: string;
focusOnMount?: boolean;
onFocused?: () => void;
- /** Replaces the waveform player, e.g. Edit's Original/Edited compare. */
+ /** Replaces the waveform player, e.g. Edit's Original/Edited or Convert's Source/Converted compare. */
player?: ReactNode;
}) {
const focusPlay = (element: HTMLDivElement | null) => {
diff --git a/studio/frontend/src/features/audio/components/convert-ab-compare-state.ts b/studio/frontend/src/features/audio/components/convert-ab-compare-state.ts
new file mode 100644
index 00000000000..2f2359095c9
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/convert-ab-compare-state.ts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+export interface ABPlayback {
+ time: number;
+ playing: boolean;
+ duration: number;
+}
+
+/** Same moment on the other side, parked just before its end so it does not finish at once. */
+export function nextPlayback(
+ prev: ABPlayback,
+ nextDuration: number,
+): { seek: number; resume: boolean } {
+ const time = Number.isFinite(prev.time) && prev.time > 0 ? prev.time : 0;
+ if (!Number.isFinite(nextDuration) || nextDuration <= 0) {
+ return { seek: time, resume: prev.playing };
+ }
+ const lastMoment = Math.max(0, nextDuration - 0.05);
+ const seek = Math.min(time, lastMoment);
+ const resume = prev.playing && time < nextDuration;
+ return { seek, resume };
+}
diff --git a/studio/frontend/src/features/audio/components/convert-ab-compare.tsx b/studio/frontend/src/features/audio/components/convert-ab-compare.tsx
new file mode 100644
index 00000000000..d20ea284ea4
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/convert-ab-compare.tsx
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { PillTabs } from "@/features/model-picker/components/model-selector/pill-tabs";
+import { type Ref, useCallback, useRef } from "react";
+import { type ABPlayback, nextPlayback } from "./convert-ab-compare-state";
+
+export interface ABSide {
+ label: string;
+ src?: string;
+ unavailable?: string;
+}
+
+export type ABSideId = "a" | "b";
+
+export function ABCompare({
+ a,
+ b,
+ side,
+ onSideChange,
+ ariaLabel,
+ playerRef,
+}: {
+ a: ABSide;
+ b: ABSide;
+ side: ABSideId;
+ onSideChange: (side: ABSideId) => void;
+ ariaLabel: string;
+ playerRef?: Ref<HTMLAudioElement>;
+}) {
+ const audioRef = useRef<HTMLAudioElement | null>(null);
+ const setAudio = useCallback(
+ (element: HTMLAudioElement | null) => {
+ audioRef.current = element;
+ if (typeof playerRef === "function") playerRef(element);
+ else if (playerRef) playerRef.current = element;
+ },
+ [playerRef],
+ );
+ const pendingRef = useRef<ABPlayback | null>(null);
+
+ const shownSide: ABSideId =
+ side === "a" && a.unavailable && !b.unavailable
+ ? "b"
+ : side === "b" && b.unavailable && !a.unavailable
+ ? "a"
+ : side;
+ const current = shownSide === "a" ? a : b;
+ const src = current.unavailable ? undefined : current.src;
+
+ const switchTo = useCallback(
+ (next: string) => {
+ if (next !== "a" && next !== "b") return;
+ if (next === shownSide) return;
+ if ((next === "a" ? a : b).unavailable) return;
+ const audio = audioRef.current;
+ if (audio) {
+ pendingRef.current = {
+ time: audio.currentTime,
+ playing: !audio.paused && !audio.ended,
+ duration: audio.duration,
+ };
+ audio.pause();
+ }
+ onSideChange(next);
+ },
+ [a, b, onSideChange, shownSide],
+ );
+
+ const handleLoadedMetadata = useCallback(() => {
+ const audio = audioRef.current;
+ const pending = pendingRef.current;
+ if (!audio || !pending) return;
+ pendingRef.current = null;
+ const { seek, resume } = nextPlayback(pending, audio.duration);
+ if (seek > 0) audio.currentTime = seek;
+ if (resume) {
+ // Autoplay rules may refuse.
+ audio.play().catch(() => undefined);
+ }
+ }, []);
+
+ const reason = a.unavailable ?? b.unavailable;
+ return (
+ <div className="grid w-full gap-2">
+ <PillTabs
+ ariaLabel={ariaLabel}
+ value={shownSide}
+ onValueChange={switchTo}
+ fit={true}
+ compact={true}
+ className="[&>button]:px-3"
+ tabs={[
+ { value: "a", label: a.label, disabled: Boolean(a.unavailable) },
+ { value: "b", label: b.label, disabled: Boolean(b.unavailable) },
+ ]}
+ />
+ {src ? (
+ // biome-ignore lint/a11y/useMediaCaption: generated speech has no caption track; the clip's text is shown above it.
+ <audio
+ ref={setAudio}
+ controls={true}
+ src={src}
+ preload="metadata"
+ onLoadedMetadata={handleLoadedMetadata}
+ className="w-full rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
+ />
+ ) : (
+ <output className="flex h-12 w-full items-center justify-center rounded-md border border-border text-ui-12 text-muted-foreground">
+ {current.unavailable ?? "Loading audio…"}
+ </output>
+ )}
+ {reason && !current.unavailable ? (
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {reason}
+ </p>
+ ) : null}
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/audio/convert-policy.ts b/studio/frontend/src/features/audio/convert-policy.ts
new file mode 100644
index 00000000000..26cb7799f11
--- /dev/null
+++ b/studio/frontend/src/features/audio/convert-policy.ts
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import type { AudioConvertCaps } from "@/features/chat/types/api";
+import type {
+ AudioSourceSelection,
+ ConvertMode,
+ ConvertStyle,
+} from "./audio-run-request";
+import type { AudioModelContext } from "./tools/types";
+
+export const CONVERT_MODEL_ORDER: string[] = [
+ "SeedVC-MLX-GGUF",
+ "RVC-GGUF",
+ "MeanVC2-GGUF",
+ "Chatterbox-GGUF",
+ "Vevo2-GGUF",
+];
+
+export function convertCaps(
+ ctx: Pick<AudioModelContext, "convert">,
+): AudioConvertCaps | null {
+ return ctx.convert && ctx.convert.modes.length > 0 ? ctx.convert : null;
+}
+
+// Singing keeps the recording's style: a sung transcript is lyrics, which STT gets wrong.
+export function effectiveConvertStyle(
+ caps: AudioConvertCaps | null,
+ mode: ConvertMode,
+ style: ConvertStyle,
+): ConvertStyle {
+ return caps?.style && mode === "speech" ? style : "source";
+}
+
+export function convertPitchSupport(
+ caps: AudioConvertCaps | null,
+ mode: ConvertMode,
+ style: ConvertStyle,
+): { show: boolean; auto: boolean; shiftWithAuto: boolean } {
+ const pitch = caps?.modes.includes(mode) ? caps.pitch[mode] : undefined;
+ if (!pitch || effectiveConvertStyle(caps, mode, style) === "target") {
+ return { show: false, auto: false, shiftWithAuto: false };
+ }
+ const auto = pitch.auto === true;
+ return {
+ show: true,
+ auto,
+ shiftWithAuto: auto && pitch.shift_with_auto === true,
+ };
+}
+
+export function convertServerTask(
+ caps: AudioConvertCaps | null,
+ workflowTasks: Readonly<Record<string, string>> | null | undefined,
+ mode: ConvertMode,
+): string | null {
+ if (!caps?.modes.includes(mode) || !workflowTasks) return null;
+ return (
+ workflowTasks[`convert:${mode}`] ??
+ (mode === "speech" ? workflowTasks.convert : undefined) ??
+ null
+ );
+}
+
+export type ConvertBlockerKind =
+ | "source"
+ | "source-error"
+ | "source-expired"
+ | "target"
+ | "target-error"
+ | "target-expired"
+ | "mode"
+ | "source-text"
+ | "panel";
+
+export interface ConvertBlockerInput {
+ source: AudioSourceSelection | null;
+ sourceBusy: boolean;
+ sourceExpired: boolean;
+ sourceError: string | null;
+ target: AudioSourceSelection | null;
+ targetBusy: boolean;
+ targetExpired: boolean;
+ targetError: string | null;
+ builtinVoice: string;
+ caps: AudioConvertCaps | null;
+ mode: ConvertMode;
+ style: ConvertStyle;
+ sourceText: string;
+ panelError: string | null;
+}
+
+export const CONVERT_SOURCE_MISSING = "Add the recording to convert.";
+export const CONVERT_TARGET_MISSING = "Add the target voice.";
+export const CONVERT_BUILTIN_MISSING = "Pick a built-in voice.";
+export const CONVERT_SINGING_UNSUPPORTED =
+ "This model does not convert singing. Switch to Speech.";
+export const CONVERT_SOURCE_TEXT_MISSING = "Add what's said in the recording.";
+export const CONVERT_INPUTS_MISSING =
+ "Add the recording to convert and the target voice.";
+
+type ConvertBlocker = { kind: ConvertBlockerKind; reason: string };
+
+function inputBlocker(
+ selection: AudioSourceSelection | null,
+ busy: boolean,
+ expired: boolean,
+ error: string | null,
+ side: "source" | "target",
+): ConvertBlocker | null {
+ const what = side === "source" ? "recording" : "target voice";
+ if (expired) {
+ return {
+ kind: `${side}-expired`,
+ reason: `The ${what} is no longer on the server.`,
+ };
+ }
+ if (busy) {
+ return {
+ kind: side,
+ reason: `Waiting for the ${what} to finish uploading.`,
+ };
+ }
+ if (error && !selection) return { kind: `${side}-error`, reason: error };
+ if (!selection) {
+ return {
+ kind: side,
+ reason:
+ side === "source" ? CONVERT_SOURCE_MISSING : CONVERT_TARGET_MISSING,
+ };
+ }
+ return null;
+}
+
+function builtinBlocker(
+ caps: AudioConvertCaps,
+ builtinVoice: string,
+): ConvertBlocker | null {
+ const voice = builtinVoice.trim();
+ const known =
+ caps.builtin_voices.length === 0 ||
+ caps.builtin_voices.some((item) => item.id === voice);
+ return voice && known
+ ? null
+ : { kind: "target", reason: CONVERT_BUILTIN_MISSING };
+}
+
+export function convertBlocker(
+ input: ConvertBlockerInput,
+): ConvertBlocker | null {
+ const source = inputBlocker(
+ input.source,
+ input.sourceBusy,
+ input.sourceExpired,
+ input.sourceError,
+ "source",
+ );
+ const target =
+ input.caps?.target === "builtin"
+ ? builtinBlocker(input.caps, input.builtinVoice)
+ : inputBlocker(
+ input.target,
+ input.targetBusy,
+ input.targetExpired,
+ input.targetError,
+ "target",
+ );
+ // Name everything still missing at once, so nothing new appears after the first fix.
+ if (
+ source?.kind === "source" &&
+ !input.source &&
+ !input.sourceBusy &&
+ target?.kind === "target" &&
+ !input.target &&
+ input.caps?.target !== "builtin"
+ ) {
+ return { kind: "source", reason: CONVERT_INPUTS_MISSING };
+ }
+ if (source?.kind === "source-expired" && target?.kind === "target-expired") {
+ return {
+ kind: "source-expired",
+ reason: "The recording and target voice are no longer on the server.",
+ };
+ }
+ const blocker = source ?? target;
+ if (blocker) return blocker;
+ if (input.caps && !input.caps.modes.includes(input.mode)) {
+ return { kind: "mode", reason: CONVERT_SINGING_UNSUPPORTED };
+ }
+ if (
+ effectiveConvertStyle(input.caps, input.mode, input.style) === "target" &&
+ !input.sourceText.trim()
+ ) {
+ return { kind: "source-text", reason: CONVERT_SOURCE_TEXT_MISSING };
+ }
+ return input.panelError ? { kind: "panel", reason: input.panelError } : null;
+}
+
+// Measured on GPU; families missing here get no estimate.
+const CONVERT_RELOAD_SECONDS: Readonly<Record<string, string>> = {
+ seed_vc: "about 4–8 s",
+ vevo2: "about 6 s",
+ chatterbox: "about 2 s",
+ meanvc2: "about 2 s",
+};
+
+export interface ConvertSwitchInput {
+ modelName: string;
+ loadedTask: string | null;
+ nextTask: string | null;
+ routeChange: boolean;
+ family?: string | null;
+}
+
+export function convertSwitchNotice(
+ input: ConvertSwitchInput,
+): { before: string; during: string } | null {
+ const taskChange = Boolean(
+ input.loadedTask && input.nextTask && input.loadedTask !== input.nextTask,
+ );
+ if (!taskChange && !input.routeChange) return null;
+ const name = input.modelName.trim() || "the model";
+ const what = !taskChange
+ ? "the new engine"
+ : input.nextTask === "svc"
+ ? "singing"
+ : "Convert";
+ const seconds = input.family
+ ? CONVERT_RELOAD_SECONDS[input.family]
+ : undefined;
+ return {
+ before: `Reloads ${name} ${taskChange ? "for" : "with"} ${what}${seconds ? `, ${seconds}` : ""}.`,
+ during: `Switching ${name} to ${what}…`,
+ };
+}
diff --git a/studio/frontend/src/features/audio/edit-adapters.ts b/studio/frontend/src/features/audio/edit-adapters.ts
index d480d34f7bc..715c021f9ad 100644
--- a/studio/frontend/src/features/audio/edit-adapters.ts
+++ b/studio/frontend/src/features/audio/edit-adapters.ts
@@ -6,7 +6,7 @@
import type { AudioOptionValues } from "./audio-options";
import type {
AudioRunEditPart,
- AudioRunRequest,
+ AudioTextRunRequest,
AudioSourceRef,
} from "./audio-run-request";
import {
@@ -248,7 +248,7 @@ export function editAdapterFor(
export function buildEditRun(
adapter: EditAdapter,
input: EditRunInput,
-): AudioRunRequest {
+): AudioTextRunRequest {
const mode: EditMode =
input.mode === "delivery" && adapter.delivery ? "delivery" : "words";
const transcript = input.transcript.trim();
@@ -256,10 +256,10 @@ export function buildEditRun(
mode === "delivery"
? transcript || input.sourceName?.trim() || "Recording"
: input.edited.trim();
- const inputs: NonNullable<AudioRunRequest["inputs"]> = {};
+ const inputs: NonNullable<AudioTextRunRequest["inputs"]> = {};
if (input.source) inputs.source = { ...input.source };
if (transcript) inputs.reference_text = transcript;
- const run: AudioRunRequest = {
+ const run: AudioTextRunRequest = {
workflow: "edit",
text,
inputs,
@@ -282,7 +282,7 @@ export function buildEditRun(
/** The progress line for a chained run, or null when it is one pass. */
export function editPhaseLabel(
adapter: EditAdapter,
- run: AudioRunRequest,
+ run: AudioTextRunRequest,
): string | null {
if (adapter.style !== "instructions") return null;
const delivery = run.edit?.mode === "delivery";
diff --git a/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx b/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
index 489e7610998..47cc917c34c 100644
--- a/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
+++ b/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
@@ -64,7 +64,7 @@ export function useAudioGallery({
prompt: string;
model: string;
saved: boolean;
- workflow: "speak" | "music" | "clone" | "edit" | "separate";
+ workflow: "speak" | "music" | "clone" | "edit" | "convert" | "separate";
} | null>(null);
const fallbackClipRef = useRef(fallbackClip);
fallbackClipRef.current = fallbackClip;
@@ -425,7 +425,7 @@ export function useAudioGallery({
[refreshGallery],
);
- const handleClearGallery = useCallback(async (workflow?: "speak" | "clone" | "edit" | "music" | "separate") => {
+ const handleClearGallery = useCallback(async (workflow?: "speak" | "clone" | "edit" | "convert" | "music" | "separate") => {
try {
await clearAudioGallery(workflow);
if (workflow) {
@@ -555,7 +555,7 @@ export function useWorkflowHistory({
loadingMoreRef,
selectClip,
}: {
- workflow: "speak" | "clone" | "edit" | "music" | "separate";
+ workflow: "speak" | "clone" | "edit" | "convert" | "music" | "separate";
enabled: boolean;
} & Pick<
AudioGallery,
diff --git a/studio/frontend/src/features/audio/hooks/use-audio-source.ts b/studio/frontend/src/features/audio/hooks/use-audio-source.ts
index 4f88067ab14..894f56a910c 100644
--- a/studio/frontend/src/features/audio/hooks/use-audio-source.ts
+++ b/studio/frontend/src/features/audio/hooks/use-audio-source.ts
@@ -198,6 +198,8 @@ export function useAudioSource({
// Each pick draws under its own key, so a slow decode of an earlier file cannot land on this one.
const localKey = `local:${++pickCount.current}`;
dispatch({ type: "upload-start", name: fileName, key: localKey });
+ // The card now shows the new file: a run must not quietly use the one it replaced.
+ onChangeRef.current(null);
void drawBlob(localKey, file);
try {
const record = await uploadAudioInput(file, fileName, {
diff --git a/studio/frontend/src/features/audio/hooks/use-clone-generation.ts b/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
index 8d3d141dfe4..89d94bc0b46 100644
--- a/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
+++ b/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
@@ -55,7 +55,7 @@ export async function showRunResult({
}: {
response: AudioRunResponse;
text: string;
- workflow: "speak" | "clone" | "edit" | "music" | "separate";
+ workflow: "speak" | "clone" | "edit" | "convert" | "music" | "separate";
} & Pick<
AudioGallery,
"refreshGallery" | "selectClip" | "setFallbackClip" | "setSelectedId"
@@ -364,6 +364,8 @@ export function useCloneGeneration({
setFallbackClip,
setSelectedId,
});
+ // The run can restart audio.cpp under another task (Chatterbox: clon after vc).
+ await refreshStatus();
} catch (error) {
if (!controller.signal.aborted) {
updateGenerationPhase("finishing");
@@ -385,8 +387,9 @@ export function useCloneGeneration({
// Kept under Generate so the reason outlives the toast; expiry is already on the card.
setGenerationError(message);
if (!expired) toast.error(message);
- await refreshStatus();
}
+ // Also after Stop: the run may already have restarted audio.cpp under another task.
+ await refreshStatus();
} finally {
generateAbort.current = null;
updateGenerationPhase(null);
diff --git a/studio/frontend/src/features/audio/hooks/use-convert-generation.ts b/studio/frontend/src/features/audio/hooks/use-convert-generation.ts
new file mode 100644
index 00000000000..5cc075bf893
--- /dev/null
+++ b/studio/frontend/src/features/audio/hooks/use-convert-generation.ts
@@ -0,0 +1,534 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { toast } from "@/lib/toast";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { AudioApiError, runAudio } from "../api";
+import {
+ type AudioOptionSpec,
+ type AudioOptionValues,
+ audioOptionLabel,
+ audioOptionsForRequest,
+ missingRequiredAudioOptions,
+ parseAudioOptions,
+} from "../audio-options";
+import {
+ type AudioSourceSelection,
+ selectionExpired,
+ sourceRefOf,
+} from "../audio-run-request";
+import type { AudioSourceInputHandle } from "../components/audio-source-input";
+import {
+ type ConvertBlockerKind,
+ convertBlocker,
+ convertCaps,
+ convertPitchSupport,
+ convertServerTask,
+ convertSwitchNotice,
+ effectiveConvertStyle,
+} from "../convert-policy";
+import type { GenerateBlocker } from "../pages/tts-workspace";
+import { useAudioCloneStore } from "../stores/audio-clone-store";
+import { useAudioConvertStore } from "../stores/audio-convert-store";
+import { audioToolPanelsFor } from "../tools/registry";
+import {
+ audioModelContextFor,
+ claimedOptionNames,
+ collectToolRequest,
+ toolValueKey,
+} from "../tools/select";
+import type { AudioHostState } from "./audio-host-state";
+import type { AudioSourceStatus } from "./audio-source-state";
+import { REFERENCE_EXPIRED_MESSAGE } from "./audio-source-state";
+import type { AudioGallery } from "./use-audio-gallery";
+import type { AudioModelSlot } from "./use-audio-model-slot";
+import { showRunResult } from "./use-clone-generation";
+import { useReferenceTranscribe } from "./use-reference-transcribe";
+
+export const CONVERT_SOURCE_TEXT_FIELD_ID = "convert-source-text";
+
+// Seed-VC starts on its V2 engine until a run picks another one.
+const SEED_VC_DEFAULT_ROUTE = "v2_vc";
+
+const IDLE: AudioSourceStatus = { phase: "idle" };
+
+function sourceBusy(status: AudioSourceStatus): boolean {
+ return status.phase === "uploading" || status.phase === "recording";
+}
+
+export function useConvertGeneration({
+ status,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ generateAbort,
+ setMode,
+ setAdvancedOpen,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ pendingTranscribeRelease,
+ replayQueuedTtsPick,
+ audioOptionSpecs,
+ audioOptionValues,
+ sttRepo,
+}: Pick<
+ AudioHostState,
+ | "status"
+ | "busyRef"
+ | "setBusy"
+ | "updateGenerationPhase"
+ | "generateAbort"
+ | "setMode"
+ | "setAdvancedOpen"
+ | "refreshStatus"
+ | "activeRef"
+ | "modeRef"
+> &
+ Pick<
+ AudioGallery,
+ "refreshGallery" | "selectClip" | "setFallbackClip" | "setSelectedId"
+ > &
+ Pick<AudioModelSlot, "pendingTranscribeRelease" | "replayQueuedTtsPick"> & {
+ audioOptionSpecs: AudioOptionSpec[];
+ audioOptionValues: AudioOptionValues;
+ sttRepo: string | null;
+ }) {
+ const source = useAudioConvertStore((state) => state.source);
+ const target = useAudioConvertStore((state) => state.target);
+ const builtinVoice = useAudioConvertStore((state) => state.builtinVoice);
+ const storedMode = useAudioConvertStore((state) => state.mode);
+ const pitchAuto = useAudioConvertStore((state) => state.pitchAuto);
+ const pitch = useAudioConvertStore((state) => state.pitch);
+ const sourceText = useAudioConvertStore((state) => state.sourceText);
+ const storedToolValues = useAudioCloneStore((state) => state.toolValues);
+ const model = status?.active_model ?? null;
+ const [generationError, setGenerationError] = useState<string | null>(null);
+ const [sourceStatus, setSourceStatus] = useState<AudioSourceStatus>(IDLE);
+ const [targetStatus, setTargetStatus] = useState<AudioSourceStatus>(IDLE);
+ const sourceHandle = useRef<AudioSourceInputHandle | null>(null);
+ const targetHandle = useRef<AudioSourceInputHandle | null>(null);
+ const [expiredIds, setExpiredIds] = useState<ReadonlySet<string>>(
+ () => new Set(),
+ );
+ // Uploading the same audio again returns the same id, which is live again.
+ const sourceId = source?.id ?? null;
+ const targetId = target?.id ?? null;
+ useEffect(() => {
+ setExpiredIds((previous) => {
+ if (![sourceId, targetId].some((id) => id && previous.has(id))) {
+ return previous;
+ }
+ const next = new Set(previous);
+ for (const id of [sourceId, targetId]) {
+ if (id) {
+ next.delete(id);
+ }
+ }
+ return next;
+ });
+ }, [sourceId, targetId]);
+ const [runningNotice, setRunningNotice] = useState<string | null>(null);
+
+ const transcriber = useReferenceTranscribe({
+ sttRepo,
+ purpose: "convert",
+ // A transcript that finishes after the recording changed belongs to the old one.
+ onText: (next, recording) => {
+ const store = useAudioConvertStore.getState();
+ if (
+ store.source?.kind === recording.kind &&
+ store.source.id === recording.id
+ ) {
+ store.setSourceText(next);
+ }
+ },
+ });
+
+ const toolContext = useMemo(
+ () =>
+ audioModelContextFor(status, {
+ musicGeneration: false,
+ cudaMusicGeneration: false,
+ musicNeedsDescription: false,
+ }),
+ [status],
+ );
+ const caps = convertCaps(toolContext);
+ const mode =
+ caps && !caps.modes.includes(storedMode)
+ ? (caps.modes[0] ?? "speech")
+ : storedMode;
+ const toolPanels = useMemo(
+ () => audioToolPanelsFor("convert", toolContext),
+ [toolContext],
+ );
+ const toolValues = useMemo(() => {
+ const values: Record<string, unknown> = {};
+ for (const panel of toolPanels) {
+ values[panel.id] =
+ storedToolValues[toolValueKey(model, "convert", panel.id)];
+ }
+ return values;
+ }, [toolPanels, storedToolValues, model]);
+ const handleToolValueChange = useCallback(
+ (panelId: string, value: unknown) =>
+ useAudioCloneStore
+ .getState()
+ .setToolValue(toolValueKey(model, "convert", panelId), value),
+ [model],
+ );
+ const convertOptionSpecs = useMemo(() => {
+ const byWorkflow = status?.audio_options_by_workflow?.convert;
+ return byWorkflow === undefined
+ ? audioOptionSpecs
+ : parseAudioOptions(byWorkflow);
+ }, [status?.audio_options_by_workflow, audioOptionSpecs]);
+ const toolRequest = useMemo(
+ () =>
+ collectToolRequest(
+ toolPanels,
+ toolValues,
+ { text: "", referenceText: sourceText, hasReference: source !== null },
+ { ...toolContext, convertMode: mode },
+ convertOptionSpecs,
+ ),
+ [
+ toolPanels,
+ toolValues,
+ sourceText,
+ source,
+ toolContext,
+ mode,
+ convertOptionSpecs,
+ ],
+ );
+ const style = effectiveConvertStyle(
+ caps,
+ mode,
+ toolRequest.patch.convert?.style ?? "source",
+ );
+ const route = toolRequest.patch.convert?.route ?? null;
+ const pitchSupport = convertPitchSupport(caps, mode, style);
+ const claimedOptions = useMemo(
+ () => claimedOptionNames(toolPanels),
+ [toolPanels],
+ );
+ const advancedOptionSpecs = useMemo(
+ () => convertOptionSpecs.filter((spec) => !claimedOptions.has(spec.name)),
+ [convertOptionSpecs, claimedOptions],
+ );
+
+ const nextTask = convertServerTask(caps, status?.audio_workflow_tasks, mode);
+ const loadedTask = status?.audio_server_task ?? null;
+ const loadedRoute =
+ status?.audio_convert_route ??
+ (loadedTask === nextTask ? SEED_VC_DEFAULT_ROUTE : null);
+ const switchNotice =
+ caps && model
+ ? convertSwitchNotice({
+ modelName: model.split("/").pop() ?? model,
+ family: status?.audio_family ?? null,
+ loadedTask,
+ nextTask,
+ routeChange:
+ caps.route_reloads &&
+ route !== null &&
+ loadedRoute !== null &&
+ route !== loadedRoute,
+ })
+ : null;
+
+ // An upload can pass its keep-until time while the page sits open.
+ const [now, setNow] = useState(() => Date.now());
+ useEffect(() => {
+ const expiring = [source, target].some(
+ (item) => item?.kind === "input" && item.expiresAt,
+ );
+ if (!expiring) return;
+ const timer = window.setInterval(() => setNow(Date.now()), 60_000);
+ return () => window.clearInterval(timer);
+ }, [source, target]);
+ const isExpired = (
+ selection: AudioSourceSelection | null,
+ cardStatus: AudioSourceStatus,
+ ) =>
+ cardStatus.phase === "expired" ||
+ selectionExpired(selection, now) ||
+ (selection !== null && expiredIds.has(selection.id));
+ const sourceExpired = isExpired(source, sourceStatus);
+ const targetExpired = isExpired(target, targetStatus);
+ const builtinTarget = caps?.target === "builtin";
+
+ const inputBlocker = convertBlocker({
+ source,
+ sourceBusy: sourceBusy(sourceStatus),
+ sourceExpired,
+ sourceError: sourceStatus.phase === "error" ? sourceStatus.message : null,
+ target: builtinTarget ? null : target,
+ targetBusy: sourceBusy(targetStatus),
+ targetExpired: !builtinTarget && targetExpired,
+ targetError:
+ !builtinTarget && targetStatus.phase === "error"
+ ? targetStatus.message
+ : null,
+ builtinVoice,
+ caps,
+ mode,
+ style,
+ sourceText,
+ panelError: toolRequest.error,
+ });
+ const againAction = (
+ handle: typeof sourceHandle,
+ clear: () => void,
+ ): GenerateBlocker["actions"] => [
+ {
+ label: "Add it again",
+ onClick: () => {
+ clear();
+ handle.current?.browse();
+ },
+ },
+ ];
+ const focusAction = (label: string, handle: typeof sourceHandle) => [
+ { label, onClick: () => handle.current?.focus() },
+ ];
+ const blockerActions = (
+ kind: ConvertBlockerKind,
+ ): GenerateBlocker["actions"] => {
+ switch (kind) {
+ case "source":
+ case "source-error":
+ return focusAction("Add a recording", sourceHandle);
+ case "target":
+ case "target-error":
+ return focusAction("Add the target voice", targetHandle);
+ case "source-expired":
+ return againAction(sourceHandle, () =>
+ useAudioConvertStore.getState().setSource(null),
+ );
+ case "target-expired":
+ return againAction(targetHandle, () =>
+ useAudioConvertStore.getState().setTarget(null),
+ );
+ case "source-text":
+ return [
+ {
+ label: "Transcribe it",
+ onClick: () => void transcriber.transcribe(source),
+ },
+ {
+ label: "type it",
+ onClick: () =>
+ document.getElementById(CONVERT_SOURCE_TEXT_FIELD_ID)?.focus(),
+ },
+ ];
+ default:
+ return undefined;
+ }
+ };
+ const blocker: GenerateBlocker | null = inputBlocker
+ ? {
+ reason: inputBlocker.reason,
+ actions: blockerActions(inputBlocker.kind),
+ }
+ : null;
+
+ const handleGenerate = useCallback(async () => {
+ const state = useAudioConvertStore.getState();
+ if (!state.source || inputBlocker || !caps) return;
+ if (busyRef.current) return;
+ busyRef.current = "generating";
+ setBusy("generating");
+ updateGenerationPhase("preparing");
+ const finishEarly = () => {
+ updateGenerationPhase(null);
+ busyRef.current = null;
+ setBusy(null);
+ };
+ const releaseInFlight = pendingTranscribeRelease.current;
+ if (releaseInFlight && !(await releaseInFlight)) {
+ finishEarly();
+ setMode("transcribe");
+ return;
+ }
+ const missingOptions = missingRequiredAudioOptions(
+ advancedOptionSpecs,
+ audioOptionValues,
+ );
+ if (missingOptions.length > 0) {
+ finishEarly();
+ setAdvancedOpen(true);
+ toast.error(
+ `Set ${missingOptions.map((spec) => audioOptionLabel(spec.name)).join(", ")} in Advanced before converting.`,
+ );
+ return;
+ }
+ const patch = toolRequest.patch;
+ const sourceSelection = state.source;
+ const targetSelection = builtinTarget ? null : state.target;
+ const controller = new AbortController();
+ generateAbort.current = controller;
+ setRunningNotice(switchNotice?.during ?? null);
+ updateGenerationPhase(switchNotice ? "switching" : "generating");
+ const autoPitch = pitchSupport.show && pitchSupport.auto && state.pitchAuto;
+ try {
+ const response = await runAudio(
+ {
+ workflow: "convert",
+ inputs: {
+ source: sourceRefOf(sourceSelection),
+ target: targetSelection ? sourceRefOf(targetSelection) : null,
+ source_text: style === "target" ? state.sourceText.trim() : null,
+ },
+ convert: {
+ mode,
+ pitch:
+ pitchSupport.show && (!autoPitch || pitchSupport.shiftWithAuto)
+ ? state.pitch
+ : null,
+ pitch_auto: autoPitch,
+ style: caps.style ? style : undefined,
+ voice: builtinTarget ? state.builtinVoice : null,
+ },
+ options: {
+ ...audioOptionsForRequest(advancedOptionSpecs, audioOptionValues),
+ ...patch.options,
+ },
+ },
+ controller.signal,
+ );
+ updateGenerationPhase("finishing");
+ await showRunResult({
+ response,
+ text: `${sourceSelection.name} → ${
+ targetSelection?.name ?? state.builtinVoice
+ }`,
+ workflow: "convert",
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ });
+ if (switchNotice) await refreshStatus();
+ } catch (error) {
+ if (!controller.signal.aborted) {
+ updateGenerationPhase("finishing");
+ // Only the expired-upload 404: a deleted clip or voice says otherwise and leaves uploads alone.
+ const gone =
+ error instanceof AudioApiError &&
+ error.status === 404 &&
+ error.message === REFERENCE_EXPIRED_MESSAGE
+ ? [sourceSelection, targetSelection].filter(
+ (item): item is AudioSourceSelection => item?.kind === "input",
+ )
+ : [];
+ if (gone.length > 0) {
+ setExpiredIds((previous) => {
+ const next = new Set(previous);
+ for (const item of gone) next.add(item.id);
+ return next;
+ });
+ for (const item of gone) {
+ (item === sourceSelection
+ ? sourceHandle
+ : targetHandle
+ ).current?.markExpired();
+ }
+ }
+ const message =
+ gone.length > 0
+ ? REFERENCE_EXPIRED_MESSAGE
+ : error instanceof Error
+ ? error.message
+ : "Voice conversion failed.";
+ setGenerationError(message);
+ if (gone.length === 0) toast.error(message);
+ await refreshStatus();
+ } else if (switchNotice) {
+ // Stopped while switching: the server may or may not run the new task now.
+ await refreshStatus();
+ }
+ } finally {
+ generateAbort.current = null;
+ setRunningNotice(null);
+ updateGenerationPhase(null);
+ busyRef.current = null;
+ setBusy(null);
+ if (activeRef.current && modeRef.current === "speak")
+ replayQueuedTtsPick();
+ }
+ }, [
+ inputBlocker,
+ caps,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ pendingTranscribeRelease,
+ setMode,
+ advancedOptionSpecs,
+ audioOptionValues,
+ setAdvancedOpen,
+ toolRequest,
+ builtinTarget,
+ generateAbort,
+ switchNotice,
+ pitchSupport,
+ style,
+ mode,
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ replayQueuedTtsPick,
+ ]);
+
+ useEffect(() => {
+ if (source || target) setGenerationError(null);
+ }, [source, target]);
+
+ return {
+ source,
+ target,
+ builtinVoice,
+ mode,
+ style,
+ pitch,
+ pitchAuto,
+ pitchSupport,
+ sourceText,
+ caps,
+ sourceHandle,
+ targetHandle,
+ sourceStatus,
+ setSourceStatus,
+ targetStatus,
+ setTargetStatus,
+ sourceExpired,
+ targetExpired,
+ transcriber,
+ toolContext,
+ toolPanels,
+ toolValues,
+ handleToolValueChange,
+ claimedOptions,
+ advancedOptionSpecs,
+ switchNotice,
+ runningNotice,
+ blocker,
+ inputsReady: inputBlocker === null,
+ handleGenerate,
+ generationError,
+ setGenerationError,
+ };
+}
+
+export type ConvertGeneration = ReturnType<typeof useConvertGeneration>;
diff --git a/studio/frontend/src/features/audio/hooks/use-reference-transcribe.ts b/studio/frontend/src/features/audio/hooks/use-reference-transcribe.ts
index 36fc1d44c31..3b6313fc4e5 100644
--- a/studio/frontend/src/features/audio/hooks/use-reference-transcribe.ts
+++ b/studio/frontend/src/features/audio/hooks/use-reference-transcribe.ts
@@ -13,9 +13,12 @@ import { sttEngineForRepoId, sttSidecarKeyFor } from "../catalog";
export function useReferenceTranscribe({
sttRepo,
onText,
+ purpose = "reference",
}: {
sttRepo: string | null;
onText: (text: string, reference: AudioSourceSelection) => void;
+ /** "convert" transcribes up to Convert's cap instead of the 30 s clone reference. */
+ purpose?: "reference" | "convert";
}) {
const [transcribing, setTranscribing] = useState(false);
const [error, setError] = useState<string | null>(null);
@@ -41,6 +44,7 @@ export function useReferenceTranscribe({
model,
engine: sttRepo ? sttEngineForRepoId(sttRepo) : sttEngineFor(model),
device: voice.sttDevice,
+ purpose,
},
controller.signal,
);
@@ -67,7 +71,7 @@ export function useReferenceTranscribe({
}
}
},
- [sttRepo],
+ [sttRepo, purpose],
);
const cancel = useCallback(() => {
diff --git a/studio/frontend/src/features/audio/hooks/use-speech-generation.ts b/studio/frontend/src/features/audio/hooks/use-speech-generation.ts
index 63c3035d84c..2cbff61fa7f 100644
--- a/studio/frontend/src/features/audio/hooks/use-speech-generation.ts
+++ b/studio/frontend/src/features/audio/hooks/use-speech-generation.ts
@@ -5,6 +5,7 @@ import { useCallback, useEffect, useMemo, useState } from "react";
import { readLastPrompt, saveLastPrompt } from "@/lib/last-prompt";
import { toast } from "@/lib/toast";
import { generateAudio, runAudio } from "../api";
+import { TTS_MAX_TOKENS } from "../audio-workspace-constants";
import {
audioOptionLabel,
audioOptionsForRequest,
@@ -35,7 +36,6 @@ import {
isMusicGenerationModel,
musicGenerationRequiresCuda,
} from "../catalog";
-import { TTS_MAX_TOKENS } from "../audio-workspace-constants";
import type { AudioHostState } from "./audio-host-state";
import { galleryCache } from "./use-audio-gallery";
import { useAudioCloneStore } from "../stores/audio-clone-store";
diff --git a/studio/frontend/src/features/audio/music/music-policy.ts b/studio/frontend/src/features/audio/music/music-policy.ts
index d4e64ab8640..eec8dd68611 100644
--- a/studio/frontend/src/features/audio/music/music-policy.ts
+++ b/studio/frontend/src/features/audio/music/music-policy.ts
@@ -5,7 +5,7 @@
import type {
AudioOptionScalar,
- AudioRunRequest,
+ AudioTextRunRequest,
AudioSourceSelection,
} from "../audio-run-request";
import { sourceRefOf } from "../audio-run-request";
@@ -285,7 +285,7 @@ export function buildMusicRunRequest(
options?: Record<string, AudioOptionScalar>;
seed?: number | null;
},
-): AudioRunRequest {
+): AudioTextRunRequest {
const { rule } = inputs;
const base = {
workflow: "music" as const,
diff --git a/studio/frontend/src/features/audio/pages/convert-page.tsx b/studio/frontend/src/features/audio/pages/convert-page.tsx
new file mode 100644
index 00000000000..57dce72e37a
--- /dev/null
+++ b/studio/frontend/src/features/audio/pages/convert-page.tsx
@@ -0,0 +1,481 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Button } from "@/components/ui/button";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
+import { ParamSlider } from "@/features/chat";
+import type { ModelOption } from "@/features/model-picker/components/model-selector/types";
+import { PillTabs } from "@/features/model-picker/components/model-selector/pill-tabs";
+import { SpeechToTextIcon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import { type ComponentProps, useEffect, useState } from "react";
+import { type AudioGalleryClip, fetchAudioBlob } from "../api";
+import { audioModelLabel } from "../audio-workspace-utils";
+import { audioCppModelFor, audioCppWorkflowsFor } from "../audio-cpp-catalog";
+import { sourceFileUrl } from "../audio-run-request";
+import { macTtsCatalogChoiceIsRunnable } from "../catalog";
+import { ABCompare } from "../components/convert-ab-compare";
+import { Field } from "../components/field";
+import {
+ AudioHistoryProvider,
+ AudioSourceInput,
+} from "../components/audio-source-input";
+import { CONVERT_MODEL_ORDER } from "../convert-policy";
+import {
+ CONVERT_SOURCE_TEXT_FIELD_ID,
+ type ConvertGeneration,
+} from "../hooks/use-convert-generation";
+import { useAudioConvertStore } from "../stores/audio-convert-store";
+import { AudioToolPanels } from "../tools/tool-panel-host";
+import { TtsFooter, TtsOutput, TtsRailFields } from "./tts-workspace";
+
+function convertRank(id: string): number {
+ const index = CONVERT_MODEL_ORDER.findIndex((name) =>
+ id.endsWith(`/${name}`),
+ );
+ return index === -1 ? CONVERT_MODEL_ORDER.length : index;
+}
+
+export function convertPageModels(
+ models: ModelOption[],
+ isMac: boolean,
+): ModelOption[] {
+ return models
+ .filter((model) => {
+ const entry = audioCppModelFor(model.id);
+ return (
+ entry !== null &&
+ audioCppWorkflowsFor(entry).includes("convert") &&
+ (!isMac || macTtsCatalogChoiceIsRunnable(model.id))
+ );
+ })
+ .map((model, index) => ({ model, index }))
+ .sort(
+ (a, b) =>
+ convertRank(a.model.id) - convertRank(b.model.id) || a.index - b.index,
+ )
+ .map(({ model }) => model);
+}
+
+type RailProps = Omit<
+ ComponentProps<typeof TtsRailFields>,
+ | "musicGeneration"
+ | "prompt"
+ | "setPrompt"
+ | "toolPanels"
+ | "inputs"
+ | "claimedOptions"
+ | "audioOptionSpecs"
+>;
+
+function ConvertInputs({
+ convert,
+ historyClips,
+ disabled,
+}: {
+ convert: ConvertGeneration;
+ historyClips: readonly AudioGalleryClip[];
+ disabled: boolean;
+}) {
+ const store = useAudioConvertStore.getState();
+ const { caps, source, pitchSupport } = convert;
+ const maxMinutes = Math.round((caps?.source_max_seconds ?? 300) / 60);
+ return (
+ <AudioHistoryProvider value={historyClips}>
+ <div data-tour="audio-convert-source">
+ <AudioSourceInput
+ id="convert-source"
+ label="Recording"
+ hint={`The speech or singing to convert. Converts the first ${maxMinutes} minutes.`}
+ value={source}
+ onChange={store.setSource}
+ disabled={disabled}
+ allowSavedVoice={false}
+ handleRef={convert.sourceHandle}
+ onStatusChange={convert.setSourceStatus}
+ usesFirstSeconds={caps?.source_max_seconds ?? 300}
+ maxRecordSeconds={caps?.source_max_seconds ?? 300}
+ expiredMessage="This recording expired. Add it again."
+ recordHint="Speak or sing what to convert."
+ />
+ </div>
+
+ <div data-tour="audio-convert-target">
+ {caps?.target === "builtin" ? (
+ <Field
+ label="Target voice"
+ htmlFor="convert-builtin-voice"
+ hint="This model converts to its built-in voices only."
+ >
+ <Select
+ value={convert.builtinVoice}
+ onValueChange={store.setBuiltinVoice}
+ disabled={disabled}
+ >
+ <SelectTrigger
+ id="convert-builtin-voice"
+ size="sm"
+ className="w-full"
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {caps.builtin_voices.map((voice) => (
+ <SelectItem key={voice.id} value={voice.id}>
+ {voice.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </Field>
+ ) : (
+ <AudioSourceInput
+ id="convert-target"
+ label="Target voice"
+ hint="A few seconds of the voice the recording should take."
+ value={convert.target}
+ onChange={store.setTarget}
+ disabled={disabled}
+ handleRef={convert.targetHandle}
+ onStatusChange={convert.setTargetStatus}
+ usesFirstSeconds={30}
+ expiredMessage="This voice clip expired. Add it again."
+ />
+ )}
+ </div>
+
+ {caps && caps.modes.length > 1 ? (
+ <div className="grid gap-1.5">
+ <span className="text-ui-13 font-medium text-foreground">
+ The recording is
+ </span>
+ <PillTabs
+ ariaLabel="Speech or singing"
+ value={convert.mode}
+ onValueChange={(mode) =>
+ store.setMode(mode === "singing" ? "singing" : "speech")
+ }
+ disabled={disabled}
+ fit={true}
+ compact={true}
+ className="[&>button]:px-3"
+ tabs={[
+ { value: "speech", label: "Speech" },
+ { value: "singing", label: "Singing" },
+ ]}
+ />
+ </div>
+ ) : null}
+
+ {pitchSupport.show ? (
+ <div className="grid gap-1.5">
+ {pitchSupport.auto ? (
+ <label
+ htmlFor="convert-pitch-auto"
+ className="flex items-center justify-between gap-3 text-ui-13 font-medium text-foreground"
+ >
+ Match the target's pitch
+ <Switch
+ id="convert-pitch-auto"
+ checked={convert.pitchAuto}
+ disabled={disabled}
+ onCheckedChange={store.setPitchAuto}
+ />
+ </label>
+ ) : null}
+ {pitchSupport.auto &&
+ convert.pitchAuto &&
+ !pitchSupport.shiftWithAuto ? (
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ Turn off to set the pitch shift yourself.
+ </p>
+ ) : (
+ <ParamSlider
+ label={
+ pitchSupport.auto && convert.pitchAuto ? "Extra shift" : "Pitch"
+ }
+ value={convert.pitch}
+ min={-24}
+ max={24}
+ step={1}
+ disabled={disabled}
+ displayValue={`${convert.pitch > 0 ? "+" : ""}${convert.pitch} st`}
+ info={
+ pitchSupport.auto && convert.pitchAuto
+ ? "Semitones added after matching the target's pitch, e.g. +12 to sing an octave up."
+ : "Semitones. +12 is an octave up; try -12 to +12 when the target voice is much lower or higher."
+ }
+ onChange={(next) => store.setPitch(Math.round(next))}
+ />
+ )}
+ </div>
+ ) : null}
+ </AudioHistoryProvider>
+ );
+}
+
+function SourceTextField({
+ convert,
+ disabled,
+}: {
+ convert: ConvertGeneration;
+ disabled: boolean;
+}) {
+ const store = useAudioConvertStore.getState();
+ const { transcriber, source } = convert;
+ const sourceUsable =
+ source !== null &&
+ !convert.sourceExpired &&
+ convert.sourceStatus.phase !== "uploading";
+ if (convert.style !== "target") return null;
+ return (
+ <div className="grid gap-1.5">
+ <div className="flex items-center justify-between gap-2">
+ <label
+ htmlFor={CONVERT_SOURCE_TEXT_FIELD_ID}
+ className="text-ui-13 font-medium text-foreground"
+ >
+ What's said in the recording
+ <span className="ml-1.5 font-normal text-muted-foreground">
+ Required
+ </span>
+ </label>
+ <Button
+ type="button"
+ variant="ghost"
+ size="sm"
+ className="h-auto px-2 py-1 text-ui-11p5"
+ disabled={!sourceUsable || transcriber.transcribing || disabled}
+ onClick={() => void transcriber.transcribe(source)}
+ >
+ <HugeiconsIcon icon={SpeechToTextIcon} className="size-3.5" />
+ {transcriber.transcribing ? "Transcribing…" : "Transcribe"}
+ </Button>
+ </div>
+ <Textarea
+ id={CONVERT_SOURCE_TEXT_FIELD_ID}
+ value={convert.sourceText}
+ disabled={transcriber.transcribing}
+ aria-busy={transcriber.transcribing}
+ onChange={(event) => store.setSourceText(event.target.value)}
+ placeholder="Type the words in the recording, or press Transcribe."
+ className="min-h-16"
+ />
+ {transcriber.error ? (
+ <p role="alert" className="text-ui-11p5 leading-snug text-destructive">
+ {transcriber.error}{" "}
+ <Button
+ type="button"
+ variant="link"
+ className="h-auto p-0 text-ui-11p5 font-medium text-foreground"
+ onClick={() => void transcriber.transcribe(source)}
+ >
+ Try again
+ </Button>
+ </p>
+ ) : (
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ Taking the target's style rebuilds the delivery from these words.
+ </p>
+ )}
+ </div>
+ );
+}
+
+export function ConvertRail({
+ convert,
+ historyClips,
+ ...props
+}: RailProps & {
+ convert: ConvertGeneration;
+ historyClips: readonly AudioGalleryClip[];
+}) {
+ const disabled = props.busy === "generating";
+ const ctx = { ...convert.toolContext, convertMode: convert.mode };
+ return (
+ <TtsRailFields
+ {...props}
+ musicGeneration={false}
+ prompt=""
+ setPrompt={() => {}}
+ audioOptionSpecs={convert.advancedOptionSpecs}
+ claimedOptions={convert.claimedOptions}
+ inputs={
+ <ConvertInputs
+ convert={convert}
+ historyClips={historyClips}
+ disabled={disabled}
+ />
+ }
+ toolPanels={
+ <>
+ <AudioToolPanels
+ workflow="convert"
+ ctx={ctx}
+ values={convert.toolValues}
+ onChange={convert.handleToolValueChange}
+ specs={convert.advancedOptionSpecs}
+ disabled={disabled}
+ core={{
+ text: "",
+ referenceText: convert.sourceText,
+ hasReference: convert.source !== null,
+ }}
+ />
+ <SourceTextField convert={convert} disabled={disabled} />
+ </>
+ }
+ />
+ );
+}
+
+export function ConvertFooter({
+ convert,
+ ...props
+}: Omit<
+ ComponentProps<typeof TtsFooter>,
+ | "prompt"
+ | "lyricsOptional"
+ | "musicNeedsDescription"
+ | "audioInstructions"
+ | "handleGenerate"
+> & { convert: ConvertGeneration }) {
+ const notice =
+ props.busy === null && props.blocker === null && !props.error
+ ? (convert.switchNotice?.before ?? null)
+ : null;
+ return (
+ <div className="flex w-full flex-col items-center gap-1.5">
+ <TtsFooter
+ {...props}
+ prompt="convert"
+ lyricsOptional={false}
+ musicNeedsDescription={false}
+ audioInstructions=""
+ handleGenerate={convert.handleGenerate}
+ />
+ {notice ? (
+ <p className="text-center text-ui-11p5 leading-snug text-muted-foreground">
+ {notice}
+ </p>
+ ) : null}
+ </div>
+ );
+}
+
+function useSourceSide(clip: AudioGalleryClip | null): {
+ src?: string;
+ unavailable?: string;
+} {
+ // The saved copy outlives the upload it was made from.
+ const savedUrl = clip?.source_saved
+ ? `/api/inference/audio/gallery/${encodeURIComponent(clip.id)}/source/file`
+ : null;
+ const sourceId = clip?.source_clip_id ?? clip?.source_input_id ?? null;
+ const kind = clip?.source_clip_id ? "clip" : "input";
+ const [side, setSide] = useState<{ src?: string; unavailable?: string }>({});
+ useEffect(() => {
+ if (!savedUrl && !sourceId) {
+ setSide({ unavailable: "The source is not saved with this clip." });
+ return;
+ }
+ let url: string | null = null;
+ let cancelled = false;
+ setSide({});
+ fetchAudioBlob(
+ savedUrl ??
+ sourceFileUrl({ kind, id: sourceId ?? "", name: "", durationS: null }),
+ )
+ .then((blob) => {
+ if (cancelled) return;
+ url = URL.createObjectURL(blob);
+ setSide({ src: url });
+ })
+ .catch(() => {
+ if (!cancelled)
+ setSide({
+ unavailable: "The source recording is no longer available.",
+ });
+ });
+ return () => {
+ cancelled = true;
+ if (url) URL.revokeObjectURL(url);
+ };
+ }, [savedUrl, sourceId, kind]);
+ return side;
+}
+
+export function ConvertOutput({
+ modelReady,
+ recommendedModels = [],
+ onPickModel,
+ ...props
+}: Omit<
+ ComponentProps<typeof TtsOutput>,
+ "emptyText" | "clipBadge" | "emptyActions" | "renderPlayer"
+> & {
+ modelReady: boolean;
+ recommendedModels?: readonly ModelOption[];
+ onPickModel?: (id: string) => void;
+}) {
+ const picks = modelReady || !onPickModel ? [] : recommendedModels.slice(0, 3);
+ const compareSide = useAudioConvertStore((state) => state.compareSide);
+ const setCompareSide = useAudioConvertStore((state) => state.setCompareSide);
+ const sourceSide = useSourceSide(props.selectedClip);
+ return (
+ <TtsOutput
+ {...props}
+ // The selected clip's line already names the model.
+ clipBadge={(clip, place) =>
+ place === "history" ? audioModelLabel(clip.model) : null
+ }
+ showCopyText={false}
+ renderPlayer={(clip, src, focusRef) => (
+ <ABCompare
+ key={clip.id}
+ ariaLabel="Compare the source and the converted voice"
+ a={{ label: "Source", ...sourceSide }}
+ b={{ label: "Converted", src }}
+ side={compareSide === "source" ? "a" : "b"}
+ onSideChange={(side) =>
+ setCompareSide(side === "a" ? "source" : "converted")
+ }
+ playerRef={focusRef}
+ />
+ )}
+ emptyActions={
+ picks.length > 0 ? (
+ <div
+ className="flex flex-wrap justify-center gap-2"
+ aria-label="Recommended models"
+ >
+ {picks.map((model) => (
+ <Button
+ key={model.id}
+ type="button"
+ variant="outline"
+ size="sm"
+ onClick={() => onPickModel?.(model.id)}
+ >
+ Use {model.name}
+ </Button>
+ ))}
+ </div>
+ ) : null
+ }
+ emptyText={
+ modelReady
+ ? "Converted recordings land here. Add a recording and the target voice, and press Generate."
+ : "Converted recordings land here. Load a model that can convert a voice, add a recording, and press Generate."
+ }
+ />
+ );
+}
diff --git a/studio/frontend/src/features/audio/pages/tts-workspace.tsx b/studio/frontend/src/features/audio/pages/tts-workspace.tsx
index 868a4e77ccf..f212356fc33 100644
--- a/studio/frontend/src/features/audio/pages/tts-workspace.tsx
+++ b/studio/frontend/src/features/audio/pages/tts-workspace.tsx
@@ -474,7 +474,10 @@ export function TtsOutput({
onFreshClipFocused,
announcement,
clipBadge,
+ renderPlayer,
selectedPlayer,
+ useAgainLabel = "Use text again",
+ showCopyText = true,
}: Pick<
AudioGallery,
| "srcById"
@@ -513,7 +516,19 @@ export function TtsOutput({
freshClipId: string | null;
onFreshClipFocused: () => void;
announcement: string;
- clipBadge?: (clip: AudioGalleryClip) => string | null;
+ /** A short tag after a clip's text, such as the voice a clone used. */
+ clipBadge?: (
+ clip: AudioGalleryClip,
+ place: "selected" | "history",
+ ) => string | null;
+ /** Replaces the selected clip's waveform, such as Convert's Source/Converted compare. */
+ renderPlayer?: (
+ clip: AudioGalleryClip,
+ src: string,
+ focusRef: ((element: HTMLAudioElement | null) => void) | undefined,
+ ) => ReactNode;
+ useAgainLabel?: string;
+ showCopyText?: boolean;
/** Replaces the plain player; `focusRef` goes on its first control. */
selectedPlayer?: (
clip: AudioGalleryClip,
@@ -565,16 +580,18 @@ export function TtsOutput({
strokeWidth={1.75}
className="size-icon"
/>
- Use text again
- </DropdownMenuItem>
- <DropdownMenuItem onClick={() => void handleCopyPrompt(clip.prompt)}>
- <HugeiconsIcon
- icon={Copy01Icon}
- strokeWidth={1.75}
- className="size-icon"
- />
- Copy text
+ {useAgainLabel}
</DropdownMenuItem>
+ {showCopyText ? (
+ <DropdownMenuItem onClick={() => void handleCopyPrompt(clip.prompt)}>
+ <HugeiconsIcon
+ icon={Copy01Icon}
+ strokeWidth={1.75}
+ className="size-icon"
+ />
+ Copy text
+ </DropdownMenuItem>
+ ) : null}
<MusicSendToMenuItems clip={clip} />
</>
}
@@ -610,8 +627,17 @@ export function TtsOutput({
: null
}
menu={clipMenu(selectedClip, "row")}
- player={customPlayer ?? undefined}
- status={clipBadge?.(selectedClip) ?? undefined}
+ player={
+ customPlayer ??
+ (renderPlayer && selectedClipSrc
+ ? renderPlayer(
+ selectedClip,
+ selectedClipSrc,
+ selectedClip.id === freshClipId ? focusFreshClip : undefined,
+ )
+ : undefined)
+ }
+ status={clipBadge?.(selectedClip, "selected") ?? undefined}
focusOnMount={selectedClip.id === freshClipId}
onFocused={onFreshClipFocused}
/>
@@ -701,7 +727,7 @@ export function TtsOutput({
<span className="min-w-0 flex-1 truncate">
{clip.prompt}
</span>
- <ClipBadge text={clipBadge?.(clip)} />
+ <ClipBadge text={clipBadge?.(clip, "history")} />
<span className="hidden min-w-0 max-w-[45%] shrink truncate text-ui-11p5 text-muted-foreground @[30rem]:block">
{audioModelLabel(clip.model)} ·{" "}
{formatRelativeShort(clip.created_at)}
diff --git a/studio/frontend/src/features/audio/stores/audio-convert-store.ts b/studio/frontend/src/features/audio/stores/audio-convert-store.ts
new file mode 100644
index 00000000000..4ca89667361
--- /dev/null
+++ b/studio/frontend/src/features/audio/stores/audio-convert-store.ts
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { create } from "zustand";
+import { persist } from "zustand/middleware";
+import type { AudioSourceSelection, ConvertMode } from "../audio-run-request";
+
+export const AUDIO_CONVERT_STORAGE_KEY = "unsloth_audio_convert_v1";
+
+type ConvertCompareSide = "source" | "converted";
+
+// Tool panel values live in useAudioCloneStore.toolValues, keyed by toolValueKey(model, "convert", id).
+interface AudioConvertState {
+ source: AudioSourceSelection | null;
+ target: AudioSourceSelection | null;
+ builtinVoice: string;
+ mode: ConvertMode;
+ pitchAuto: boolean;
+ pitch: number;
+ sourceText: string;
+ compareSide: ConvertCompareSide;
+ setSource: (source: AudioSourceSelection | null) => void;
+ setTarget: (target: AudioSourceSelection | null) => void;
+ setBuiltinVoice: (builtinVoice: string) => void;
+ setMode: (mode: ConvertMode) => void;
+ setPitchAuto: (pitchAuto: boolean) => void;
+ setPitch: (pitch: number) => void;
+ setSourceText: (sourceText: string) => void;
+ setCompareSide: (compareSide: ConvertCompareSide) => void;
+}
+
+const clampPitch = (pitch: number) =>
+ Number.isFinite(pitch) ? Math.min(24, Math.max(-24, Math.round(pitch))) : 0;
+
+export const useAudioConvertStore = create<AudioConvertState>()(
+ persist(
+ (set) => ({
+ source: null,
+ target: null,
+ builtinVoice: "default",
+ mode: "speech",
+ pitchAuto: true,
+ pitch: 0,
+ sourceText: "",
+ compareSide: "converted",
+ // The transcript belongs to the recording: a new one brings its own, or none.
+ setSource: (source) =>
+ set((state) =>
+ state.source?.kind === source?.kind && state.source?.id === source?.id
+ ? { source }
+ : { source, sourceText: source?.transcript?.trim() ?? "" },
+ ),
+ setTarget: (target) => set({ target }),
+ setBuiltinVoice: (builtinVoice) => set({ builtinVoice }),
+ setMode: (mode) => set({ mode }),
+ setPitchAuto: (pitchAuto) => set({ pitchAuto }),
+ setPitch: (pitch) => set({ pitch: clampPitch(pitch) }),
+ setSourceText: (sourceText) => set({ sourceText }),
+ setCompareSide: (compareSide) => set({ compareSide }),
+ }),
+ {
+ name: AUDIO_CONVERT_STORAGE_KEY,
+ version: 1,
+ partialize: (state) => ({
+ source: state.source,
+ target: state.target,
+ builtinVoice: state.builtinVoice,
+ mode: state.mode,
+ pitchAuto: state.pitchAuto,
+ pitch: state.pitch,
+ sourceText: state.sourceText,
+ compareSide: state.compareSide,
+ }),
+ },
+ ),
+);
diff --git a/studio/frontend/src/features/audio/tools/convert-panel-logic.ts b/studio/frontend/src/features/audio/tools/convert-panel-logic.ts
new file mode 100644
index 00000000000..8c254bbc3e9
--- /dev/null
+++ b/studio/frontend/src/features/audio/tools/convert-panel-logic.ts
@@ -0,0 +1,216 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Seed-VC and RVC refuse undeclared options, so each panel sends only what its engine reads.
+
+import type { AudioOptionSpec } from "../audio-options";
+import type { ConvertStyle } from "../audio-run-request";
+import type { AudioModelContext, AudioToolPanel } from "./types";
+
+type AudioToolPanelLogic<V> = Omit<AudioToolPanel<V>, "Component">;
+
+const clamp = (value: number, min: number, max: number) =>
+ Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : min;
+
+function specDefault(
+ specs: readonly AudioOptionSpec[],
+ name: string,
+ fallback: number,
+): number {
+ const value = specs.find((spec) => spec.name === name)?.default;
+ return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+
+const isSinging = (ctx: AudioModelContext | undefined) =>
+ ctx?.convertMode === "singing";
+
+export type SeedVcEngine =
+ | "v2_vc"
+ | "v1_whisper_bigvgan_vc"
+ | "v1_xlsr_hift_vc";
+
+export const SEED_VC_ENGINES: readonly {
+ value: SeedVcEngine;
+ label: string;
+}[] = [
+ { value: "v2_vc", label: "V2" },
+ { value: "v1_whisper_bigvgan_vc", label: "V1 Whisper" },
+ { value: "v1_xlsr_hift_vc", label: "V1 XLSR" },
+];
+
+export const SEED_VC_SINGING_ROUTE = "v1_svc";
+
+export interface SeedVcValue {
+ engine: SeedVcEngine;
+ similarity: number;
+ intelligibility: number;
+ guidance: number;
+ length: number;
+ steps: number;
+ anonymize: boolean;
+ more?: boolean;
+}
+
+export const SEED_VC_LENGTH_RANGE = { min: 0.5, max: 2 } as const;
+export const SEED_VC_STEPS_RANGE = { min: 1, max: 100 } as const;
+export const SEED_VC_GUIDANCE_RANGE = { min: 0, max: 2 } as const;
+
+export function seedVcRoute(
+ value: SeedVcValue,
+ ctx?: AudioModelContext,
+): string {
+ if (isSinging(ctx)) return SEED_VC_SINGING_ROUTE;
+ return SEED_VC_ENGINES.some((engine) => engine.value === value.engine)
+ ? value.engine
+ : "v2_vc";
+}
+
+export const seedVcLogic: AudioToolPanelLogic<SeedVcValue> = {
+ id: "seed-vc",
+ families: ["seed_vc"],
+ workflows: ["convert"],
+ title: "Seed-VC",
+ // f0_condition fails on the speech engines, so it is claimed and never shown.
+ claims: [
+ "route",
+ "similarity_guidance_scale",
+ "intelligibility_guidance_scale",
+ "inference_guidance_scale",
+ "length_adjust",
+ "num_inference_steps",
+ "voice_anonymization",
+ "f0_condition",
+ "auto_f0_adjust",
+ "semitone_shift",
+ ],
+ initial: (specs) => ({
+ engine: "v2_vc",
+ similarity: specDefault(specs, "similarity_guidance_scale", 0.7),
+ intelligibility: specDefault(specs, "intelligibility_guidance_scale", 0.7),
+ guidance: specDefault(specs, "inference_guidance_scale", 0.7),
+ length: specDefault(specs, "length_adjust", 1),
+ steps: specDefault(specs, "num_inference_steps", 30),
+ anonymize: false,
+ }),
+ toRequest: (value, ctx) => {
+ const route = seedVcRoute(value, ctx);
+ const shared = {
+ route,
+ length_adjust: clamp(
+ value.length,
+ SEED_VC_LENGTH_RANGE.min,
+ SEED_VC_LENGTH_RANGE.max,
+ ),
+ num_inference_steps: Math.round(
+ clamp(value.steps, SEED_VC_STEPS_RANGE.min, SEED_VC_STEPS_RANGE.max),
+ ),
+ };
+ const guidance = (v: number) =>
+ clamp(v, SEED_VC_GUIDANCE_RANGE.min, SEED_VC_GUIDANCE_RANGE.max);
+ const options =
+ route === "v2_vc"
+ ? {
+ ...shared,
+ similarity_guidance_scale: guidance(value.similarity),
+ intelligibility_guidance_scale: guidance(value.intelligibility),
+ voice_anonymization: value.anonymize === true,
+ }
+ : { ...shared, inference_guidance_scale: guidance(value.guidance) };
+ return { options, convert: { route } };
+ },
+};
+
+export interface RvcValue {
+ blend: number;
+ protect: number;
+ rms: number;
+}
+
+export const rvcLogic: AudioToolPanelLogic<RvcValue> = {
+ id: "rvc",
+ families: ["rvc"],
+ workflows: ["convert"],
+ title: "RVC",
+ claims: [
+ "retrieval_blend",
+ "unvoiced_protection",
+ "rms_mix_rate",
+ "voice_id",
+ "semitone_shift",
+ ],
+ initial: (specs) => ({
+ blend: specDefault(specs, "retrieval_blend", 0),
+ protect: specDefault(specs, "unvoiced_protection", 0.33),
+ rms: specDefault(specs, "rms_mix_rate", 0.25),
+ }),
+ toRequest: (value) => ({
+ options: {
+ retrieval_blend: clamp(value.blend, 0, 1),
+ unvoiced_protection: clamp(value.protect, 0, 0.5),
+ rms_mix_rate: clamp(value.rms, 0, 1),
+ },
+ }),
+};
+
+export interface ChatterboxConvertValue {
+ guidance: number;
+ steps: number;
+ more?: boolean;
+}
+
+export const CHATTERBOX_CONVERT_GUIDANCE_RANGE = { min: 0, max: 3 } as const;
+export const CHATTERBOX_CONVERT_STEPS_RANGE = { min: 1, max: 50 } as const;
+
+export const chatterboxConvertLogic: AudioToolPanelLogic<ChatterboxConvertValue> =
+ {
+ id: "chatterbox-convert",
+ families: ["chatterbox"],
+ workflows: ["convert"],
+ title: "Chatterbox",
+ claims: ["s3gen_cfg_rate", "num_inference_steps"],
+ initial: (specs) => ({
+ guidance: specDefault(specs, "s3gen_cfg_rate", 0.7),
+ steps: specDefault(specs, "num_inference_steps", 10),
+ }),
+ toRequest: (value) => ({
+ options: {
+ s3gen_cfg_rate: clamp(
+ value.guidance,
+ CHATTERBOX_CONVERT_GUIDANCE_RANGE.min,
+ CHATTERBOX_CONVERT_GUIDANCE_RANGE.max,
+ ),
+ num_inference_steps: Math.round(
+ clamp(
+ value.steps,
+ CHATTERBOX_CONVERT_STEPS_RANGE.min,
+ CHATTERBOX_CONVERT_STEPS_RANGE.max,
+ ),
+ ),
+ },
+ }),
+ };
+
+export interface Vevo2StyleValue {
+ style: ConvertStyle;
+}
+
+export const vevo2StyleLogic: AudioToolPanelLogic<Vevo2StyleValue> = {
+ id: "vevo2-style",
+ families: ["vevo2"],
+ workflows: ["convert"],
+ title: "Vevo2",
+ claims: [],
+ initial: () => ({ style: "source" }),
+ toRequest: (value, ctx) => ({
+ convert: {
+ style: !isSinging(ctx) && value.style === "target" ? "target" : "source",
+ },
+ }),
+};
+
+export const CONVERT_PANEL_LOGIC = [
+ seedVcLogic,
+ rvcLogic,
+ chatterboxConvertLogic,
+ vevo2StyleLogic,
+] as const;
diff --git a/studio/frontend/src/features/audio/tools/convert-panels.tsx b/studio/frontend/src/features/audio/tools/convert-panels.tsx
new file mode 100644
index 00000000000..24051e172bd
--- /dev/null
+++ b/studio/frontend/src/features/audio/tools/convert-panels.tsx
@@ -0,0 +1,339 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { Switch } from "@/components/ui/switch";
+import { ParamSlider } from "@/features/chat";
+import { PillTabs } from "@/features/model-picker/components/model-selector/pill-tabs";
+import { ChevronDownStandardIcon } from "@/lib/chevron-icons";
+import { cn } from "@/lib/utils";
+import { HugeiconsIcon } from "@hugeicons/react";
+import type { ReactNode } from "react";
+import {
+ CHATTERBOX_CONVERT_GUIDANCE_RANGE,
+ CHATTERBOX_CONVERT_STEPS_RANGE,
+ type ChatterboxConvertValue,
+ type RvcValue,
+ SEED_VC_ENGINES,
+ SEED_VC_GUIDANCE_RANGE,
+ SEED_VC_LENGTH_RANGE,
+ SEED_VC_STEPS_RANGE,
+ type SeedVcEngine,
+ type SeedVcValue,
+ type Vevo2StyleValue,
+ chatterboxConvertLogic,
+ rvcLogic,
+ seedVcLogic,
+ seedVcRoute,
+ vevo2StyleLogic,
+} from "./convert-panel-logic";
+import type { AudioToolPanel } from "./types";
+
+function PanelSection({
+ title,
+ hint,
+ children,
+}: {
+ title: string;
+ hint?: string;
+ children: ReactNode;
+}) {
+ return (
+ <section className="grid gap-3" aria-label={title}>
+ <div className="grid gap-0.5">
+ <h3 className="text-ui-13 font-medium text-foreground">{title}</h3>
+ {hint ? (
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {hint}
+ </p>
+ ) : null}
+ </div>
+ {children}
+ </section>
+ );
+}
+
+function MoreSettings({
+ open,
+ onOpenChange,
+ children,
+}: {
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+ children: ReactNode;
+}) {
+ return (
+ <div className="grid gap-3">
+ <button
+ type="button"
+ aria-expanded={open}
+ onClick={() => onOpenChange(!open)}
+ className="flex w-fit items-center gap-1 rounded-full text-ui-11p5 font-medium text-muted-foreground transition-colors hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
+ >
+ More settings
+ <HugeiconsIcon
+ icon={ChevronDownStandardIcon}
+ className={cn(
+ "size-3 transition-transform duration-150 motion-reduce:transition-none",
+ open && "rotate-180",
+ )}
+ />
+ </button>
+ {open ? children : null}
+ </div>
+ );
+}
+
+export const seedVcPanel: AudioToolPanel<SeedVcValue> = {
+ ...seedVcLogic,
+ Component: ({ value, onChange, disabled, ctx }) => {
+ const singing = ctx.convertMode === "singing";
+ const v2 = seedVcRoute(value, ctx) === "v2_vc";
+ return (
+ <PanelSection
+ title="Seed-VC"
+ hint={singing ? "Singing uses the V1 singing engine." : undefined}
+ >
+ {singing ? null : (
+ <div className="grid gap-1.5">
+ <label
+ htmlFor="convert-seed-vc-engine"
+ className="text-ui-13 font-medium text-foreground"
+ >
+ Engine
+ </label>
+ <Select
+ value={value.engine}
+ onValueChange={(engine) =>
+ onChange({ ...value, engine: engine as SeedVcEngine })
+ }
+ disabled={disabled}
+ >
+ <SelectTrigger
+ id="convert-seed-vc-engine"
+ size="sm"
+ className="w-full"
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {SEED_VC_ENGINES.map((engine) => (
+ <SelectItem key={engine.value} value={engine.value}>
+ {engine.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ )}
+ {v2 ? (
+ <>
+ <ParamSlider
+ label="Sound like the target"
+ value={value.similarity}
+ min={SEED_VC_GUIDANCE_RANGE.min}
+ max={SEED_VC_GUIDANCE_RANGE.max}
+ step={0.05}
+ disabled={disabled}
+ info="Higher follows the target voice more closely."
+ onChange={(similarity) => onChange({ ...value, similarity })}
+ />
+ <ParamSlider
+ label="Keep words clear"
+ value={value.intelligibility}
+ min={SEED_VC_GUIDANCE_RANGE.min}
+ max={SEED_VC_GUIDANCE_RANGE.max}
+ step={0.05}
+ disabled={disabled}
+ info="Higher keeps the words easier to understand."
+ onChange={(intelligibility) =>
+ onChange({ ...value, intelligibility })
+ }
+ />
+ </>
+ ) : (
+ <ParamSlider
+ label="Guidance"
+ value={value.guidance}
+ min={SEED_VC_GUIDANCE_RANGE.min}
+ max={SEED_VC_GUIDANCE_RANGE.max}
+ step={0.05}
+ disabled={disabled}
+ info="How strongly the output is pushed toward the target voice."
+ onChange={(guidance) => onChange({ ...value, guidance })}
+ />
+ )}
+ <MoreSettings
+ open={value.more === true}
+ onOpenChange={(more) => onChange({ ...value, more })}
+ >
+ <ParamSlider
+ label="Length"
+ value={value.length}
+ min={SEED_VC_LENGTH_RANGE.min}
+ max={SEED_VC_LENGTH_RANGE.max}
+ step={0.05}
+ disabled={disabled}
+ displayValue={`${value.length.toFixed(2)}×`}
+ info="Below 1 speeds the result up, above 1 slows it down."
+ onChange={(length) => onChange({ ...value, length })}
+ />
+ <ParamSlider
+ label="Steps"
+ value={value.steps}
+ min={SEED_VC_STEPS_RANGE.min}
+ max={SEED_VC_STEPS_RANGE.max}
+ step={1}
+ disabled={disabled}
+ info="More steps can sound cleaner but take longer."
+ onChange={(steps) => onChange({ ...value, steps })}
+ />
+ {v2 ? (
+ <div className="grid gap-1.5">
+ <label
+ htmlFor="convert-seed-vc-anonymize"
+ className="flex items-center justify-between gap-3 text-ui-13 font-medium text-foreground"
+ >
+ Anonymize
+ <Switch
+ id="convert-seed-vc-anonymize"
+ checked={value.anonymize}
+ disabled={disabled}
+ onCheckedChange={(anonymize) =>
+ onChange({ ...value, anonymize })
+ }
+ />
+ </label>
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ Makes the voice hard to recognise instead of copying the target.
+ </p>
+ </div>
+ ) : null}
+ </MoreSettings>
+ </PanelSection>
+ );
+ },
+};
+
+export const rvcPanel: AudioToolPanel<RvcValue> = {
+ ...rvcLogic,
+ Component: ({ value, onChange, disabled }) => (
+ <PanelSection title="RVC">
+ <ParamSlider
+ label="Index blend"
+ value={value.blend}
+ min={0}
+ max={1}
+ step={0.05}
+ disabled={disabled}
+ info="Higher brings in more of the voice's accent and texture; too high can blur words."
+ onChange={(blend) => onChange({ ...value, blend })}
+ />
+ <ParamSlider
+ label="Protect consonants"
+ value={value.protect}
+ min={0}
+ max={0.5}
+ step={0.01}
+ disabled={disabled}
+ info="Keeps breaths and soft consonants from the recording. 0.5 turns it off."
+ onChange={(protect) => onChange({ ...value, protect })}
+ />
+ <ParamSlider
+ label="Volume envelope mix"
+ value={value.rms}
+ min={0}
+ max={1}
+ step={0.05}
+ disabled={disabled}
+ info="0 keeps the recording's loudness changes; 1 uses the voice's own."
+ onChange={(rms) => onChange({ ...value, rms })}
+ />
+ </PanelSection>
+ ),
+};
+
+export const chatterboxConvertPanel: AudioToolPanel<ChatterboxConvertValue> = {
+ ...chatterboxConvertLogic,
+ Component: ({ value, onChange, disabled }) => (
+ <PanelSection title="Chatterbox">
+ <ParamSlider
+ label="Guidance"
+ value={value.guidance}
+ min={CHATTERBOX_CONVERT_GUIDANCE_RANGE.min}
+ max={CHATTERBOX_CONVERT_GUIDANCE_RANGE.max}
+ step={0.05}
+ disabled={disabled}
+ info="How strongly the output is pushed toward the target voice."
+ onChange={(guidance) => onChange({ ...value, guidance })}
+ />
+ <MoreSettings
+ open={value.more === true}
+ onOpenChange={(more) => onChange({ ...value, more })}
+ >
+ <ParamSlider
+ label="Steps"
+ value={value.steps}
+ min={CHATTERBOX_CONVERT_STEPS_RANGE.min}
+ max={CHATTERBOX_CONVERT_STEPS_RANGE.max}
+ step={1}
+ disabled={disabled}
+ info="More steps can sound cleaner but take longer."
+ onChange={(steps) => onChange({ ...value, steps })}
+ />
+ </MoreSettings>
+ </PanelSection>
+ ),
+};
+
+// ---- Vevo2 -----------------------------------------------------------------------------------
+
+const VEVO2_HINTS = {
+ singing: "Singing keeps the recording's own style.",
+ target:
+ "Takes the target's accent and delivery too. Needs what's said in the recording.",
+ source: "Changes only the voice; the recording's delivery stays.",
+} as const;
+
+export const vevo2StylePanel: AudioToolPanel<Vevo2StyleValue> = {
+ ...vevo2StyleLogic,
+ Component: ({ value, onChange, disabled, ctx }) => (
+ <PanelSection
+ title="Vevo2"
+ hint={
+ VEVO2_HINTS[ctx.convertMode === "singing" ? "singing" : value.style]
+ }
+ >
+ {ctx.convertMode === "singing" ? null : (
+ <PillTabs
+ ariaLabel="Style"
+ value={value.style}
+ onValueChange={(style) =>
+ onChange({ style: style === "target" ? "target" : "source" })
+ }
+ disabled={disabled}
+ fit={true}
+ compact={true}
+ className="[&>button]:px-3"
+ tabs={[
+ { value: "source", label: "Keep source style" },
+ { value: "target", label: "Take target style" },
+ ]}
+ />
+ )}
+ </PanelSection>
+ ),
+};
+
+export const CONVERT_TOOL_PANELS = [
+ seedVcPanel,
+ rvcPanel,
+ chatterboxConvertPanel,
+ vevo2StylePanel,
+] as const;
diff --git a/studio/frontend/src/features/audio/tools/registry.tsx b/studio/frontend/src/features/audio/tools/registry.tsx
index 682be6c0dce..5c2d863a40e 100644
--- a/studio/frontend/src/features/audio/tools/registry.tsx
+++ b/studio/frontend/src/features/audio/tools/registry.tsx
@@ -9,6 +9,7 @@ import {
import type { AudioWorkflowId } from "../workflows";
import { CLONE_TOOL_PANELS } from "./clone-panels";
import { EDIT_TOOL_PANELS } from "./edit-panels";
+import { CONVERT_TOOL_PANELS } from "./convert-panels";
import { MUSIC_TOOL_PANELS } from "./music-panels";
import { SEPARATE_TOOL_PANELS } from "./separate-panels";
import {
@@ -117,6 +118,7 @@ export const AUDIO_TOOL_PANELS: readonly AnyAudioToolPanel[] = [
...SPEAK_TOOL_PANELS.slice(1),
...CLONE_TOOL_PANELS,
...EDIT_TOOL_PANELS,
+ ...CONVERT_TOOL_PANELS,
...MUSIC_TOOL_PANELS,
...SEPARATE_TOOL_PANELS,
];
diff --git a/studio/frontend/src/features/audio/tools/select.ts b/studio/frontend/src/features/audio/tools/select.ts
index 6eb59860c41..9377c3fdc04 100644
--- a/studio/frontend/src/features/audio/tools/select.ts
+++ b/studio/frontend/src/features/audio/tools/select.ts
@@ -3,6 +3,7 @@
// Which tool panels a model gets. Free of JSX so the node test runner can load it.
+import type { AudioConvertCaps } from "@/features/chat/types/api";
import type { AudioOptionSpec } from "../audio-options";
import {
type NativeAudioInstructionsKind,
@@ -64,6 +65,7 @@ export function audioModelContextFor(
audio_workflows?: readonly string[] | null;
audio_required_inputs?: readonly string[] | null;
audio_reference_text?: string | null;
+ audio_convert?: AudioConvertCaps | null;
audio_music?: unknown;
} | null,
page: {
@@ -91,6 +93,35 @@ export function audioModelContextFor(
? (referenceText as AudioReferenceTextMode)
: null,
audioMusic: parseMusicCapabilities(status?.audio_music) !== null,
+ convert: convertCapsOf(status?.audio_convert),
+ };
+}
+
+function convertCapsOf(value: unknown): AudioConvertCaps | null {
+ if (!isPlainObject(value)) return null;
+ const caps = value as Partial<AudioConvertCaps>;
+ const modes = Array.isArray(caps.modes)
+ ? caps.modes.filter((mode) => mode === "speech" || mode === "singing")
+ : [];
+ if (modes.length === 0) return null;
+ return {
+ modes,
+ target: caps.target === "builtin" ? "builtin" : "audio",
+ builtin_voices: Array.isArray(caps.builtin_voices)
+ ? caps.builtin_voices.filter(
+ (voice) =>
+ isPlainObject(voice) &&
+ typeof voice.id === "string" &&
+ typeof voice.label === "string",
+ )
+ : [],
+ pitch: isPlainObject(caps.pitch) ? caps.pitch : {},
+ style: caps.style === true,
+ route_reloads: caps.route_reloads === true,
+ source_max_seconds:
+ typeof caps.source_max_seconds === "number" && caps.source_max_seconds > 0
+ ? caps.source_max_seconds
+ : 300,
};
}
@@ -132,9 +163,10 @@ export function collectToolRequest(
for (const panel of panels) {
const value = panelValue(panel, values, specs);
error ??= panel.validate?.(value, core, ctx) ?? null;
- const part = panel.toRequest(value);
+ const part = panel.toRequest(value, ctx);
if (part.options) patch.options = { ...patch.options, ...part.options };
if (part.inputs) patch.inputs = { ...patch.inputs, ...part.inputs };
+ if (part.convert) patch.convert = { ...patch.convert, ...part.convert };
for (const key of [
"instructions",
"language",
diff --git a/studio/frontend/src/features/audio/tools/types.ts b/studio/frontend/src/features/audio/tools/types.ts
index 55cb54f0c99..a59158d5703 100644
--- a/studio/frontend/src/features/audio/tools/types.ts
+++ b/studio/frontend/src/features/audio/tools/types.ts
@@ -1,9 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import type { AudioConvertCaps } from "@/features/chat/types/api";
import type { ComponentType } from "react";
import type { AudioOptionSpec, AudioOptionValues } from "../audio-options";
-import type { AudioSourceRef } from "../audio-run-request";
+import type {
+ AudioSourceRef,
+ ConvertMode,
+ ConvertStyle,
+} from "../audio-run-request";
import type { AudioWorkflowId } from "../workflows";
export type AudioReferenceTextMode = "required" | "optional" | "unused";
@@ -18,6 +23,8 @@ export interface AudioModelContext {
audioWorkflows?: readonly string[];
requiredInputs?: readonly string[];
referenceTextMode?: AudioReferenceTextMode | null;
+ convert?: AudioConvertCaps | null;
+ convertMode?: ConvertMode;
audioMusic?: boolean;
/** Ids of the saved voices, once the list has loaded. */
savedVoiceIds?: readonly string[] | null;
@@ -32,6 +39,8 @@ export interface AudioRunPatch {
text?: string;
speed?: number;
referenceTextMode?: "required" | "optional" | "hidden";
+ /** `route` mirrors options.route so the page can tell a run reloads the model. */
+ convert?: { style?: ConvertStyle; route?: string };
}
export interface CoreInputs {
@@ -69,7 +78,7 @@ export interface AudioToolPanel<V> {
appliesTo?: (ctx: AudioModelContext) => boolean;
initial: (specs: AudioOptionSpec[]) => V;
Component: ComponentType<AudioToolPanelProps<V>>;
- toRequest: (value: V) => AudioRunPatch;
+ toRequest: (value: V, ctx?: AudioModelContext) => AudioRunPatch;
validate?: (
value: V,
core: CoreInputs,
diff --git a/studio/frontend/src/features/audio/tour/steps.tsx b/studio/frontend/src/features/audio/tour/steps.tsx
index 3c15a96d8cd..5d3a1152424 100644
--- a/studio/frontend/src/features/audio/tour/steps.tsx
+++ b/studio/frontend/src/features/audio/tour/steps.tsx
@@ -11,10 +11,11 @@ const modeStep: TourStep = {
body: (
<>
Open the page title to switch pages. Speak turns text into speech, Clone
- speaks in the voice from a short recording, Music makes songs and sound
- effects, Separate splits a track into vocals and instruments, and
- Transcribe turns a recording into text. Each lists only the models that
- can do it, so the picker above follows the page.
+ speaks in the voice from a short recording, Edit changes words in a
+ recording, Convert makes a recording sound like another voice, Music
+ makes songs and sound effects, Separate splits a track into vocals and
+ instruments, and Transcribe turns a recording into text. Each lists only
+ the models that can do it, so the picker above follows the page.
</>
),
};
@@ -25,6 +26,8 @@ const MODEL_STEP_BODY: Record<AudioWorkflowId, string> = {
clone:
"Models that can speak in the voice of a short recording you give them.",
edit: "Models that can change words in a recording and keep the voice.",
+ convert:
+ "Voice conversion models. Loading one replaces the model in the main slot.",
music: "Music models. Loading one replaces the model in the main slot.",
separate:
"Source separation models. Loading one replaces the model in the main slot.",
@@ -142,6 +145,44 @@ export function buildAudioTourSteps({
];
}
+ if (workflow === "convert") {
+ return [
+ modeStep,
+ modelStep(workflow),
+ {
+ id: "source",
+ target: "audio-convert-source",
+ title: "Recording",
+ body: (
+ <>
+ Upload, record or pick the speech or singing to convert. Its words
+ and timing stay; only the voice changes.
+ </>
+ ),
+ },
+ {
+ id: "target",
+ target: "audio-convert-target",
+ title: "Target voice",
+ body: (
+ <>
+ The voice it should sound like: a short clip, a saved voice, or one
+ of the model's built-in voices.
+ </>
+ ),
+ },
+ {
+ ...outputStep,
+ body: (
+ <>
+ Results play here and stay in the history list. Switch between
+ Source and Converted to hear the difference at the same moment.
+ </>
+ ),
+ },
+ ];
+ }
+
if (workflow === "edit") {
return [
modeStep,
diff --git a/studio/frontend/src/features/audio/workflows.ts b/studio/frontend/src/features/audio/workflows.ts
index 20d35e5e1d7..a33b3ac874f 100644
--- a/studio/frontend/src/features/audio/workflows.ts
+++ b/studio/frontend/src/features/audio/workflows.ts
@@ -9,6 +9,7 @@ import {
QuillWrite01Icon,
SpeechToTextIcon,
SplitIcon,
+ UserSwitchIcon,
VoiceIdIcon,
} from "@hugeicons/core-free-icons";
import type { IconSvgElement } from "@hugeicons/react";
@@ -18,6 +19,7 @@ export type AudioWorkflowId =
| "speak"
| "clone"
| "edit"
+ | "convert"
| "music"
| "separate"
| "transcribe";
@@ -60,6 +62,15 @@ export const AUDIO_WORKFLOWS: ReadonlyArray<{
slot: "speak",
createTrain: false,
},
+ {
+ id: "convert",
+ label: "Convert",
+ heading: "Convert voice",
+ icon: UserSwitchIcon,
+ hint: "Make a recording sound like another voice",
+ slot: "speak",
+ createTrain: false,
+ },
{
id: "music",
label: "Music",
@@ -131,11 +142,12 @@ export function audioWorkflowForAudioType(
export function clipWorkflow(clip: {
workflow?: string | null;
audio_type?: string | null;
-}): "speak" | "clone" | "edit" | "music" | "separate" {
+}): "speak" | "clone" | "edit" | "convert" | "music" | "separate" {
if (
clip.workflow === "speak" ||
clip.workflow === "clone" ||
clip.workflow === "edit" ||
+ clip.workflow === "convert" ||
clip.workflow === "music" ||
clip.workflow === "separate"
) {
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 7ac978c7df5..f27918e2c34 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -382,6 +382,12 @@ export interface InferenceStatusResponse {
/** Audio page workflows the loaded model can run ("speak", "clone", "music", "transcribe"); empty when it is not an audio model. */
audio_workflows?: string[] | null;
audio_reference_text?: "required" | "optional" | "unused" | null;
+ audio_options_by_workflow?: Record<string, unknown> | null;
+ /** e.g. {"clone": "clon", "convert": "vc", "convert:singing": "svc"}; a task other than audio_server_task reloads. */
+ audio_workflow_tasks?: Record<string, string> | null;
+ audio_server_task?: string | null;
+ audio_convert_route?: string | null;
+ audio_convert?: AudioConvertCaps | null;
/** e.g. Maya1: "instruct" (its voice description). */
audio_required_inputs?: string[] | null;
/** Unknown-shaped on purpose: validated by parseMusicCapabilities. */
@@ -823,3 +829,15 @@ export interface OpenAIChatChunk {
prompt_target?: number;
};
}
+
+export interface AudioConvertCaps {
+ modes: ("speech" | "singing")[];
+ target: "audio" | "builtin";
+ builtin_voices: { id: string; label: string }[];
+ pitch: Partial<
+ Record<"speech" | "singing", { auto: boolean; shift_with_auto?: boolean }>
+ >;
+ style: boolean;
+ route_reloads: boolean;
+ source_max_seconds: number;
+}
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
index 61686a60b35..b6f3811f0a1 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
@@ -8,6 +8,7 @@
import { normalizeDenseQuantSchemes } from "../../../../lib/dense-quant-schemes.ts";
import {
AUDIO_CPP_MODELS,
+ type AudioCppModel,
type AudioCppTask,
audioCppDisplayName,
} from "../../../audio/audio-cpp-catalog.ts";
@@ -180,6 +181,19 @@ const AUDIO_GGUF_DESCRIPTIONS: Record<AudioCppTask, string> = {
sep: "Source separation",
};
+function audioGgufDescription(model: AudioCppModel): string {
+ const workflows = model.workflows;
+ if (!workflows || workflows.includes("speak")) {
+ return AUDIO_GGUF_DESCRIPTIONS[model.task];
+ }
+ const clones = workflows.includes("clone");
+ const converts = workflows.includes("convert");
+ if (clones && converts) return "Voice cloning and conversion";
+ if (converts) return "Voice conversion";
+ if (clones) return "Voice cloning";
+ return AUDIO_GGUF_DESCRIPTIONS[model.task];
+}
+
// Recommended audio GGUFs the backend runs on its audio runtime. They are plain GGUF rows, so the
// quant ladder, fit and downloads work as for any other; the backend routes them by GGUF header.
// Music lives in Speak like MiniMax Music 3, since both load into the main audio slot.
@@ -187,10 +201,7 @@ const audioGgufGroups = (tasks: readonly AudioCppTask[]): CatalogGroup[] =>
AUDIO_CPP_MODELS.filter((model) => tasks.includes(model.task)).map((model) => ({
canonicalId: model.id,
displayName: audioCppDisplayName(model.id),
- description:
- model.workflows?.includes("clone") && !model.workflows.includes("speak")
- ? "Voice cloning"
- : AUDIO_GGUF_DESCRIPTIONS[model.task],
+ description: audioGgufDescription(model),
scope: "audio",
task: model.task === "asr" ? "stt" : "tts",
artifacts: [gguf(model.id)],
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx
index c6fd495c7c4..0603490db8e 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pill-tabs.tsx
@@ -8,6 +8,7 @@ export interface PillTab {
value: string;
label: string;
icon?: ReactNode;
+ disabled?: boolean;
}
/** Segmented pill toggle reusing the Hub's .hub-tab-toggle styling (extended in hub.css to also
@@ -75,13 +76,17 @@ export function PillTabs({
// Roving tabindex: only the active tab is in the tab order; Arrow Left/Right move between tabs
// (WAI-ARIA tablist pattern). ArrowDown bubbles so the picker's "enter the list" handler runs.
tabIndex={value === tab.value ? 0 : -1}
- disabled={disabled}
+ disabled={disabled || tab.disabled}
onKeyDown={(e) => {
if (e.key !== "ArrowRight" && e.key !== "ArrowLeft") return;
e.preventDefault();
- const next =
- (index + (e.key === "ArrowRight" ? 1 : -1) + tabs.length) %
- tabs.length;
+ const step = e.key === "ArrowRight" ? 1 : -1;
+ let next = (index + step + tabs.length) % tabs.length;
+ // Skip disabled tabs; stay put when none is open.
+ while (tabs[next].disabled && next !== index) {
+ next = (next + step + tabs.length) % tabs.length;
+ }
+ if (next === index) return;
onValueChange(tabs[next].value);
e.currentTarget.parentElement
?.querySelectorAll<HTMLElement>('button[role="tab"]')
@@ -91,7 +96,7 @@ export function PillTabs({
onClick={() => onValueChange(tab.value)}
className={cn(
"relative z-10 inline-flex items-center justify-center gap-1.5 rounded-full transition-colors",
- disabled && "cursor-not-allowed opacity-50",
+ (disabled || tab.disabled) && "cursor-not-allowed opacity-50",
fit ? "min-w-0 shrink" : "min-w-0 flex-1",
compact
? "h-7 px-2.5 text-ui-11"
diff --git a/studio/frontend/tests/audio-convert-ab-compare.test.ts b/studio/frontend/tests/audio-convert-ab-compare.test.ts
new file mode 100644
index 00000000000..b12bee6ad32
--- /dev/null
+++ b/studio/frontend/tests/audio-convert-ab-compare.test.ts
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const { nextPlayback } = await import(
+ "../src/features/audio/components/convert-ab-compare-state.ts"
+);
+
+test("switching sides keeps the moment and play state, clamped to the new side", () => {
+ const cases: [number, boolean, number, number, boolean][] = [
+ // time, playing, next duration -> seek, resume
+ [4.2, true, 10, 4.2, true],
+ [4.2, false, 12, 4.2, false],
+ [3, true, 6, 3, true],
+ [2, true, Number.NaN, 2, true],
+ [Number.NaN, false, 5, 0, false],
+ [-1, true, 5, 0, true],
+ ];
+ for (const [time, playing, next, seek, resume] of cases) {
+ assert.deepEqual(nextPlayback({ time, playing, duration: 10 }, next), {
+ seek,
+ resume,
+ });
+ }
+ const past = nextPlayback({ time: 9, playing: true, duration: 10 }, 6);
+ assert.ok(past.seek < 6 && past.seek > 5.9, String(past.seek));
+ assert.equal(past.resume, false);
+});
+
+test("one player, swapped by a Tab-reachable toggle", () => {
+ const jsx = readSrc("features/audio/components/convert-ab-compare.tsx");
+ assert.equal(jsx.match(/<audio\b/g)?.length, 1);
+ assert.match(jsx, /<PillTabs\b/);
+ assert.doesNotMatch(jsx, /autoPlay/);
+ const tabs = readSrc(
+ "features/model-picker/components/model-selector/pill-tabs.tsx",
+ );
+ assert.match(tabs, /tabIndex=\{value === tab\.value \? 0 : -1\}/);
+ assert.match(tabs, /disabled=\{disabled \|\| tab\.disabled\}/);
+});
diff --git a/studio/frontend/tests/audio-convert-panels.test.ts b/studio/frontend/tests/audio-convert-panels.test.ts
new file mode 100644
index 00000000000..1269e2be797
--- /dev/null
+++ b/studio/frontend/tests/audio-convert-panels.test.ts
@@ -0,0 +1,274 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import type { AudioModelContext } from "../src/features/audio/tools/types.ts";
+import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const { audioModelContextFor, collectToolRequest, panelApplies, panelValue } =
+ await import("../src/features/audio/tools/select.ts");
+const {
+ CONVERT_PANEL_LOGIC,
+ SEED_VC_ENGINES,
+ chatterboxConvertLogic,
+ rvcLogic,
+ seedVcLogic,
+ vevo2StyleLogic,
+} = await import("../src/features/audio/tools/convert-panel-logic.ts");
+const { CLONE_PANEL_LOGIC } = await import(
+ "../src/features/audio/tools/panel-logic.ts"
+);
+
+const ctx = (
+ overrides: Partial<AudioModelContext> = {},
+): AudioModelContext => ({
+ audioType: "audiocpp_tts",
+ audioFamily: null,
+ musicGeneration: false,
+ cudaMusicGeneration: false,
+ musicNeedsDescription: false,
+ audioWorkflows: ["convert"],
+ requiredInputs: [],
+ referenceTextMode: null,
+ ...overrides,
+});
+
+const shown = (
+ family: string,
+ workflow: "speak" | "clone" | "convert" = "convert",
+) =>
+ CONVERT_PANEL_LOGIC.filter((panel) =>
+ panelApplies(panel, workflow, ctx({ audioFamily: family })),
+ ).map((panel) => panel.id);
+
+test("each converting family gets its one titled panel, only on Convert", () => {
+ assert.deepEqual(shown("seed_vc"), ["seed-vc"]);
+ assert.deepEqual(shown("rvc"), ["rvc"]);
+ assert.deepEqual(shown("chatterbox"), ["chatterbox-convert"]);
+ assert.deepEqual(shown("vevo2"), ["vevo2-style"]);
+ assert.deepEqual(shown("meanvc2"), []);
+ for (const family of ["seed_vc", "rvc", "chatterbox", "vevo2"]) {
+ assert.deepEqual(shown(family, "clone"), [], family);
+ assert.deepEqual(shown(family, "speak"), [], family);
+ }
+ const cloneOnConvert = CLONE_PANEL_LOGIC.filter((panel) =>
+ panelApplies(panel, "convert", ctx({ audioFamily: "chatterbox" })),
+ );
+ assert.deepEqual(cloneOnConvert, []);
+ assert.deepEqual(
+ CONVERT_PANEL_LOGIC.map((panel) => panel.title),
+ ["Seed-VC", "RVC", "Chatterbox", "Vevo2"],
+ );
+ const jsx = readSrc("features/audio/tools/convert-panels.tsx");
+ for (const label of [
+ "Sound like the target",
+ "Keep words clear",
+ "Index blend",
+ "Protect consonants",
+ "Volume envelope mix",
+ "Keep source style",
+ "Take target style",
+ "Engine",
+ ]) {
+ assert.ok(jsx.includes(label), label);
+ }
+ assert.doesNotMatch(jsx, /label="[a-z]+_[a-z_]+"/);
+});
+
+test("claims name the runtime's own options, so Advanced leaves them out", () => {
+ assert.deepEqual([...seedVcLogic.claims].sort(), [
+ "auto_f0_adjust",
+ "f0_condition",
+ "inference_guidance_scale",
+ "intelligibility_guidance_scale",
+ "length_adjust",
+ "num_inference_steps",
+ "route",
+ "semitone_shift",
+ "similarity_guidance_scale",
+ "voice_anonymization",
+ ]);
+ assert.deepEqual([...rvcLogic.claims].sort(), [
+ "retrieval_blend",
+ "rms_mix_rate",
+ "semitone_shift",
+ "unvoiced_protection",
+ "voice_id",
+ ]);
+ assert.deepEqual([...chatterboxConvertLogic.claims].sort(), [
+ "num_inference_steps",
+ "s3gen_cfg_rate",
+ ]);
+ assert.deepEqual(vevo2StyleLogic.claims, []);
+});
+
+test("Seed-VC sends only what the chosen engine reads", () => {
+ const value = seedVcLogic.initial([]);
+ assert.equal(value.engine, "v2_vc");
+ const speech = ctx({ audioFamily: "seed_vc", convertMode: "speech" });
+ const v2 = seedVcLogic.toRequest(value, speech);
+ assert.deepEqual(v2.convert, { route: "v2_vc" });
+ assert.deepEqual(Object.keys(v2.options ?? {}).sort(), [
+ "intelligibility_guidance_scale",
+ "length_adjust",
+ "num_inference_steps",
+ "route",
+ "similarity_guidance_scale",
+ "voice_anonymization",
+ ]);
+ for (const engine of ["v1_whisper_bigvgan_vc", "v1_xlsr_hift_vc"] as const) {
+ const v1 = seedVcLogic.toRequest({ ...value, engine }, speech);
+ assert.deepEqual(v1.convert, { route: engine });
+ assert.deepEqual(Object.keys(v1.options ?? {}).sort(), [
+ "inference_guidance_scale",
+ "length_adjust",
+ "num_inference_steps",
+ "route",
+ ]);
+ }
+});
+
+test("Seed-VC singing runs the singing engine, whatever engine speech had", () => {
+ const value = {
+ ...seedVcLogic.initial([]),
+ engine: "v1_xlsr_hift_vc" as const,
+ };
+ const singing = seedVcLogic.toRequest(
+ value,
+ ctx({ audioFamily: "seed_vc", convertMode: "singing" }),
+ );
+ assert.deepEqual(singing.convert, { route: "v1_svc" });
+ assert.equal(singing.options?.route, "v1_svc");
+ assert.equal("similarity_guidance_scale" in (singing.options ?? {}), false);
+ assert.equal("inference_guidance_scale" in (singing.options ?? {}), true);
+ assert.deepEqual(
+ SEED_VC_ENGINES.map((engine) => engine.label),
+ ["V2", "V1 Whisper", "V1 XLSR"],
+ );
+});
+
+test("Seed-VC clamps out-of-range values and takes defaults from the schema", () => {
+ const value = seedVcLogic.initial([
+ { name: "num_inference_steps", type: "int", default: 25 },
+ ]);
+ assert.equal(value.steps, 25);
+ const patch = seedVcLogic.toRequest(
+ { ...value, length: 9, steps: 0, similarity: -1 },
+ ctx({ convertMode: "speech" }),
+ );
+ assert.equal(patch.options?.length_adjust, 2);
+ assert.equal(patch.options?.num_inference_steps, 1);
+ assert.equal(patch.options?.similarity_guidance_scale, 0);
+});
+
+test("RVC and Chatterbox send their settings within range", () => {
+ assert.deepEqual(rvcLogic.toRequest({ blend: 2, protect: 0.9, rms: -1 }), {
+ options: { retrieval_blend: 1, unvoiced_protection: 0.5, rms_mix_rate: 0 },
+ });
+ assert.deepEqual(rvcLogic.initial([]), {
+ blend: 0,
+ protect: 0.33,
+ rms: 0.25,
+ });
+ assert.deepEqual(chatterboxConvertLogic.initial([]), {
+ guidance: 0.7,
+ steps: 10,
+ });
+ assert.deepEqual(
+ chatterboxConvertLogic.toRequest({ guidance: 0.5, steps: 12.4 }),
+ {
+ options: { s3gen_cfg_rate: 0.5, num_inference_steps: 12 },
+ },
+ );
+ // The option allows 3; Chatterbox's own default is 0.7.
+ assert.equal(
+ chatterboxConvertLogic.toRequest({ guidance: 2.8, steps: 10 }).options
+ ?.s3gen_cfg_rate,
+ 2.8,
+ );
+ assert.equal(
+ chatterboxConvertLogic.toRequest({ guidance: 9, steps: 10 }).options
+ ?.s3gen_cfg_rate,
+ 3,
+ );
+});
+
+test("Vevo2's style reaches the convert settings; singing always keeps the source's", () => {
+ const speech = ctx({ audioFamily: "vevo2", convertMode: "speech" });
+ const singing = ctx({ audioFamily: "vevo2", convertMode: "singing" });
+ for (const [style, mode, sent] of [
+ ["target", speech, "target"],
+ ["source", speech, "source"],
+ ["target", singing, "source"],
+ ] as const) {
+ assert.deepEqual(vevo2StyleLogic.toRequest({ style }, mode), {
+ convert: { style: sent },
+ });
+ }
+});
+
+test("collectToolRequest merges each panel's convert part with its options", () => {
+ const panels = [seedVcLogic, vevo2StyleLogic].map((logic) => ({
+ ...logic,
+ Component: () => null,
+ }));
+ const { patch, error } = collectToolRequest(
+ panels,
+ { "vevo2-style": { style: "target" } },
+ { text: "" },
+ ctx({ convertMode: "speech" }),
+ );
+ assert.equal(error, null);
+ assert.deepEqual(patch.convert, { route: "v2_vc", style: "target" });
+ assert.equal(patch.options?.route, "v2_vc");
+ assert.equal(
+ panelValue(seedVcLogic, { "seed-vc": { engine: "v1_xlsr_hift_vc" } }, [])
+ .steps,
+ 30,
+ );
+});
+
+test("the tool context reads the Convert caps from the status", () => {
+ const context = audioModelContextFor(
+ {
+ audio_family: "vevo2",
+ audio_workflows: ["clone", "convert"],
+ audio_convert: {
+ modes: ["speech", "singing"],
+ target: "audio",
+ builtin_voices: [],
+ pitch: { speech: { auto: true }, singing: { auto: true } },
+ style: true,
+ route_reloads: false,
+ source_max_seconds: 300,
+ },
+ },
+ {
+ musicGeneration: false,
+ cudaMusicGeneration: false,
+ musicNeedsDescription: false,
+ },
+ );
+ assert.deepEqual(context.convert?.modes, ["speech", "singing"]);
+ assert.equal(context.convert?.style, true);
+ const none = audioModelContextFor(
+ { audio_family: "kokoro", audio_convert: null },
+ {
+ musicGeneration: false,
+ cudaMusicGeneration: false,
+ musicNeedsDescription: false,
+ },
+ );
+ assert.equal(none.convert, null);
+});
+
+test("the registry lists the Convert panels after Clone's", () => {
+ const registry = readSrc("features/audio/tools/registry.tsx");
+ assert.match(
+ registry,
+ /\.\.\.CLONE_TOOL_PANELS,[\s\S]*?\.\.\.CONVERT_TOOL_PANELS,/,
+ );
+});
diff --git a/studio/frontend/tests/audio-convert-policy.test.ts b/studio/frontend/tests/audio-convert-policy.test.ts
new file mode 100644
index 00000000000..b1220ea6ee1
--- /dev/null
+++ b/studio/frontend/tests/audio-convert-policy.test.ts
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import type { ConvertBlockerInput } from "../src/features/audio/convert-policy.ts";
+import type { AudioConvertCaps } from "../src/features/chat/types/api.ts";
+import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const {
+ CONVERT_BUILTIN_MISSING,
+ CONVERT_MODEL_ORDER,
+ CONVERT_SINGING_UNSUPPORTED,
+ CONVERT_INPUTS_MISSING,
+ CONVERT_SOURCE_MISSING,
+ CONVERT_SOURCE_TEXT_MISSING,
+ CONVERT_TARGET_MISSING,
+ convertBlocker,
+ convertCaps,
+ convertPitchSupport,
+ convertServerTask,
+ convertSwitchNotice,
+} = await import("../src/features/audio/convert-policy.ts");
+const { AUDIO_CPP_REPO, audioCppModelFor, audioCppWorkflowsFor } = await import(
+ "../src/features/audio/audio-cpp-catalog.ts"
+);
+
+const base: AudioConvertCaps = {
+ modes: ["speech"],
+ target: "audio",
+ builtin_voices: [],
+ pitch: {},
+ style: false,
+ route_reloads: false,
+ source_max_seconds: 300,
+};
+
+const CAPS: Record<string, AudioConvertCaps> = {
+ rvc: {
+ ...base,
+ target: "builtin",
+ builtin_voices: [
+ { id: "default", label: "Default" },
+ { id: "manthos", label: "Manthos" },
+ ],
+ pitch: { speech: { auto: false } },
+ },
+ seed_vc: {
+ ...base,
+ modes: ["speech", "singing"],
+ pitch: { singing: { auto: true, shift_with_auto: true } },
+ route_reloads: true,
+ },
+ meanvc2: base,
+ chatterbox: base,
+ vevo2: {
+ ...base,
+ modes: ["speech", "singing"],
+ pitch: {
+ speech: { auto: true, shift_with_auto: false },
+ singing: { auto: true, shift_with_auto: false },
+ },
+ style: true,
+ },
+};
+
+const recording = {
+ kind: "clip" as const,
+ id: "c1",
+ name: "Take 1",
+ durationS: 8,
+};
+const voice = {
+ kind: "voice" as const,
+ id: "v1",
+ name: "Narrator",
+ durationS: 6,
+};
+
+const ready: ConvertBlockerInput = {
+ source: recording,
+ sourceBusy: false,
+ sourceExpired: false,
+ sourceError: null,
+ target: voice,
+ targetBusy: false,
+ targetExpired: false,
+ targetError: null,
+ builtinVoice: "default",
+ caps: CAPS.seed_vc,
+ mode: "speech" as const,
+ style: "source" as const,
+ sourceText: "",
+ panelError: null,
+};
+
+test("the picker order names only seeded convert models", () => {
+ for (const name of CONVERT_MODEL_ORDER) {
+ const model = audioCppModelFor(`${AUDIO_CPP_REPO}/${name}`);
+ assert.ok(model, name);
+ assert.ok(audioCppWorkflowsFor(model).includes("convert"), name);
+ }
+ assert.equal(CONVERT_MODEL_ORDER[0], "SeedVC-MLX-GGUF");
+});
+
+test("caps come from the tool context; a model without modes cannot convert", () => {
+ assert.equal(convertCaps({ convert: null }), null);
+ assert.equal(convertCaps({}), null);
+ assert.equal(convertCaps({ convert: { ...base, modes: [] } }), null);
+ assert.equal(convertCaps({ convert: CAPS.rvc }), CAPS.rvc);
+});
+
+test("blockers come in rail order and explain themselves", () => {
+ assert.equal(convertBlocker(ready), null);
+ const cases: [Partial<ConvertBlockerInput>, string, RegExp | string][] = [
+ [
+ { source: null, target: null, panelError: "x" },
+ "source",
+ CONVERT_INPUTS_MISSING,
+ ],
+ [{ source: null }, "source", CONVERT_SOURCE_MISSING],
+ [{ target: null, panelError: "x" }, "target", CONVERT_TARGET_MISSING],
+ [
+ { caps: CAPS.vevo2, style: "target", sourceText: " ", panelError: "x" },
+ "source-text",
+ CONVERT_SOURCE_TEXT_MISSING,
+ ],
+ [{ panelError: "Pick one." }, "panel", "Pick one."],
+ [
+ { caps: CAPS.meanvc2, mode: "singing" },
+ "mode",
+ CONVERT_SINGING_UNSUPPORTED,
+ ],
+ [
+ { sourceExpired: true, targetExpired: true },
+ "source-expired",
+ /recording and target voice are no longer on the server/,
+ ],
+ [{ sourceExpired: true }, "source-expired", /recording is no longer/],
+ [{ sourceBusy: true }, "source", /Waiting for the recording/],
+ [{ source: null, sourceError: "Too long." }, "source-error", "Too long."],
+ [{ targetExpired: true }, "target-expired", /target voice is no longer/],
+ [{ targetBusy: true }, "target", /Waiting for the target voice/],
+ [{ target: null, targetError: "Bad file." }, "target-error", "Bad file."],
+ ];
+ for (const [patch, kind, reason] of cases) {
+ const blocker = convertBlocker({ ...ready, ...patch });
+ assert.equal(blocker?.kind, kind, JSON.stringify(patch));
+ if (typeof reason === "string") assert.equal(blocker?.reason, reason);
+ else assert.match(blocker?.reason ?? "", reason);
+ }
+});
+
+test("RVC needs a built-in voice, not a target recording", () => {
+ const rvc = { ...ready, caps: CAPS.rvc, target: null };
+ assert.equal(convertBlocker(rvc), null);
+ assert.equal(convertBlocker({ ...rvc, targetExpired: true }), null);
+ assert.deepEqual(convertBlocker({ ...rvc, builtinVoice: "" }), {
+ kind: "target",
+ reason: CONVERT_BUILTIN_MISSING,
+ });
+ assert.deepEqual(convertBlocker({ ...rvc, builtinVoice: "gone" }), {
+ kind: "target",
+ reason: CONVERT_BUILTIN_MISSING,
+ });
+});
+
+test("Take target style needs the transcript only in Speech", () => {
+ const vevo = { ...ready, caps: CAPS.vevo2, style: "target" as const };
+ assert.equal(convertBlocker(vevo)?.kind, "source-text");
+ assert.equal(convertBlocker({ ...vevo, sourceText: "Hello." }), null);
+ assert.equal(convertBlocker({ ...vevo, mode: "singing" }), null);
+ assert.equal(convertBlocker({ ...ready, style: "target" }), null);
+});
+
+test("pitch shows per family, mode and style; only Seed-VC adds a shift to Auto", () => {
+ const cases: [
+ AudioConvertCaps | null,
+ "speech" | "singing",
+ "source" | "target",
+ boolean,
+ boolean,
+ boolean,
+ ][] = [
+ [CAPS.rvc, "speech", "source", true, false, false],
+ [CAPS.rvc, "singing", "source", false, false, false],
+ [CAPS.seed_vc, "speech", "source", false, false, false],
+ [CAPS.seed_vc, "singing", "source", true, true, true],
+ [CAPS.vevo2, "speech", "source", true, true, false],
+ [CAPS.vevo2, "speech", "target", false, false, false],
+ [CAPS.vevo2, "singing", "target", true, true, false],
+ [CAPS.meanvc2, "speech", "source", false, false, false],
+ [CAPS.chatterbox, "speech", "source", false, false, false],
+ [null, "speech", "source", false, false, false],
+ ];
+ for (const [caps, mode, style, show, auto, shiftWithAuto] of cases) {
+ assert.deepEqual(convertPitchSupport(caps, mode, style), {
+ show,
+ auto,
+ shiftWithAuto,
+ });
+ }
+});
+
+test("the server task per mode comes from the workflow tasks", () => {
+ const tasks = { clone: "tts", convert: "vc", "convert:singing": "svc" };
+ assert.equal(convertServerTask(CAPS.vevo2, tasks, "speech"), "vc");
+ assert.equal(convertServerTask(CAPS.vevo2, tasks, "singing"), "svc");
+ assert.equal(
+ convertServerTask(CAPS.vevo2, { "convert:speech": "vc" }, "speech"),
+ "vc",
+ );
+ assert.equal(convertServerTask(CAPS.meanvc2, tasks, "singing"), null);
+ assert.equal(convertServerTask(null, tasks, "speech"), null);
+ assert.equal(convertServerTask(CAPS.vevo2, null, "speech"), null);
+});
+
+test("a run that reloads the model says so before and during; a repeat run does not", () => {
+ const notice = (
+ modelName: string,
+ loadedTask: string | null,
+ nextTask: string,
+ routeChange: boolean,
+ family?: string,
+ ) =>
+ convertSwitchNotice({
+ modelName,
+ loadedTask,
+ nextTask,
+ routeChange,
+ family,
+ });
+ assert.deepEqual(notice("Chatterbox", "clon", "vc", false, "chatterbox"), {
+ before: "Reloads Chatterbox for Convert, about 2 s.",
+ during: "Switching Chatterbox to Convert…",
+ });
+ assert.deepEqual(notice("Vevo2", "vc", "svc", false, "vevo2"), {
+ before: "Reloads Vevo2 for singing, about 6 s.",
+ during: "Switching Vevo2 to singing…",
+ });
+ assert.deepEqual(notice("Seed-VC", "vc", "vc", true, "seed_vc"), {
+ before: "Reloads Seed-VC with the new engine, about 4–8 s.",
+ during: "Switching Seed-VC to the new engine…",
+ });
+ assert.equal(notice("Seed-VC", "vc", "vc", false, "seed_vc"), null);
+ assert.equal(notice("X", null, "vc", false), null);
+ assert.equal(
+ notice("Custom", "tts", "vc", false)?.before,
+ "Reloads Custom for Convert.",
+ );
+});
+
+test("a transcript lands only on the recording it was made from", () => {
+ const hook = readSrc("features/audio/hooks/use-convert-generation.ts");
+ assert.match(
+ hook,
+ /onText: \(next, recording\) => \{\s*const store = useAudioConvertStore\.getState\(\);\s*if \(\s*store\.source\?\.kind === recording\.kind &&\s*store\.source\.id === recording\.id\s*\) \{\s*store\.setSourceText\(next\);/,
+ );
+});
+
+test("Use again restores an upload or history clip target, not only a saved voice", () => {
+ const page = readSrc("features/audio/audio-page.tsx");
+ assert.match(
+ page,
+ /clip\.target_clip_id\s*\?\s*\{ kind: "clip" as const, id: clip\.target_clip_id \}/,
+ );
+ assert.match(
+ page,
+ /clip\.target_input_id\s*\?\s*\{ kind: "input" as const, id: clip\.target_input_id \}/,
+ );
+});
+
+test("Convert transcribes its source as far as it converts", () => {
+ const hook = readSrc("features/audio/hooks/use-convert-generation.ts");
+ assert.match(hook, /purpose: "convert",/);
+ const transcribe = readSrc(
+ "features/audio/hooks/use-reference-transcribe.ts",
+ );
+ assert.match(transcribe, /device: voice\.sttDevice,\s*purpose,/);
+});
+
+test("only the expired-upload 404 expires uploads, and a re-upload clears it", () => {
+ const hook = readSrc("features/audio/hooks/use-convert-generation.ts");
+ assert.match(
+ hook,
+ /error\.status === 404 &&\s*error\.message === REFERENCE_EXPIRED_MESSAGE/,
+ );
+ assert.match(
+ hook,
+ /for \(const id of \[sourceId, targetId\]\) \{\s*if \(id\) \{\s*next\.delete\(id\);/,
+ );
+});
+
+// Chatterbox restarts under clon for Clone and vc for Convert; a stale task hides the reload.
+const CLONE_REFRESHES_AFTER_RUN =
+ /await showRunResult\(\{[\s\S]*?\}\);\s*(?:\/\/[^\n]*\n\s*)*await refreshStatus\(\);\s*\} catch \(error\)/;
+const CLONE_REFRESHES_AFTER_STOP =
+ /if \(!expired\) toast\.error\(message\);\s*\}\s*(?:\/\/[^\n]*\n\s*)*await refreshStatus\(\);\s*\} finally/;
+const CONVERT_REFRESHES_AFTER_STOPPED_SWITCH =
+ /\} else if \(switchNotice\) \{\s*(?:\/\/[^\n]*\n\s*)*await refreshStatus\(\);\s*\}\s*\} finally/;
+const CONVERT_REFRESHES_ON_ARRIVAL =
+ /useEffect\(\(\) => \{\s*if \(active && ttsWorkflow === "convert" && initialReadySent\.current\) \{\s*void refreshStatus\(\);\s*\}\s*\}, \[active, ttsWorkflow, refreshStatus\]\);/;
+const CONVERTS_ONLY =
+ /const loadedConvertsOnly =\s*!!status\?\.audio_workflows\?\.includes\("convert"\) &&\s*!status\.audio_workflows\.includes\("clone"\);/;
+const SPEAK_BLOCKER_OPENS_CONVERT =
+ /ttsWorkflow === "speak" && loadedConvertsOnly\s*\?\s*\{\s*reason: "The loaded model converts recordings\.",[\s\S]*?label: "open Convert",\s*onClick: \(\) => transitionWorkflow\("convert"\)/;
+const SPEAK_STATUS_LINE_CONVERTS =
+ /ttsWorkflow === "speak" && loadedConvertsOnly\s*\?\s*"The loaded model converts recordings\. Pick a speech model\."/;
+
+test("Convert's reload notice reads a status refreshed after a Clone run, a stopped Clone or a stopped switch", () => {
+ const clone = readSrc("features/audio/hooks/use-clone-generation.ts");
+ assert.match(clone, CLONE_REFRESHES_AFTER_RUN);
+ assert.match(clone, CLONE_REFRESHES_AFTER_STOP);
+ const convert = readSrc("features/audio/hooks/use-convert-generation.ts");
+ assert.match(convert, CONVERT_REFRESHES_AFTER_STOPPED_SWITCH);
+ // A stopped Clone run can restart audio.cpp after Stop returns; arriving on Convert reads it again.
+ const page = readSrc("features/audio/audio-page.tsx");
+ assert.match(page, CONVERT_REFRESHES_ON_ARRIVAL);
+});
+
+test("Speak sends a convert-only model to Convert, not to Clone", () => {
+ const page = readSrc("features/audio/audio-page.tsx");
+ assert.match(page, CONVERTS_ONLY);
+ assert.match(page, SPEAK_BLOCKER_OPENS_CONVERT);
+ assert.match(page, SPEAK_STATUS_LINE_CONVERTS);
+});
+
+test("the Convert source card states Convert's own length cap", () => {
+ const page = readSrc("features/audio/pages/convert-page.tsx");
+ assert.match(
+ page,
+ /usesFirstSeconds=\{caps\?\.source_max_seconds \?\? 300\}/,
+ );
+ assert.match(
+ page,
+ /maxRecordSeconds=\{caps\?\.source_max_seconds \?\? 300\}/,
+ );
+ const card = readSrc("features/audio/components/audio-source-input.tsx");
+ // Five minutes reads as minutes, not 300 s.
+ assert.match(card, /`Uses the first \$\{usesFirstSeconds \/ 60\} min\.`/);
+});
+
+test("Use again re-uploads a conversion's kept source instead of its expiring upload id", () => {
+ const page = readSrc("features/audio/audio-page.tsx");
+ assert.match(page, /if \(!clip\.source_clip_id && clip\.source_saved\) \{/);
+ assert.match(
+ page,
+ /\/source\/file`,\s*\)\s*\.then\(\(blob\) => uploadAudioInput\(blob, name\)\)/,
+ );
+});
+
+const SEND_TO_HAS_CONVERT = /convert: \(\) => handleSendToConvert\(clip\),/;
+const SEND_TO_CLONE_KEEPS_LABELS_OUT =
+ /adoptReference\(clipReference\(\{ \.\.\.clip, workflow: clipWorkflow\(clip\) \}\)\)/;
+const WORKSPACE_FOCUSES_A_FRESH_CUSTOM_PLAYER =
+ /renderPlayer\(\s*selectedClip,\s*selectedClipSrc,\s*selectedClip\.id === freshClipId \? focusFreshClip : undefined,?\s*\)/;
+
+test("Send to offers Convert, and a Convert clip sent to Clone brings no label as its transcript", () => {
+ const page = readSrc("features/audio/audio-page.tsx");
+ assert.match(page, SEND_TO_HAS_CONVERT);
+ assert.match(page, SEND_TO_CLONE_KEEPS_LABELS_OUT);
+});
+
+test("a fresh Convert clip focuses its player, not the Source tab", () => {
+ const workspace = readSrc("features/audio/pages/tts-workspace.tsx");
+ assert.match(workspace, WORKSPACE_FOCUSES_A_FRESH_CUSTOM_PLAYER);
+ const page = readSrc("features/audio/pages/convert-page.tsx");
+ assert.match(page, /playerRef=\{focusRef\}/);
+});
diff --git a/studio/frontend/tests/audio-convert-store.test.ts b/studio/frontend/tests/audio-convert-store.test.ts
new file mode 100644
index 00000000000..a58303c1175
--- /dev/null
+++ b/studio/frontend/tests/audio-convert-store.test.ts
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ installLocalStorageFake,
+ registerBundlerResolver,
+} from "./helpers/kit.ts";
+
+const { store } = installLocalStorageFake();
+registerBundlerResolver();
+
+const SAVED = {
+ state: {
+ source: { kind: "clip", id: "c1", name: "Take 1", durationS: 8 },
+ target: { kind: "voice", id: "v1", name: "Narrator", durationS: 6 },
+ builtinVoice: "manthos",
+ mode: "singing",
+ pitchAuto: false,
+ pitch: 3,
+ sourceText: "La la la.",
+ compareSide: "source",
+ },
+ version: 1,
+};
+store.set("unsloth_audio_convert_v1", JSON.stringify(SAVED));
+
+const { AUDIO_CONVERT_STORAGE_KEY, useAudioConvertStore } = await import(
+ "../src/features/audio/stores/audio-convert-store.ts"
+);
+
+test("the draft comes back after a reload", () => {
+ assert.equal(AUDIO_CONVERT_STORAGE_KEY, "unsloth_audio_convert_v1");
+ const state = useAudioConvertStore.getState();
+ for (const [key, value] of Object.entries(SAVED.state)) {
+ assert.deepEqual(state[key as keyof typeof SAVED.state], value, key);
+ }
+});
+
+test("every draft field is written through, and no action is", () => {
+ const state = useAudioConvertStore.getState();
+ state.setSource({
+ kind: "input",
+ id: "abc",
+ name: "take.webm",
+ durationS: 4,
+ expiresAt: "2026-10-04T00:00:00Z",
+ });
+ state.setTarget(null);
+ state.setBuiltinVoice("fraise");
+ state.setMode("speech");
+ state.setPitchAuto(true);
+ state.setPitch(-5);
+ state.setSourceText("Hello.");
+ state.setCompareSide("converted");
+ const saved = JSON.parse(store.get(AUDIO_CONVERT_STORAGE_KEY) ?? "{}");
+ assert.equal(saved.version, 1);
+ assert.deepEqual(saved.state, {
+ source: {
+ kind: "input",
+ id: "abc",
+ name: "take.webm",
+ durationS: 4,
+ expiresAt: "2026-10-04T00:00:00Z",
+ },
+ target: null,
+ builtinVoice: "fraise",
+ mode: "speech",
+ pitchAuto: true,
+ pitch: -5,
+ sourceText: "Hello.",
+ compareSide: "converted",
+ });
+});
+
+test("pitch stays a whole number of semitones within two octaves", () => {
+ const state = useAudioConvertStore.getState();
+ state.setPitch(30);
+ assert.equal(useAudioConvertStore.getState().pitch, 24);
+ state.setPitch(-2.6);
+ assert.equal(useAudioConvertStore.getState().pitch, -3);
+ state.setPitch(Number.NaN);
+ assert.equal(useAudioConvertStore.getState().pitch, 0);
+});
+
+test("a new recording drops the old transcript or brings its own", () => {
+ const s = useAudioConvertStore.getState();
+ s.setSource({ kind: "clip", id: "a", name: "A", durationS: 2 });
+ s.setSourceText("words of A");
+ s.setSource({ kind: "clip", id: "a", name: "A again", durationS: 2 });
+ assert.equal(useAudioConvertStore.getState().sourceText, "words of A");
+ s.setSource({ kind: "input", id: "b", name: "B", durationS: 3 });
+ assert.equal(useAudioConvertStore.getState().sourceText, "");
+ s.setSource({
+ kind: "voice",
+ id: "v",
+ name: "V",
+ durationS: 4,
+ transcript: " words of V ",
+ });
+ assert.equal(useAudioConvertStore.getState().sourceText, "words of V");
+ s.setSource(null);
+ assert.equal(useAudioConvertStore.getState().sourceText, "");
+});
diff --git a/studio/frontend/tests/audio-cpp-catalog.test.ts b/studio/frontend/tests/audio-cpp-catalog.test.ts
index 2523a8a420b..3ac0958b387 100644
--- a/studio/frontend/tests/audio-cpp-catalog.test.ts
+++ b/studio/frontend/tests/audio-cpp-catalog.test.ts
@@ -20,6 +20,7 @@ import {
audioCppDisplayName,
audioCppModelFor,
audioCppSizeLabel,
+ audioCppWorkflowsFor,
isAudioCppFolderId,
} from "../src/features/audio/audio-cpp-catalog.ts";
import {
@@ -161,6 +162,28 @@ test("recommended models are plain GGUF Audio rows named as on the Hub", () => {
}
});
+test("voice conversion models are seeded with the pages they run on", () => {
+ for (const [name, workflows, description] of [
+ ["RVC-GGUF", ["convert"], "Voice conversion"],
+ ["SeedVC-MLX-GGUF", ["convert"], "Voice conversion"],
+ ["MeanVC2-GGUF", ["convert"], "Voice conversion"],
+ ["Chatterbox-GGUF", ["clone", "convert"], "Voice cloning and conversion"],
+ ["Vevo2-GGUF", ["clone", "edit", "convert"], "Voice cloning and conversion"],
+ ["IndexTTS2-GGUF", ["clone"], "Voice cloning"],
+ ] as const) {
+ const id = `${AUDIO_CPP_REPO}/${name}`;
+ const model = audioCppModelFor(id);
+ assert.ok(model, name);
+ assert.equal(model.task, "tts", name);
+ assert.deepEqual(audioCppWorkflowsFor(model), workflows, name);
+ assert.equal(
+ groupForRepoId(id, AUDIO_CATALOG)?.description,
+ description,
+ name,
+ );
+ }
+});
+
test("GGUF heuristics treat these rows like any llama.cpp GGUF", () => {
for (const model of AUDIO_CPP_MODELS) {
assert.equal(isGgufId(model.id), true, model.id);
@@ -525,6 +548,8 @@ test("the capability line names GGUF audio and music, never the runtime's intern
assert.equal(audioCapabilityLine("tts", "audiocpp_tts"), "Text-to-speech · GGUF");
assert.equal(audioCapabilityLine("music", "audiocpp_music"), "Music generation · GGUF");
assert.equal(audioCapabilityLine("clone", "audiocpp_tts"), "Voice cloning · GGUF");
+ assert.equal(audioCapabilityLine("convert", "audiocpp_tts"), "Voice conversion · GGUF");
+ assert.equal(audioCapabilityLine("convert"), "Voice conversion");
assert.equal(audioCapabilityLine("tts", "higgs_tts2"), "Text-to-speech · higgs_tts2");
assert.equal(audioCapabilityLine("stt", "ready"), "Speech-to-text · ready");
});
diff --git a/studio/frontend/tests/audio-generation-stop.test.ts b/studio/frontend/tests/audio-generation-stop.test.ts
index 2bec80bdc76..bd94d71c2c4 100644
--- a/studio/frontend/tests/audio-generation-stop.test.ts
+++ b/studio/frontend/tests/audio-generation-stop.test.ts
@@ -8,6 +8,30 @@ import { readAudioWorkspaceSource } from "./helpers/audio-workspace.ts";
const source = readAudioWorkspaceSource();
+const { audioGenerationPresentation } = await import(
+ "../src/features/audio/audio-page-policy.ts"
+);
+
+test("a run that reloads the model first can be stopped while it switches", () => {
+ assert.deepEqual(audioGenerationPresentation("switching"), {
+ status: "Switching model…",
+ actionLabel: "Stop",
+ canStop: true,
+ });
+ assert.deepEqual(
+ audioGenerationPresentation("switching", "Switching Chatterbox to Convert…"),
+ {
+ status: "Switching Chatterbox to Convert…",
+ actionLabel: "Stop",
+ canStop: true,
+ },
+ );
+ assert.equal(
+ audioGenerationPresentation("generating", "Switching…")?.status,
+ "Generating audio…",
+ );
+});
+
test("generation exposes Stop only while the request controller can abort", () => {
assert.match(
source,
diff --git a/studio/frontend/tests/audio-handoff-intent.test.ts b/studio/frontend/tests/audio-handoff-intent.test.ts
index 8eb7b12e3b6..2ce7d1442f6 100644
--- a/studio/frontend/tests/audio-handoff-intent.test.ts
+++ b/studio/frontend/tests/audio-handoff-intent.test.ts
@@ -24,7 +24,7 @@ test("unknown tasks, workflows and audio types are dropped", () => {
assert.deepEqual(
validateAudioSearch({
task: "text-to-image",
- workflow: "convert",
+ workflow: "remix",
audioType: "not-a-type",
loadId: " ",
}),
diff --git a/studio/frontend/tests/audio-picker-filter.test.ts b/studio/frontend/tests/audio-picker-filter.test.ts
index b70889f45f7..0f271e54fdf 100644
--- a/studio/frontend/tests/audio-picker-filter.test.ts
+++ b/studio/frontend/tests/audio-picker-filter.test.ts
@@ -18,8 +18,15 @@ const { audioCppModelSpeaks } = await import(
"../src/features/audio/audio-cpp-catalog.ts"
);
-type Workflow = "speak" | "clone" | "edit" | "music" | "transcribe";
-const WORKFLOWS: Workflow[] = ["speak", "clone", "edit", "music", "transcribe"];
+type Workflow = "speak" | "clone" | "edit" | "convert" | "music" | "transcribe";
+const WORKFLOWS: Workflow[] = [
+ "speak",
+ "clone",
+ "edit",
+ "convert",
+ "music",
+ "transcribe",
+];
type Row = Parameters<typeof audioRowMatchesWorkflow>[0];
@@ -79,6 +86,40 @@ test("a clone-only speech model lists only on Clone", () => {
);
});
+test("voice conversion models list on Convert only; Chatterbox and Vevo2 on Clone and Convert", () => {
+ for (const [name, workflows] of [
+ ["RVC-GGUF", ["convert"]],
+ ["SeedVC-MLX-GGUF", ["convert"]],
+ ["MeanVC2-GGUF", ["convert"]],
+ ["Chatterbox-GGUF", ["clone", "convert"]],
+ ["Vevo2-GGUF", ["clone", "edit", "convert"]],
+ ] as const) {
+ const id = `audio-cpp/audio.cpp-gguf/${name}`;
+ assert.deepEqual(
+ workflowsFor({ id, task: "text-to-speech", audioType: "audiocpp_tts" }),
+ workflows,
+ name,
+ );
+ assert.equal(audioCppModelSpeaks(id), false, name);
+ }
+ assert.deepEqual(
+ workflowsFor({
+ id: "audio-cpp/audio.cpp-gguf/RVC-GGUF",
+ task: "text-to-speech",
+ audioWorkflows: ["convert"],
+ }),
+ ["convert"],
+ );
+ assert.deepEqual(
+ workflowsFor({
+ id: "audio-cpp/audio.cpp-gguf/Kokoro-82M-GGUF",
+ task: "text-to-speech",
+ audioType: "audiocpp_tts",
+ }),
+ ["speak"],
+ );
+});
+
test("Speak's catalog rows leave out the clone-only models", () => {
const speakIds = audioModelsForTask("tts")
.map((model) => model.id)
@@ -182,7 +223,7 @@ test("a speech editing model lists on Edit beside its other pages", () => {
// DotTTS's other packages do not edit.
["DotTTS-MF-GGUF", { audioWorkflows: ["speak"] }, ["speak"]],
// Without the backend's list, the catalog seed decides.
- ["Vevo2-GGUF", { audioType: "audiocpp_tts" }, ["clone", "edit"]],
+ ["Vevo2-GGUF", { audioType: "audiocpp_tts" }, ["clone", "edit", "convert"]],
];
for (const [folder, row, workflows] of rows) {
const id = `audio-cpp/audio.cpp-gguf/${folder}`;
diff --git a/studio/frontend/tests/audio-run-api.test.ts b/studio/frontend/tests/audio-run-api.test.ts
index b27501f11ef..932d623bc62 100644
--- a/studio/frontend/tests/audio-run-api.test.ts
+++ b/studio/frontend/tests/audio-run-api.test.ts
@@ -87,6 +87,92 @@ test("whatever a caller spreads in, no path or bytes reach the body", () => {
assert.doesNotMatch(JSON.stringify(body), PATHLIKE);
});
+test("a convert run sends the source, the target and the convert settings, and no text", () => {
+ const body = buildAudioRunBody({
+ workflow: "convert",
+ inputs: {
+ source: { clip_id: "c1" },
+ target: { voice_id: "v1" },
+ source_text: " Hello there. ",
+ },
+ convert: { mode: "speech", pitch: 3.4, pitch_auto: false, style: "target" },
+ options: { route: "v2_vc", length_adjust: 1.1, bad: Number.NaN },
+ seed: 7,
+ });
+ assert.deepEqual(body, {
+ workflow: "convert",
+ inputs: {
+ source: { clip_id: "c1" },
+ target: { voice_id: "v1" },
+ source_text: "Hello there.",
+ },
+ convert: { mode: "speech", pitch: 3, pitch_auto: false, style: "target" },
+ options: { route: "v2_vc", length_adjust: 1.1 },
+ seed: 7,
+ });
+ assert.equal("text" in body, false);
+});
+
+test("a convert run to a built-in voice has no target, and Auto keeps its extra shift", () => {
+ const body = buildAudioRunBody({
+ workflow: "convert",
+ inputs: { source: { input_id: "i1" }, target: null, source_text: " " },
+ convert: {
+ mode: "singing",
+ pitch: 5,
+ pitch_auto: true,
+ voice: " manthos ",
+ },
+ });
+ assert.deepEqual(body, {
+ workflow: "convert",
+ inputs: { source: { input_id: "i1" } },
+ convert: {
+ mode: "singing",
+ pitch: 5,
+ pitch_auto: true,
+ voice: "manthos",
+ },
+ });
+ const low = buildAudioRunBody({
+ workflow: "convert",
+ inputs: { source: { input_id: "i1" } },
+ convert: { mode: "speech", pitch: -40, pitch_auto: false },
+ });
+ assert.equal((low.convert as { pitch: number }).pitch, -24);
+});
+
+test("whatever a caller spreads into a convert run, no path or unknown key reaches the body", () => {
+ const sneaky = {
+ workflow: "convert" as const,
+ text: "should not be sent",
+ voice_ref: "/abs/ref.wav",
+ inputs: {
+ source: { input_id: "i1", path: "/x.wav" } as never,
+ target: { voice_id: "v1", clip_id: "c2" } as never,
+ reference: { voice_id: "v9" },
+ target_voice: "/abs/target.wav",
+ } as never,
+ convert: {
+ mode: "humming",
+ pitch: null,
+ pitch_auto: false,
+ route: "v1_svc",
+ source_audio: "/abs/src.wav",
+ } as never,
+ options: { retrieval_blend: 0.5, nested: { path: "/x" } } as never,
+ };
+ const body = buildAudioRunBody(sneaky);
+ assert.deepEqual(body, {
+ workflow: "convert",
+ inputs: { source: { input_id: "i1" } },
+ convert: { mode: "speech", pitch: null, pitch_auto: false },
+ options: { retrieval_blend: 0.5 },
+ });
+ assert.doesNotMatch(JSON.stringify(body), PATHLIKE);
+ assert.doesNotMatch(JSON.stringify(body), /target_voice|source_audio|"text"/);
+});
+
test("selections become one-id refs and server URLs", () => {
assert.deepEqual(
sourceRefOf({ kind: "input", id: "i1", name: "a", durationS: 1 }),
diff --git a/studio/frontend/tests/audio-source-input.test.ts b/studio/frontend/tests/audio-source-input.test.ts
index 3d33e920386..d7debec3252 100644
--- a/studio/frontend/tests/audio-source-input.test.ts
+++ b/studio/frontend/tests/audio-source-input.test.ts
@@ -250,7 +250,10 @@ test("a superseded microphone request releases its stream", () => {
hook,
/activeRef\.current = active;\s*if \(!active\) stopRecording\(\);/,
);
- assert.match(card, /useAudioSource\(\{ value, onChange, maxRecordSeconds, active \}\)/);
+ assert.match(
+ card,
+ /useAudioSource\(\{ value, onChange, maxRecordSeconds, active \}\)/,
+ );
assert.match(
hook,
/const abortAll = useCallback\(\(\) => \{\s*acquisition\.current \+= 1;/,
@@ -270,6 +273,45 @@ test("picking a new source clears an earlier error", () => {
);
});
+test("a history pick carries a transcript only from speech clips", async () => {
+ assert.match(
+ card,
+ /onChange\(clipReference\(\{ \.\.\.clip, workflow: clipWorkflow\(clip\) \}\)\)/,
+ );
+ const { clipReference } = await import(
+ "../src/features/audio/audio-run-request.ts"
+ );
+ const clip = { id: "c", prompt: "Hello there.", duration_s: 2 };
+ for (const workflow of ["speak", "clone", "edit"]) {
+ assert.equal(clipReference({ ...clip, workflow }).transcript, "Hello there.");
+ }
+ // Convert and Music prompts are labels, not what the clip says.
+ assert.equal(clipReference({ ...clip, workflow: "convert" }).transcript, null);
+ assert.equal(clipReference({ ...clip, workflow: "music" }).transcript, null);
+});
+
+test("a new selection clears the old one's error or expiry so it loads", () => {
+ assert.match(
+ hook,
+ /if \(valueKey && \(phase === "error" \|\| phase === "expired"\)\) \{\s*dispatch\(\{ type: "reset" \}\);\s*\}/,
+ );
+ const failed = audioSourceReducer(INITIAL_AUDIO_SOURCE_STATE, {
+ type: "fail",
+ message: "x",
+ });
+ assert.deepEqual(
+ audioSourceReducer(failed, { type: "reset" }),
+ INITIAL_AUDIO_SOURCE_STATE,
+ );
+});
+
+test("an upload that replaces a selection clears it, so a failed upload cannot run the old one", () => {
+ assert.match(
+ hook,
+ /dispatch\(\{ type: "upload-start", name: fileName, key: localKey \}\);\s*(\/\/[^\n]*\n\s*)?onChangeRef\.current\(null\);/,
+ );
+});
+
test("a long history prompt or voice name truncates and keeps the duration visible", () => {
// Grid items default to min-width: auto; without a minmax(0, 1fr) column one long prompt
// made its row thousands of pixels wide and pushed the duration off the card.
diff --git a/studio/frontend/tests/audio-workflows.test.ts b/studio/frontend/tests/audio-workflows.test.ts
index a36d6e40422..395dff3124e 100644
--- a/studio/frontend/tests/audio-workflows.test.ts
+++ b/studio/frontend/tests/audio-workflows.test.ts
@@ -20,16 +20,19 @@ const {
workflowForLoadedModel,
} = await import("../src/features/audio/workflows.ts");
-test("the Audio page offers Speak, Clone, Edit, Music, Separate and Transcribe in that order", () => {
+test("the Audio page offers Speak, Clone, Edit, Convert, Music, Separate and Transcribe in that order", () => {
assert.deepEqual(
AUDIO_WORKFLOWS.map((tab) => tab.id),
- ["speak", "clone", "edit", "music", "separate", "transcribe"],
+ ["speak", "clone", "edit", "convert", "music", "separate", "transcribe"],
);
assert.equal(audioWorkflowTab("separate").heading, "Separate audio");
assert.equal(audioWorkflowTab("speak").heading, "Text to speech");
assert.equal(audioWorkflowTab("clone").heading, "Clone a voice");
assert.equal(audioWorkflowTab("edit").heading, "Edit speech");
assert.equal(audioWorkflowTab("edit").label, "Edit");
+ assert.equal(audioWorkflowTab("convert").heading, "Convert voice");
+ assert.equal(audioWorkflowTab("convert").label, "Convert");
+ assert.equal(audioWorkflowTab("convert").hint, "Make a recording sound like another voice");
assert.equal(audioWorkflowTab("music").heading, "Create music");
assert.equal(audioWorkflowTab("transcribe").heading, "Transcribe");
});
@@ -38,6 +41,7 @@ test("Speak, Clone and Music share the main slot; only Transcribe uses the sidec
assert.equal(slotForWorkflow("speak"), "speak");
assert.equal(slotForWorkflow("clone"), "speak");
assert.equal(slotForWorkflow("edit"), "speak");
+ assert.equal(slotForWorkflow("convert"), "speak");
assert.equal(slotForWorkflow("music"), "speak");
assert.equal(slotForWorkflow("separate"), "speak");
assert.equal(slotForWorkflow("transcribe"), "transcribe");
@@ -47,6 +51,7 @@ test("Create|Train is offered on Speak, Clone and Transcribe, not Music", () =>
assert.equal(audioWorkflowTab("speak").createTrain, true);
assert.equal(audioWorkflowTab("clone").createTrain, true);
assert.equal(audioWorkflowTab("edit").createTrain, false);
+ assert.equal(audioWorkflowTab("convert").createTrain, false);
assert.equal(audioWorkflowTab("music").createTrain, false);
assert.equal(audioWorkflowTab("separate").createTrain, false);
assert.equal(audioWorkflowTab("transcribe").createTrain, true);
@@ -60,7 +65,8 @@ test("pipeline tags map onto workflows and anything else is ignored", () => {
assert.equal(audioWorkflowForTask(undefined), null);
assert.equal(isAudioWorkflowId("music"), true);
assert.equal(isAudioWorkflowId("separate"), true);
- assert.equal(isAudioWorkflowId("convert"), false);
+ assert.equal(isAudioWorkflowId("convert"), true);
+ assert.equal(isAudioWorkflowId("remix"), false);
});
test("a clip without a workflow falls back to its audio type", () => {
@@ -71,6 +77,7 @@ test("a clip without a workflow falls back to its audio type", () => {
assert.equal(clipWorkflow({ workflow: "music", audio_type: "snac" }), "music");
assert.equal(clipWorkflow({ workflow: "clone", audio_type: "audiocpp_tts" }), "clone");
assert.equal(clipWorkflow({ workflow: "edit", audio_type: "audiocpp_tts" }), "edit");
+ assert.equal(clipWorkflow({ workflow: "convert", audio_type: "audiocpp_tts" }), "convert");
assert.equal(clipWorkflow({ workflow: "separate", audio_type: "audiocpp_sep" }), "separate");
assert.equal(clipWorkflow({ workflow: "remix", audio_type: "audiocpp_music" }), "music");
assert.equal(audioWorkflowForAudioType(null), "speak");
@@ -116,3 +123,17 @@ test("only a model that lists Clone can run the Clone page", () => {
assert.equal(loadedModelRunsWorkflow({ workflow: "clone", audioWorkflows: undefined, music: false }), false);
assert.equal(loadedModelRunsWorkflow({ workflow: "music", audioWorkflows: undefined, music: true }), true);
});
+
+test("a convert-only model opens Convert and runs nothing else", () => {
+ assert.equal(
+ workflowForLoadedModel({ current: "speak", audioWorkflows: ["convert"], music: false }),
+ "convert",
+ );
+ assert.equal(
+ workflowForLoadedModel({ current: "convert", audioWorkflows: ["clone", "convert"], music: false }),
+ "convert",
+ );
+ assert.equal(loadedModelRunsWorkflow({ workflow: "convert", audioWorkflows: ["convert"], music: false }), true);
+ assert.equal(loadedModelRunsWorkflow({ workflow: "speak", audioWorkflows: ["convert"], music: false }), false);
+ assert.equal(loadedModelRunsWorkflow({ workflow: "convert", audioWorkflows: ["clone"], music: false }), false);
+});
diff --git a/studio/frontend/tests/audio-workspace-split.test.ts b/studio/frontend/tests/audio-workspace-split.test.ts
index 435d83ce714..ff60b2e382b 100644
--- a/studio/frontend/tests/audio-workspace-split.test.ts
+++ b/studio/frontend/tests/audio-workspace-split.test.ts
@@ -94,7 +94,7 @@ test("Generate says why it is off and runs from Mod+Enter anywhere on the page",
assert.match(host, /!pageRootRef\.current\?\.contains\(event\.target\)/);
assert.match(
host,
- /const handlePageGenerate =\s*ttsWorkflow === "separate" \? separate\.handleGenerate :\s*ttsWorkflow === "clone" \? clone\.handleGenerate :\s*ttsWorkflow === "edit" \? edit\.handleGenerate : handleGenerate;/,
+ /const handlePageGenerate =\s*ttsWorkflow === "separate" \? separate\.handleGenerate :\s*ttsWorkflow === "clone" \? clone\.handleGenerate :\s*ttsWorkflow === "edit" \? edit\.handleGenerate :\s*ttsWorkflow === "convert" \? convert\.handleGenerate : handleGenerate;/,
);
assert.match(host, /if \(canGenerate\) void handlePageGenerate\(\);/);
});