Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: add the Separate audio workflow with a synced stem mixer (#12610)

Etherl4c09908b2cc3
diff --git a/studio/backend/core/inference/audio_cpp_backend.py b/studio/backend/core/inference/audio_cpp_backend.py
index 2f6b6e354d9..e0667cbf548 100644
--- a/studio/backend/core/inference/audio_cpp_backend.py
+++ b/studio/backend/core/inference/audio_cpp_backend.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Worker backend for audio.cpp speech and music models in the main audio slot.
+"""Worker backend for audio.cpp speech, music and separation models in the main audio slot.
Selected by the inference worker in place of ``NativeAudioBackend`` when the model
is an audio.cpp TTS or music GGUF, so loading, auto-switching, idle eviction,
@@ -181,6 +181,8 @@ class AudioCppBackend:
self._model: Optional[AudioCppModel] = None
self._server: Optional[AudioCppServer] = None
self._server_lock = threading.RLock()
+ # ``model_options`` is not part of model equality: a changed overlap is only seen here.
+ self._served_session: dict = {}
# Request-raised session options (Stable Audio max_batch); cleared when another model loads.
self._session_overrides: dict[str, str] = {}
self._status_patch: Optional[dict[str, Any]] = None
@@ -295,6 +297,7 @@ class AudioCppBackend:
except AudioCppStartCancelledError as exc:
_raise_if_cancelled(cancel_event)
raise RuntimeError(str(exc)) from exc
+ self._served_session = dict((served.model_options or {}).get("session_options") or {})
self.device = "cpu" if self._server.backend == "cpu" else self._server.backend
def _stop_server_locked(self) -> None:
@@ -341,6 +344,8 @@ class AudioCppBackend:
if model is None:
raise RuntimeError("No active audio model")
_raise_if_cancelled(cancel_event)
+ if model.task == "sep":
+ raise RuntimeError(f"{model.display_name} separates audio; open Separate.")
if workflow == "music" and music is not None:
options = validate_options(model.options, audio_options)
return self._run_music(
@@ -401,6 +406,71 @@ class AudioCppBackend:
_raise_if_cancelled(cancel_event)
return wav, _wav_sample_rate(wav)
+ def separate_audio(
+ self,
+ source_path: str,
+ output_dir: str,
+ options: Optional[dict] = None,
+ cancel_event = None,
+ ) -> list[dict[str, Any]]:
+ """Split a 44.1 kHz WAV into stems under ``output_dir``; [] when none decodes."""
+ from core.inference.audio_cpp_outputs import SeparationOutputError, extract_named_outputs
+
+ if not self.active_model_name or self.active_model_name not in self.models:
+ raise RuntimeError("No active audio model")
+ model = self._model
+ if model is None:
+ raise RuntimeError("No active audio model")
+ spec = model.separation
+ if spec is None:
+ raise RuntimeError(f"{model.display_name} cannot separate audio.")
+ _raise_if_cancelled(cancel_event)
+ session = dict((model.model_options or {}).get("session_options") or {})
+ overlap = (options or {}).get("num_overlap")
+ if spec.overlap_option and overlap is not None:
+ session[spec.overlap_option] = str(max(1, min(8, int(overlap))))
+ with self._server_lock:
+ if self._server is None or not self._server.alive() or self._served_session != session:
+ logger.info("audio.cpp: (re)starting the server for %s with %s", model.id, session)
+ self._start_server(
+ replace(
+ model, model_options = {**model.model_options, "session_options": session}
+ ),
+ cancel_event,
+ )
+ server = self._server
+ response_path = Path(output_dir) / ".response.json"
+ try:
+ # The separation families refuse any other key.
+ server.post_json_to_file(
+ "/v1/tasks/run",
+ {"model": server.model_id, "audio": str(source_path)},
+ response_path,
+ timeout = _GENERATE_TIMEOUT_SECONDS,
+ cancel_event = cancel_event,
+ )
+ _raise_if_cancelled(cancel_event)
+ outputs = extract_named_outputs(response_path, output_dir)
+ except AudioCppRequestCancelledError:
+ self._restart_after_cancel()
+ _raise_if_cancelled(cancel_event)
+ raise
+ except AudioCppRequestError as exc:
+ from core.inference.audio_errors import AudioRuntimeError
+ raise AudioRuntimeError(
+ f"The audio runtime could not separate the track: {exc.detail}", status = exc.status
+ ) from exc
+ except SeparationOutputError as exc:
+ logger.warning("audio.cpp: %s answered without decodable stems: %s", model.id, exc)
+ return []
+ finally:
+ try:
+ response_path.unlink(missing_ok = True)
+ except OSError:
+ pass
+ _raise_if_cancelled(cancel_event)
+ return outputs
+
def _restart_after_cancel(self) -> None:
# The server keeps computing the abandoned request; stopping it frees the GPU now and the next
# request starts a fresh one.
diff --git a/studio/backend/core/inference/audio_cpp_models.py b/studio/backend/core/inference/audio_cpp_models.py
index 48ac7efa5c6..96c64b8faa6 100644
--- a/studio/backend/core/inference/audio_cpp_models.py
+++ b/studio/backend/core/inference/audio_cpp_models.py
@@ -45,13 +45,18 @@ _UMBRELLA_PREFIX = AUDIO_CPP_REPO.lower() + "/"
# The audio_type Studio records for a model served by audio.cpp, per task.
AUDIO_CPP_TTS_AUDIO_TYPE = "audiocpp_tts"
AUDIO_CPP_MUSIC_AUDIO_TYPE = "audiocpp_music"
-AUDIO_CPP_AUDIO_TYPES = frozenset((AUDIO_CPP_TTS_AUDIO_TYPE, AUDIO_CPP_MUSIC_AUDIO_TYPE))
+AUDIO_CPP_SEP_AUDIO_TYPE = "audiocpp_sep"
+AUDIO_CPP_AUDIO_TYPES = frozenset(
+ (AUDIO_CPP_TTS_AUDIO_TYPE, AUDIO_CPP_MUSIC_AUDIO_TYPE, AUDIO_CPP_SEP_AUDIO_TYPE)
+)
# Studio task -> the Hub pipeline task its rows carry.
HUB_TASKS = {
"tts": "text-to-speech",
"music": "text-to-audio",
"asr": "automatic-speech-recognition",
+ # Shares its tag with kinds Studio has no page for; audio_type tells them apart.
+ "sep": "audio-to-audio",
}
DEFAULT_AUDIO_CPP_STT_MODEL = f"{AUDIO_CPP_REPO}/Qwen3-ASR-0.6B-GGUF"
@@ -97,6 +102,14 @@ class CloneSpec:
instructions: bool = False
+@dataclass(frozen = True)
+class SeparationSpec:
+ """How a family splits a track into stems on /v1/tasks/run (task ``sep``, 44.1 kHz input)."""
+
+ # Changing it restarts the server.
+ overlap_option: Optional[str] = None
+
+
@dataclass(frozen = True)
class CompanionModel:
"""A second model a family loads beside its own (MioTTS's MioCodec), by session option."""
@@ -295,13 +308,14 @@ def _bindings_for(
}
if task == "asr":
return {"transcribe": WorkflowBinding("asr", "transcriptions", None, ("audio",))}
+ if task == "sep":
+ return {"separate": WorkflowBinding("sep", "tasks", None, ("audio",))}
return {}
@dataclass(frozen = True)
class AudioCppFamily:
family: str
- # Studio task: ``tts`` (speech), ``music`` (generation) or ``asr``; empty when Studio has no feature for it.
task: str
# audiocpp_server task when it is not the Studio task's default (a voice-design package runs "vdes").
server_task: Optional[str] = None
@@ -323,6 +337,7 @@ class AudioCppFamily:
speaks: bool = True
clone: Optional[CloneSpec] = None
companions: tuple[CompanionModel, ...] = ()
+ separation: Optional[SeparationSpec] = None
music: Optional[MusicSpec] = field(default = None, hash = False, compare = False)
@property
@@ -505,6 +520,14 @@ _CLONE_FAMILIES: tuple[AudioCppFamily, ...] = (
)
+_SEPARATION_FAMILIES: tuple[AudioCppFamily, ...] = (
+ AudioCppFamily("htdemucs", "sep", separation = SeparationSpec()),
+ AudioCppFamily("htdemucs_6stems", "sep", separation = SeparationSpec()),
+ AudioCppFamily("bs_roformer", "sep", separation = SeparationSpec("num_overlap")),
+ AudioCppFamily("mel_band_roformer", "sep", separation = SeparationSpec("num_overlap")),
+)
+
+
_FAMILY_LIST: tuple[AudioCppFamily, ...] = (
# Text to speech. Kokoro, KittenTTS, Piper and Inflect phonemize with eSpeak-ng, which the Unsloth
# bundles link statically (GPL-3.0-or-later) with its data file beside the server.
@@ -722,6 +745,7 @@ _FAMILY_LIST: tuple[AudioCppFamily, ...] = (
"audio8_asr",
)
),
+ *_SEPARATION_FAMILIES,
# Package families Studio does not lay out yet.
*(
AudioCppFamily(
@@ -742,7 +766,6 @@ _FAMILY_LIST: tuple[AudioCppFamily, ...] = (
for names, what in (
(("qwen3_forced_aligner", "mms_forced_aligner"), "Forced-alignment"),
(("sortformer_diar", "sortformer_diar_v2", "nemotron_3_diar"), "Speaker diarization"),
- (("htdemucs", "bs_roformer", "mel_band_roformer"), "Source separation"),
(("pulsevad", "silero_vad", "marblenet_vad"), "Voice activity detection"),
(("muscriptor", "sheetsage2"), "Music transcription"),
(("meanvc2", "rvc", "seed_vc"), "Voice conversion"),
@@ -773,7 +796,7 @@ _SPEC_TO_SERVER_TASK = {
"design": "vdes",
"speaker": "spk",
}
-_SERVER_TO_STUDIO_TASK = {"tts": "tts", "gen": "music", "asr": "asr"}
+_SERVER_TO_STUDIO_TASK = {"tts": "tts", "gen": "music", "asr": "asr", "sep": "sep"}
_TASK_NAMES = {
"align": "Forced-alignment",
"diar": "Speaker diarization",
@@ -804,8 +827,8 @@ def _family_from_spec_tasks(
tasks = [str(t).lower() for t in (spec or {}).get("tasks") or [] if isinstance(t, str)]
tokens = [_SPEC_TO_SERVER_TASK.get(t, t) for t in tasks]
# Speech first, then music, then transcription: a model that speaks is most useful spoken.
- # A clone-only family comes last: it loads as a cloning session and offers Clone alone.
- for token in ("tts", "vdes", "gen", "asr", "clon"):
+ # A clone-only family comes next: it loads as a cloning session and offers Clone alone.
+ for token in ("tts", "vdes", "gen", "asr", "clon", "sep"):
if token in tokens:
if runtime_knows_family(family) is False:
return AudioCppFamily(
@@ -821,6 +844,8 @@ def _family_from_spec_tasks(
speaks = False,
clone = CloneSpec("optional"),
)
+ if token == "sep":
+ return AudioCppFamily(family, "sep", separation = SeparationSpec())
studio = "tts" if token == "vdes" else _SERVER_TO_STUDIO_TASK[token]
return AudioCppFamily(family, studio, server_task = token)
if tokens:
@@ -1679,6 +1704,8 @@ def option_schema(
dozens of planner and debugging knobs, and the published GGUFs embed a stale copy. Everyone
else gets the runtime's spec, else the one embedded in the GGUF.
"""
+ if policy.task == "sep":
+ return ()
if policy.options:
source = {"options": {"request": list(policy.options)}}
else:
@@ -1787,6 +1814,7 @@ class AudioCppModel:
clone: Optional[CloneSpec] = None
companions: tuple[CompanionModel, ...] = ()
required_inputs: tuple[str, ...] = ()
+ separation: Optional[SeparationSpec] = None
music: Optional[MusicSpec] = field(default = None, hash = False, compare = False)
# A strict spec (``schema_version``) makes the runtime refuse any undeclared option.
request_keys: Optional[frozenset[str]] = field(default = None, hash = False, compare = False)
@@ -1813,6 +1841,8 @@ class AudioCppModel:
return AUDIO_CPP_TTS_AUDIO_TYPE
if self.task == "music":
return AUDIO_CPP_MUSIC_AUDIO_TYPE
+ if self.task == "sep":
+ return AUDIO_CPP_SEP_AUDIO_TYPE
return None
@property
@@ -2157,6 +2187,7 @@ def _resolve_uncached(
clone = policy.clone,
companions = policy.companions,
required_inputs = _required_inputs(spec, embedded, policy),
+ separation = policy.separation,
music = music_with_spec_bounds(policy.music, _raw_request_options(spec, embedded)),
request_keys = _request_keys(spec, embedded),
)
@@ -2282,7 +2313,8 @@ def require_runnable(model: AudioCppModel, task: Optional[str] = None) -> None:
raise AudioCppModelError(model.unsupported)
if task == "asr" and model.task != "asr":
raise AudioCppModelError(f"{model.display_name} is not a speech-to-text model.")
- if task in ("tts", "music") and model.task not in ("tts", "music"):
+ # The main audio slot, which speech, music and separation models share.
+ if task in ("tts", "music") and model.task not in ("tts", "music", "sep"):
if model.task == "asr":
raise AudioCppModelError(
f"{model.display_name} is a speech-to-text model; choose it for dictation in "
diff --git a/studio/backend/core/inference/audio_cpp_outputs.py b/studio/backend/core/inference/audio_cpp_outputs.py
new file mode 100644
index 00000000000..bebef68ba30
--- /dev/null
+++ b/studio/backend/core/inference/audio_cpp_outputs.py
@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Stems out of a saved separation answer, decoded in chunks from an mmap.
+
+Not ``json.loads``: ~42 MB of base64 per stem for a 180 s track would be held three times over.
+"""
+
+from __future__ import annotations
+
+import binascii
+import json
+import mmap
+import re
+import struct
+from pathlib import Path
+from typing import Any, Optional
+
+# Keys whose string value can carry base64 audio, matching ``_audio_candidates`` in the backend.
+_AUDIO_KEYS = ("audio", "wav", "data", "audio_base64", "b64_json")
+_AUDIO_KEY_RE = re.compile(rb'(?<!\\)"(' + "|".join(_AUDIO_KEYS).encode() + rb')"\s*:\s*"')
+_PLACEHOLDER = "\u0000span"
+_ID_UNSAFE_RE = re.compile(r"[^A-Za-z0-9_-]")
+_B64_ALPHABET = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+# Everything else is dropped before decoding: JSON's ``\\/`` escape, line breaks, padding spaces.
+_B64_DROP = bytes(b for b in range(256) if b not in _B64_ALPHABET)
+NO_STEMS = "The audio runtime returned no stems."
+
+
+class SeparationOutputError(RuntimeError):
+ """The answer has no stem Studio can decode."""
+
+ def __init__(self, detail: str = NO_STEMS):
+ super().__init__(detail)
+
+
+def _string_end(mm: mmap.mmap, start: int) -> int:
+ """Index of the unescaped quote that closes the JSON string starting at ``start``."""
+ pos = start
+ while True:
+ end = mm.find(b'"', pos)
+ if end < 0:
+ raise SeparationOutputError()
+ slashes = 0
+ while end - 1 - slashes >= start and mm[end - 1 - slashes] == 0x5C:
+ slashes += 1
+ if slashes % 2 == 0:
+ return end
+ pos = end + 1
+
+
+def _audio_spans(mm: mmap.mmap) -> list[tuple[int, int]]:
+ spans = []
+ pos = 0
+ while True:
+ match = _AUDIO_KEY_RE.search(mm, pos)
+ if match is None:
+ return spans
+ start = match.end()
+ end = _string_end(mm, start)
+ spans.append((start, end))
+ pos = end + 1
+
+
+def _redacted(mm: mmap.mmap, spans: list[tuple[int, int]]) -> Any:
+ """The document with every audio string replaced by ``"\\u0000span<k>"``, parsed."""
+ parts = []
+ prev = 0
+ for index, (start, end) in enumerate(spans):
+ parts.append(mm[prev:start])
+ parts.append(f"\\u0000span{index}".encode("ascii"))
+ prev = end
+ parts.append(mm[prev:])
+ try:
+ return json.loads(b"".join(parts).decode("utf-8"))
+ except (UnicodeDecodeError, ValueError) as exc:
+ raise SeparationOutputError() from exc
+
+
+def _find_outputs(node: Any) -> Optional[list]:
+ if isinstance(node, dict):
+ outputs = node.get("named_audio_outputs")
+ if isinstance(outputs, list):
+ return outputs
+ for value in node.values():
+ found = _find_outputs(value)
+ if found is not None:
+ return found
+ elif isinstance(node, list):
+ for item in node:
+ found = _find_outputs(item)
+ if found is not None:
+ return found
+ return None
+
+
+def _span_index(item: dict) -> Optional[int]:
+ for key in _AUDIO_KEYS:
+ value = item.get(key)
+ if isinstance(value, str) and value.startswith(_PLACEHOLDER):
+ try:
+ return int(value[len(_PLACEHOLDER) :])
+ except ValueError:
+ return None
+ return None
+
+
+def safe_stem_id(raw: Any, taken: set[str]) -> str:
+ """A file-safe id (``[A-Za-z0-9_-]{1,64}``), unique among ``taken`` by a ``_2`` style suffix."""
+ base = _ID_UNSAFE_RE.sub("_", str(raw or "").strip())[:64] or "stem"
+ stem_id = base
+ n = 2
+ while stem_id in taken:
+ suffix = f"_{n}"
+ stem_id = base[: 64 - len(suffix)] + suffix
+ n += 1
+ taken.add(stem_id)
+ return stem_id
+
+
+def _decode_span(mm: mmap.mmap, start: int, end: int, dest: Path, chunk: int) -> None:
+ head = mm[start : min(end, start + 128)]
+ if head.startswith(b"data:"):
+ comma = head.find(b",")
+ if comma >= 0:
+ start += comma + 1
+ chunk = max(4, chunk - chunk % 4)
+ carry = b""
+ with open(dest, "wb") as out:
+ pos = start
+ while pos < end:
+ stop = min(end, pos + chunk)
+ raw = mm[pos:stop]
+ text = raw.translate(None, _B64_DROP)
+ del raw
+ if carry:
+ text = carry + text
+ usable = len(text) - len(text) % 4
+ if usable:
+ out.write(binascii.a2b_base64(memoryview(text)[:usable]))
+ carry = text[usable:]
+ del text
+ pos = stop
+ if carry.strip(b"="):
+ out.write(binascii.a2b_base64(carry + b"=" * (-len(carry) % 4)))
+
+
+def riff_info(path: Path) -> dict[str, Any]:
+ """``sample_rate``, ``channels``, ``frames`` from a RIFF/WAVE header; ValueError otherwise."""
+ with open(path, "rb") as f:
+ header = f.read(12)
+ if len(header) < 12 or header[:4] != b"RIFF" or header[8:12] != b"WAVE":
+ raise ValueError("not a RIFF/WAVE file")
+ size = path.stat().st_size
+ fmt = None
+ while True:
+ chunk_head = f.read(8)
+ if len(chunk_head) < 8:
+ raise ValueError("no data chunk")
+ name, length = chunk_head[:4], struct.unpack("<I", chunk_head[4:])[0]
+ if name == b"fmt ":
+ body = f.read(length)
+ if len(body) < 16:
+ raise ValueError("short fmt chunk")
+ audio_format, channels, rate, _byte_rate, block_align, _bits = struct.unpack(
+ "<HHIIHH", body[:16]
+ )
+ if audio_format not in (1, 3, 0xFFFE) or not channels or not rate:
+ raise ValueError("unsupported WAV encoding")
+ fmt = (channels, rate, block_align or 1)
+ if length % 2:
+ f.seek(1, 1)
+ elif name == b"data":
+ if fmt is None:
+ raise ValueError("data before fmt")
+ # A streamed writer leaves the size at 0 or 0xFFFFFFFF; the file length is the truth.
+ available = size - f.tell()
+ if length == 0 or length > available:
+ length = available
+ channels, rate, block_align = fmt
+ return {"sample_rate": rate, "channels": channels, "frames": length // block_align}
+ else:
+ f.seek(length + (length % 2), 1)
+
+
+def extract_named_outputs(
+ json_path: Path | str,
+ out_dir: Path | str,
+ *,
+ chunk: int = 4 << 20,
+) -> list[dict[str, Any]]:
+ """Decode ``named_audio_outputs[*]`` into ``out_dir/<id>.wav``, all or none.
+
+ Top-level ``audio`` is ignored: in a batch answer it repeats the first named output."""
+ json_path = Path(json_path)
+ out_dir = Path(out_dir)
+ try:
+ size = json_path.stat().st_size
+ except OSError as exc:
+ raise SeparationOutputError() from exc
+ if size == 0:
+ raise SeparationOutputError()
+ written: list[Path] = []
+ try:
+ with open(json_path, "rb") as f, mmap.mmap(f.fileno(), 0, access = mmap.ACCESS_READ) as mm:
+ spans = _audio_spans(mm)
+ outputs = _find_outputs(_redacted(mm, spans))
+ if not outputs:
+ raise SeparationOutputError()
+ taken: set[str] = set()
+ results = []
+ for item in outputs:
+ if not isinstance(item, dict) or item.get("id") in (None, ""):
+ raise SeparationOutputError()
+ index = _span_index(item)
+ if index is None or index >= len(spans):
+ raise SeparationOutputError()
+ stem_id = safe_stem_id(item["id"], taken)
+ dest = out_dir / f"{stem_id}.wav"
+ written.append(dest)
+ start, end = spans[index]
+ try:
+ _decode_span(mm, start, end, dest, chunk)
+ info = riff_info(dest)
+ except (binascii.Error, ValueError) as exc:
+ raise SeparationOutputError() from exc
+ results.append(
+ {
+ "id": stem_id,
+ "path": str(dest),
+ "sample_rate": info["sample_rate"],
+ "channels": info["channels"],
+ "duration_s": round(info["frames"] / info["sample_rate"], 3),
+ }
+ )
+ return results
+ except BaseException:
+ for path in written:
+ try:
+ path.unlink(missing_ok = True)
+ except OSError:
+ pass
+ raise
diff --git a/studio/backend/core/inference/audio_cpp_server.py b/studio/backend/core/inference/audio_cpp_server.py
index add4f60e1f1..8df89599381 100644
--- a/studio/backend/core/inference/audio_cpp_server.py
+++ b/studio/backend/core/inference/audio_cpp_server.py
@@ -28,7 +28,7 @@ import threading
import time
import uuid
from pathlib import Path
-from typing import Any, Optional
+from typing import Any, BinaryIO, Optional
from core.inference.audio_cpp_models import AudioCppModel
from core.inference.audio_errors import sanitize_runtime_tail
@@ -50,6 +50,9 @@ INSTALL_RECORD = "UNSLOTH_AUDIO_CPP_PREBUILT_INFO.json"
# Model load happens before the server answers, and a multi-GB music model can take a while from a cold disk.
_SERVER_START_TIMEOUT_SECONDS = 600.0
_PROBE_TIMEOUT_SECONDS = 2.0
+# A streamed answer is copied to its file in blocks this size; an error body is read up to the cap.
+_SINK_CHUNK_BYTES = 1 << 20
+_ERROR_BODY_BYTES = 64 * 1024
# CUDA graph replay wedges these transducers mid-request (server spins, never answers); graphs off fixes it.
_NO_CUDA_GRAPH_FAMILIES = frozenset({"nemotron_asr", "parakeet_tdt"})
_GPU_HOST_THREADS = 8
@@ -537,8 +540,12 @@ class AudioCppServer:
content_type: str = "application/json",
timeout: float = 3600.0,
cancel_event: Optional[threading.Event] = None,
+ sink: Optional[BinaryIO] = None,
) -> tuple[int, str, bytes]:
- """One HTTP round trip. A set ``cancel_event`` closes the socket and raises."""
+ """One HTTP round trip; a set ``cancel_event`` closes the socket and raises.
+
+ With a ``sink`` a 2xx body is streamed into it and ``b""`` returned (separation answers are
+ hundreds of MB)."""
if cancel_event is not None and cancel_event.is_set():
raise AudioCppRequestCancelledError("Request cancelled.")
connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout = timeout)
@@ -549,7 +556,19 @@ class AudioCppServer:
try:
connection.request(method, path, body = body, headers = {"Content-Type": content_type})
with connection.getresponse() as response:
- payload = response.read()
+ if sink is None:
+ payload = response.read()
+ elif 200 <= response.status < 300:
+ while True:
+ if cancel_event is not None and cancel_event.is_set():
+ raise AudioCppRequestCancelledError("Request cancelled.")
+ block = response.read(_SINK_CHUNK_BYTES)
+ if not block:
+ break
+ sink.write(block)
+ payload = b""
+ else:
+ payload = response.read(_ERROR_BODY_BYTES)
outcome["result"] = (
response.status,
response.getheader("Content-Type") or "",
@@ -601,6 +620,16 @@ class AudioCppServer:
raise AudioCppRequestError(status, _error_detail(data))
return ctype, data
+ def post_json_to_file(self, path: str, payload: dict, dest: Path, **kwargs) -> str:
+ """POST ``payload`` and stream a 2xx body into ``dest``; return the content type."""
+ with open(dest, "wb") as sink:
+ status, ctype, data = self.request(
+ "POST", path, body = json.dumps(payload).encode("utf-8"), sink = sink, **kwargs
+ )
+ if not 200 <= status < 300:
+ raise AudioCppRequestError(status, _error_detail(data))
+ return ctype
+
def post_multipart(
self,
path: str,
diff --git a/studio/backend/core/inference/audio_gallery.py b/studio/backend/core/inference/audio_gallery.py
index 614216c68dd..0156b7bca4b 100644
--- a/studio/backend/core/inference/audio_gallery.py
+++ b/studio/backend/core/inference/audio_gallery.py
@@ -10,9 +10,11 @@ valid record. Dumb storage: the route owns the schema, this reads, writes and so
from __future__ import annotations
+import errno
import json
import os
import re
+import shutil
import uuid
from collections.abc import Callable
from pathlib import Path
@@ -67,6 +69,44 @@ def save(wav_bytes: bytes, meta: dict[str, Any]) -> dict[str, Any]:
return _record(audio_id, meta)
+def save_file(
+ src: Path,
+ meta: dict[str, Any],
+ *,
+ prune: bool = True,
+) -> dict[str, Any]:
+ """Move a WAV on disk into the gallery with its sidecar; all or nothing.
+
+ Multi-file runs pass ``prune=False`` and prune once at the end, so a cap never splits a run.
+ """
+ audio_id = uuid.uuid4().hex
+ directory = gallery_dir()
+ wav_path = directory / f"{audio_id}.wav"
+ wav_tmp = directory / f".{audio_id}.wav.tmp"
+ sidecar = directory / f"{audio_id}.json"
+ sidecar_tmp = directory / f".{audio_id}.json.tmp"
+ try:
+ try:
+ os.replace(src, wav_path)
+ except OSError as exc:
+ if exc.errno != errno.EXDEV:
+ raise
+ shutil.copyfile(src, wav_tmp)
+ os.replace(wav_tmp, wav_path)
+ sidecar_tmp.write_text(json.dumps(meta), encoding = "utf-8")
+ os.replace(sidecar_tmp, sidecar)
+ except BaseException:
+ for path in (wav_tmp, sidecar_tmp, wav_path, sidecar):
+ try:
+ path.unlink(missing_ok = True)
+ except OSError:
+ pass
+ raise
+ if prune:
+ _prune_to_cap()
+ return _record(audio_id, meta)
+
+
# The OpenAI-compatible /v1/audio/speech route persists every call, so an automated client can grow the gallery until
# the disk fills. Bounded here rather than at that route so the UI's own runaway is covered too. Generous by default:
# this is a convenience gallery, and the clip is returned to the caller either way.
@@ -139,6 +179,8 @@ def _prune_to_cap() -> int:
if running > byte_cap and index > 0:
keep = index
break
+ # A run's clips (a separation's stems) go together: a kept clip keeps its siblings.
+ kept_groups = {r["group_id"] for r, _ in entries[:keep] if r.get("group_id")}
if keep >= len(entries):
return 0
@@ -148,6 +190,8 @@ def _prune_to_cap() -> int:
flags = gallery_flags.read_trusted(directory)
pruned: list[str] = []
for record, _cursor in entries[keep:]:
+ if record.get("group_id") in kept_groups:
+ continue
audio_id = record["id"]
if gallery_flags.is_archived(flags, audio_id) or gallery_flags.pin_rank(
flags, audio_id
@@ -207,7 +251,7 @@ def _record(
def _workflow(meta: dict[str, Any]) -> str:
workflow = meta.get("workflow")
- if workflow in ("speak", "clone", "music"):
+ if workflow in ("speak", "clone", "music", "separate"):
return workflow
return workflow_for_audio_type(meta.get("audio_type"))
@@ -430,8 +474,8 @@ def clear(include_archived: bool = False, workflow: Optional[str] = None) -> int
orphan WAVs are preserved, since list_audio already hides them.
Archived clips are spared unless ``include_archived``, and sparing them raises
- FlagsUnavailable when the flag store cannot be read. A ``workflow`` (speak, clone or music)
- spares the other workflows' clips."""
+ FlagsUnavailable when the flag store cannot be read. A ``workflow`` (speak, clone, music or
+ separate) spares the other workflows' clips."""
removed = 0
directory = gallery_dir()
with gallery_flags.exclusive(directory, require_file_lock = not include_archived):
diff --git a/studio/backend/core/inference/audio_workflows.py b/studio/backend/core/inference/audio_workflows.py
index c9adc8e569b..e46521d20fb 100644
--- a/studio/backend/core/inference/audio_workflows.py
+++ b/studio/backend/core/inference/audio_workflows.py
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Which Audio page workflows (Speak, Clone, Music, Transcribe) a model or a clip belongs to.
+"""Which Audio page workflows (Speak, Clone, Music, Separate, Transcribe) a model or a clip
+belongs to.
Mirrors ``studio/frontend/src/features/audio/workflows.ts``. This module reads the audio.cpp
constants; ``audio_cpp_models`` never imports it.
@@ -11,9 +12,13 @@ from __future__ import annotations
from typing import Optional
-from core.inference.audio_cpp_models import AUDIO_CPP_MUSIC_AUDIO_TYPE, HUB_TASKS
+from core.inference.audio_cpp_models import (
+ AUDIO_CPP_MUSIC_AUDIO_TYPE,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
+ HUB_TASKS,
+)
-AUDIO_WORKFLOW_IDS = ("speak", "clone", "music", "transcribe")
+AUDIO_WORKFLOW_IDS = ("speak", "clone", "music", "separate", "transcribe")
# Generation audio types that make music; every other generation audio type speaks.
MUSIC_AUDIO_TYPES = frozenset(("minimax_music3", AUDIO_CPP_MUSIC_AUDIO_TYPE))
@@ -33,6 +38,8 @@ def status_audio_workflows(is_audio: bool, audio_type: Optional[str]) -> list[st
return []
if audio_type in _TRANSCRIBE_AUDIO_TYPES:
return ["transcribe"]
+ if audio_type == AUDIO_CPP_SEP_AUDIO_TYPE:
+ return ["separate"]
return [workflow_for_audio_type(audio_type)]
@@ -42,8 +49,12 @@ def inventory_audio_workflows(
"""Workflows a cached or local model row serves, from its pipeline task first.
audio.cpp music rows carry ``text-to-audio`` with no audio_type, so the task decides before
- the audio_type does. None for a row that is not an audio model.
+ the audio_type does. A separation row is the exception: its ``audio-to-audio`` task is shared
+ with the kinds Studio has no page for, so its audio_type decides. None for a row that is not
+ an audio model.
"""
+ if audio_type == AUDIO_CPP_SEP_AUDIO_TYPE:
+ return ["separate"]
if task == HUB_TASKS["music"]:
return ["music"]
if task == HUB_TASKS["asr"]:
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index f24b03694dc..71f246a7f37 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -3334,6 +3334,26 @@ class InferenceOrchestrator:
except RuntimeError:
self._teardown_not_sent()
+ def separate_audio_response(
+ self,
+ source_path: str,
+ output_dir: str,
+ audio_options: Optional[dict] = None,
+ cancel_event = None,
+ ) -> list[dict]:
+ """Split a prepared 44.1 kHz WAV into stems under ``output_dir``; the full token budget
+ gives the watchdog the same hour the runtime request has."""
+ outputs, _sample_rate = self.generate_audio_response(
+ text = "",
+ max_new_tokens = AUDIO_GENERATION_MAX_TOKENS,
+ cancel_event = cancel_event,
+ audio_options = audio_options,
+ workflow = "separate",
+ audio_inputs = {"source": source_path},
+ output_dir = output_dir,
+ )
+ return outputs
+
def generate_audio_response(
self,
text: str,
@@ -3358,8 +3378,9 @@ class InferenceOrchestrator:
stats_holder: Optional[dict] = None,
) -> Tuple[bytes, int]:
"""Generate TTS audio. Returns (wav_bytes, sample_rate). Blocking: sends the command and
- waits for the full audio response. ``audio_inputs`` maps a role (reference, emotion) to a
- server-local WAV path; audio bytes never cross the queue."""
+ waits for the full audio response. ``audio_inputs`` maps a role (reference, emotion,
+ source) to a server-local WAV path; audio bytes never cross the queue. A separation
+ (``output_dir`` set) returns (outputs, sample_rate) instead, see ``separate_audio_response``."""
if not self._ensure_subprocess_alive():
raise RuntimeError("Inference subprocess is not running")
if not self.active_model_name:
@@ -3505,6 +3526,8 @@ class InferenceOrchestrator:
if rtype == "audio_done":
if cancel_event is not None and cancel_event.is_set():
raise AudioGenerationCancelledError("Audio generation cancelled")
+ if resp.get("outputs") is not None:
+ return resp["outputs"], int(resp.get("sample_rate") or 0)
wav_bytes = base64.b64decode(resp["wav_base64"])
sample_rate = resp["sample_rate"]
status_patch = resp.get("status_patch")
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 67bffd721b4..40772009b88 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -30,6 +30,7 @@ logger = get_logger(__name__)
from core.inference.audio_errors import (
AUDIO_RUNTIME_ERROR_CODE,
AUDIO_UNSUPPORTED_CODE,
+ AudioBackendUnsupportedError,
AudioRuntimeError,
)
from core.inference.context_refusal import ContextBudgetExceeded
@@ -1597,10 +1598,34 @@ def _handle_share_object(backend, cmd: dict, resp_queue: Any) -> None:
def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
- """Handle TTS audio generation — returns WAV bytes + sample_rate."""
+ """Handle TTS audio generation — returns WAV bytes + sample_rate.
+
+ A separation returns the paths of the stems it wrote under the route's ``output_dir``
+ instead: hundreds of megabytes of audio never cross the queue."""
request_id = cmd.get("request_id", "")
try:
logger.info("Starting audio generation for request_id=%s", request_id)
+ if cmd.get("workflow") == "separate":
+ separate = getattr(backend, "separate_audio", None)
+ if separate is None:
+ raise AudioBackendUnsupportedError("This model cannot separate audio.")
+ outputs = separate(
+ source_path = cmd["audio_inputs"]["source"],
+ output_dir = cmd["output_dir"],
+ options = cmd.get("audio_options"),
+ cancel_event = cancel_event,
+ )
+ _send_response(
+ resp_queue,
+ {
+ "type": "audio_done",
+ "request_id": request_id,
+ "outputs": outputs,
+ "sample_rate": 44100,
+ },
+ )
+ logger.info("Finished audio separation for request_id=%s", request_id)
+ return
# Only audio.cpp models take per-model options; other backends never see the keyword.
extra = {"audio_options": cmd["audio_options"]} if cmd.get("audio_options") else {}
# Run fields reach only a backend that takes them: a Speak run on a native TTS model carries a
@@ -1661,6 +1686,9 @@ def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) ->
if isinstance(exc, AudioRuntimeError):
response["code"] = AUDIO_RUNTIME_ERROR_CODE
response["status"] = exc.status
+ elif isinstance(exc, AudioBackendUnsupportedError):
+ response["code"] = AUDIO_UNSUPPORTED_CODE
+ response["hint"] = exc.hint
_send_response(resp_queue, response)
diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.py
index 9e1435f6678..bf0cb3310f9 100644
--- a/studio/backend/hub/services/models/cache_inventory.py
+++ b/studio/backend/hub/services/models/cache_inventory.py
@@ -670,13 +670,16 @@ def _scan_cached_gguf(
)
row_audio_type = None
row_audio_workflows = None
- if row_task == "text-to-speech":
+ if row_task in ("text-to-speech", "audio-to-audio"):
try:
from hub.services.models import catalog_classification
+
row_audio_type = catalog_classification._repo_gguf_audio_type(
repo_info, gguf_identity.load_snapshot or gguf_snapshot
)
- if row_audio_type == "audiocpp_tts":
+ if row_task == "audio-to-audio" and row_audio_type != "audiocpp_sep":
+ row_audio_type = None
+ if row_audio_type in ("audiocpp_tts", "audiocpp_sep"):
row_audio_workflows = catalog_classification._gguf_path_audio_workflows(
gguf_identity.load_snapshot
or gguf_snapshot
diff --git a/studio/backend/hub/services/models/catalog_classification.py b/studio/backend/hub/services/models/catalog_classification.py
index 38df93f8a78..f012fca3863 100644
--- a/studio/backend/hub/services/models/catalog_classification.py
+++ b/studio/backend/hub/services/models/catalog_classification.py
@@ -202,6 +202,7 @@ def _audio_cpp_classification(
audio_type = {
"tts": acm.AUDIO_CPP_TTS_AUDIO_TYPE,
"music": acm.AUDIO_CPP_MUSIC_AUDIO_TYPE,
+ "sep": acm.AUDIO_CPP_SEP_AUDIO_TYPE,
}.get(policy.task)
return task, audio_type
@@ -568,7 +569,11 @@ def _local_model_classification_for_task(
model, task: Optional[str]
) -> tuple[Optional[str], Optional[str]]:
"""Add decoder provenance to an already classified local-row task."""
- audio_type = _local_model_audio_type(model) if task is None or task == _SPEECH_TASK else None
+ probe = task is None or task in (_SPEECH_TASK, "audio-to-audio")
+ audio_type = _local_model_audio_type(model) if probe else None
+ # As for cached rows: only a separation GGUF is an audio-to-audio row Studio has a page for.
+ if task == "audio-to-audio" and audio_type != "audiocpp_sep":
+ audio_type = None
if task is None and audio_type is not None:
from utils.audio_tokens import is_output_audio_type
if is_output_audio_type(audio_type):
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index e428447ec2f..ceb17379ac9 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -5149,7 +5149,7 @@ class AudioGalleryItem(BaseModel):
pinned: bool = Field(False, description = "Pinned to the top of history")
archived: bool = Field(False, description = "Moved to the archived shelf, hidden from history")
workflow: Optional[str] = Field(
- None, description = "Audio page workflow that made the clip: speak, clone or music"
+ None, description = "Audio page workflow that made the clip: speak, clone, music or separate"
)
order_at: Optional[float] = Field(
None,
@@ -5204,9 +5204,9 @@ class AudioRunInputs(BaseModel):
model_config = ConfigDict(extra = "forbid")
reference: Optional[AudioSourceRef] = None
+ source: Optional[AudioSourceRef] = None
reference_text: Optional[str] = Field(None, max_length = 4000)
emotion: Optional[AudioSourceRef] = None
- source: Optional[AudioSourceRef] = None
class AudioMusicRange(BaseModel):
@@ -5236,8 +5236,9 @@ class AudioRunRequest(BaseModel):
model_config = ConfigDict(extra = "forbid")
- workflow: Literal["clone", "speak", "music"]
- text: str
+ workflow: Literal["clone", "speak", "music", "separate"]
+ # Required except for a separation (the route answers a separation given text with a 400).
+ text: Optional[str] = None
language: Optional[str] = Field(None, max_length = 64)
instructions: Optional[str] = Field(None, max_length = 4000)
inputs: AudioRunInputs = Field(default_factory = AudioRunInputs)
@@ -5271,9 +5272,10 @@ class AudioRunRequest(BaseModel):
if self.workflow == "music":
if self.mode is None:
raise ValueError("Pick a music mode: song, sfx or edit.")
- else:
- if not self.text:
- raise ValueError("text must not be empty.")
+ if self.text is None:
+ raise ValueError("text is required for music.")
+ elif self.workflow != "separate" and not self.text:
+ raise ValueError("text must not be empty.")
return self
@@ -5294,6 +5296,7 @@ class AudioRunAudio(BaseModel):
class AudioRunResponse(BaseModel):
clips: List[AudioRunClip] = Field(default_factory = list)
+ # One separation's stems share it; None for a single clip.
group_id: Optional[str] = None
model: str
audio: Optional[AudioRunAudio] = Field(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index e716568fbda..595399bbe44 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7,6 +7,7 @@ Inference API routes for model loading and text generation.
import math
import os
+import shutil
import sys
import time
import uuid
@@ -102,7 +103,9 @@ from core.inference.audio_errors import (
AudioGenerationCancelledError,
AudioRuntimeError,
audio_runtime_http_error,
+ sanitize_runtime_detail,
)
+from core.inference.audio_cpp_outputs import NO_STEMS
from core.inference import context_refusal
from core.inference.context_window import (
estimate_message_tokens as _estimate_message_tokens,
@@ -22176,6 +22179,11 @@ async def _generate_tts_wav(
stats_holder = stats_holder,
)
+ if audio_type == "audiocpp_sep":
+ raise HTTPException(
+ status_code = 400,
+ detail = f"{_audio_model_label(model_name)} separates audio into stems. Open Audio, then Separate.",
+ )
if audio_type not in supported_audio_types:
raise HTTPException(
status_code = 400,
@@ -22747,9 +22755,11 @@ async def run_audio_workflow(
request: Request,
current_subject: str = Depends(get_current_subject),
):
- """Run one Audio page workflow (Clone, Speak in a saved voice, or Music); clips go to history.
- Audio is named by id and resolved here in the caller's account; the worker gets a prepared
- copy's path, never bytes."""
+ """Run one Audio page workflow (Clone, Speak in a saved voice, Music or Separate); clips go to
+ history. Audio is named by id and resolved here in the caller's account; the worker gets a
+ prepared copy's path, never bytes."""
+ if body.workflow == "separate":
+ return await _run_separation(body, request, current_subject)
import base64
from core.inference import audio_inputs
@@ -22839,6 +22849,254 @@ async def run_audio_workflow(
return AudioRunResponse(clips = [clip], model = model_name)
+# Stems in the order the Separate page lists them; ids a runtime adds follow in its own order.
+_STEM_ORDER = ("vocals", "drums", "bass", "guitar", "piano", "other", "instrumental")
+_SEPARATE_MAX_SECONDS = 600.0
+_SEPARATE_RATE = 44100
+
+
+def _separation_options(body: AudioRunRequest, family: Optional[str], label: str) -> Optional[dict]:
+ """The run's separation options, checked against what the loaded family takes."""
+ from core.inference.audio_cpp_models import FAMILIES
+
+ options = body.options or {}
+ if not options:
+ return None
+ policy = FAMILIES.get(family or "")
+ if policy is None or policy.separation is None or not policy.separation.overlap_option:
+ raise HTTPException(status_code = 400, detail = f"{label} has no separation options.")
+ for name in options:
+ if name != "num_overlap":
+ raise HTTPException(status_code = 400, detail = f"Unknown option '{name}'.")
+ value = options["num_overlap"]
+ if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= 8:
+ raise HTTPException(
+ status_code = 400, detail = "num_overlap must be a whole number from 1 to 8."
+ )
+ return {"num_overlap": value}
+
+
+def _clock(seconds: float) -> str:
+ whole = int(math.ceil(seconds))
+ return f"{whole // 60}:{whole % 60:02d}"
+
+
+def _probe_audio_with_av(path) -> Optional[tuple[int, float]]:
+ """``(channels, seconds)`` of the first audio stream, through FFmpeg; None if unreadable."""
+ try:
+ import av
+
+ from core.inference.audio_inputs import _av_open
+ with _av_open(av, str(path)) as container:
+ stream = container.streams.audio[0]
+ if stream.duration is not None and stream.time_base is not None:
+ seconds = float(stream.duration * stream.time_base)
+ elif container.duration is not None:
+ seconds = container.duration / 1_000_000
+ else:
+ return None
+ return int(stream.channels), seconds
+ except Exception: # noqa: BLE001 - any failure reads as an unreadable track
+ return None
+
+
+def _prepare_separation_source(ref: dict[str, Any]):
+ """``(source, prepared 44.1 kHz path)``, after the 10-minute cap; stereo stays stereo."""
+ import wave
+
+ from core.inference import audio_inputs
+
+ source = audio_inputs.resolve_source(ref)
+ try:
+ with wave.open(str(source.path), "rb") as wav:
+ channels = wav.getnchannels()
+ seconds = wav.getnframes() / wav.getframerate()
+ except Exception: # noqa: BLE001 - e.g. a float stem from history, which wave cannot open
+ probed = _probe_audio_with_av(source.path)
+ if probed is None:
+ raise audio_inputs.AudioInputError(400, "This track could not be read. Add it again.")
+ channels, seconds = probed
+ if seconds > _SEPARATE_MAX_SECONDS:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Separate tracks up to 10 minutes long. This one is "
+ f"{_clock(seconds)}. Pick a shorter track.",
+ )
+ return source, audio_inputs.prepared_path(source, _SEPARATE_RATE, stereo = channels == 2)
+
+
+def _stem_rank(ids: list[str]):
+ runtime = {stem_id: index for index, stem_id in enumerate(ids)}
+ return lambda stem_id: (
+ _STEM_ORDER.index(stem_id) if stem_id in _STEM_ORDER else len(_STEM_ORDER),
+ runtime[stem_id],
+ )
+
+
+def _checked_stems(outputs: Any, staging: Path) -> list[dict]:
+ """The worker's stems, each a WAV inside ``staging``, in display order; 502 otherwise."""
+ if not isinstance(outputs, list) or not outputs:
+ raise HTTPException(status_code = 502, detail = NO_STEMS)
+ root = staging.resolve()
+ stems = []
+ for output in outputs:
+ try:
+ path = Path(str(output["path"])).resolve()
+ path.relative_to(root)
+ with open(path, "rb") as f:
+ riff = f.read(4) == b"RIFF"
+ stem = {
+ "id": str(output["id"]),
+ "path": path,
+ "sample_rate": int(output["sample_rate"]),
+ "duration_s": float(output["duration_s"]),
+ }
+ except (KeyError, TypeError, ValueError, OSError):
+ riff = False
+ if not riff:
+ logger.warning("audio.separate: the worker returned an unusable stem")
+ raise HTTPException(status_code = 502, detail = NO_STEMS)
+ stems.append(stem)
+ rank = _stem_rank([stem["id"] for stem in stems])
+ return sorted(stems, key = lambda stem: rank(stem["id"]))
+
+
+def _save_stems(stems: list[dict], base_meta: dict[str, Any]) -> list[dict]:
+ """Save every stem to history under one group, all or none, then prune once."""
+ from core.inference import audio_gallery
+
+ records: list[dict] = []
+ try:
+ for stem in stems:
+ meta = {
+ **base_meta,
+ "sample_rate": stem["sample_rate"],
+ "duration_s": stem["duration_s"],
+ "role": stem["id"],
+ }
+ records.append(audio_gallery.save_file(stem["path"], meta, prune = False))
+ except Exception:
+ for record in records:
+ audio_gallery.delete(record["id"])
+ raise
+ audio_gallery._prune_to_cap()
+ return records
+
+
+async def _run_separation(
+ body: AudioRunRequest, request: Request, current_subject: str
+) -> AudioRunResponse:
+ """Split a track into stems saved as one history group; worker paths are checked to stay in
+ the account's gallery and never reach the client."""
+ from core.inference import audio_gallery, audio_inputs
+
+ if body.text is not None:
+ raise HTTPException(status_code = 400, detail = "Separate takes no text.")
+ if any(
+ name in body.model_fields_set and getattr(body, name) not in (None, False, 1)
+ for name in _MUSIC_ONLY_FIELDS
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = "A music mode, lyrics, variations and edits apply to Music only.",
+ )
+ inputs = body.inputs
+ if inputs.reference is not None or inputs.emotion is not None or inputs.reference_text:
+ raise HTTPException(status_code = 400, detail = "Separate takes only a track.")
+ source_ref = inputs.source
+ if source_ref is None:
+ raise HTTPException(status_code = 400, detail = "Add a track to separate.")
+ if source_ref.voice_id:
+ raise HTTPException(status_code = 400, detail = "Pick a track to separate, not a saved voice.")
+ await _maybe_auto_switch_model(
+ _RELOAD_ONLY_MODEL, request, current_subject, claim_resident = False
+ )
+ backend = await asyncio.to_thread(get_inference_backend)
+ model_info = (
+ backend.models.get(backend.active_model_name, {}) if backend.active_model_name else {}
+ )
+ if "separate" not in (model_info.get("audio_workflows") or []):
+ raise HTTPException(status_code = 400, detail = "Load a model that can separate audio.")
+ model_name = _orchestrator_public_model_id(backend)
+ options = _separation_options(
+ body, model_info.get("audio_family"), _audio_model_label(model_name)
+ )
+ ref = source_ref.model_dump(exclude_none = True)
+ try:
+ source, prepared = await asyncio.to_thread(_prepare_separation_source, ref)
+ except audio_inputs.AudioInputError as exc:
+ if exc.status == 404 and source_ref.input_id:
+ raise HTTPException(status_code = 404, detail = "This track expired. Add it again.")
+ raise _audio_source_error(exc) from None
+
+ staging = await asyncio.to_thread(
+ lambda: audio_gallery.gallery_dir() / f".separate-{uuid.uuid4().hex}"
+ )
+ try:
+ await asyncio.to_thread(staging.mkdir, parents = True)
+ cancel = threading.Event()
+ with _TrackedCancel(cancel, model = model_name, kind = "audio"):
+ watcher = asyncio.create_task(_await_disconnect_then_cancel(request, cancel))
+ try:
+ outputs = await asyncio.to_thread(
+ backend.separate_audio_response, str(prepared), str(staging), options, cancel
+ )
+ except Exception as e:
+ if cancel.is_set() or isinstance(e, AudioGenerationCancelledError):
+ raise HTTPException(status_code = 499, detail = "Audio generation cancelled")
+ if isinstance(e, AudioBackendUnsupportedError):
+ raise HTTPException(status_code = 501, detail = e.message)
+ if isinstance(e, AudioRuntimeError):
+ status_code, detail = audio_runtime_http_error(e)
+ logger.warning("Audio separation refused by the runtime: %s", e)
+ raise HTTPException(status_code = status_code, detail = detail)
+ logger.error(f"Audio separation error: {e}", exc_info = True)
+ raise HTTPException(status_code = 500, detail = safe_error_detail(e))
+ finally:
+ await _stop_local_disconnect_cancel_watcher(watcher)
+ stems = _checked_stems(outputs, staging)
+ group_id = uuid.uuid4().hex
+ base_meta: dict[str, Any] = {
+ "prompt": source.name,
+ "model": model_name,
+ "audio_type": "audiocpp_sep",
+ "workflow": "separate",
+ "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+ "group_id": group_id,
+ "settings": {
+ "stems": [stem["id"] for stem in stems],
+ "num_overlap": (options or {}).get("num_overlap"),
+ },
+ }
+ if source.kind == "clip":
+ base_meta["source_clip_id"] = source.id
+ try:
+ records = await asyncio.to_thread(_save_stems, stems, base_meta)
+ except Exception as exc: # noqa: BLE001 - every stem of the group was rolled back
+ logger.warning("audio_gallery.separate_save_failed: %s", exc)
+ reason = sanitize_runtime_detail(str(exc)) or "unknown error"
+ raise HTTPException(
+ status_code = 500, detail = f"Could not save the stems to history: {reason}"
+ )
+ finally:
+ await asyncio.to_thread(shutil.rmtree, staging, True)
+ return AudioRunResponse(
+ clips = [
+ {
+ "id": record["id"],
+ "role": record["role"],
+ "url": record["url"],
+ "sample_rate": record["sample_rate"],
+ "duration_s": record["duration_s"],
+ "workflow": record["workflow"],
+ }
+ for record in records
+ ],
+ group_id = group_id,
+ model = model_name,
+ )
+
+
def _refuse_decision_connection(provider_type: Optional[str], api_type: Optional[str]) -> None:
if answers_decisions_only(provider_type, api_type):
raise HTTPException(
@@ -34403,7 +34661,7 @@ def _audio_cpp_speech_model_objects(created: int) -> list[dict]:
return []
objects = []
for model in downloaded_models():
- if model.audio_type is None:
+ if model.audio_type is None or model.task not in ("tts", "music"):
continue
if account_access.managed_account() and not account_access.model_visible(
model.repo_id or model.id
@@ -45162,7 +45420,7 @@ async def delete_gallery_audio(audio_id: str, current_subject: str = Depends(get
@studio_router.delete("/audio/gallery")
async def clear_gallery_audio(
- workflow: Optional[Literal["speak", "clone", "music"]] = None,
+ workflow: Optional[Literal["speak", "clone", "music", "separate"]] = None,
current_subject: str = Depends(get_current_subject),
):
from core.inference import audio_gallery
diff --git a/studio/backend/tests/test_audio_cpp_models.py b/studio/backend/tests/test_audio_cpp_models.py
index c93555cf05e..29db7a6670b 100644
--- a/studio/backend/tests/test_audio_cpp_models.py
+++ b/studio/backend/tests/test_audio_cpp_models.py
@@ -318,7 +318,8 @@ def test_an_unknown_family_follows_its_spec_tasks_or_is_refused(hub):
("new_clone_music", ["clone", "music"], "music", "gen"),
("new_clone_asr", ["clone", "asr"], "asr", "asr"),
("new_clone_only", ["clone"], "tts", "clon"),
- ("new_sep", ["sep"], "", ""),
+ ("new_sep", ["sep"], "sep", "sep"),
+ ("new_vad", ["vad"], "", ""),
):
repo = f"someone/{family}-GGUF"
_put(_snapshot(hub, repo), "m-q8_0.gguf", _gguf_bytes(family = family, spec = {"tasks": tasks}))
@@ -326,7 +327,10 @@ def test_an_unknown_family_follows_its_spec_tasks_or_is_refused(hub):
assert (model.task, model.server_task) == (task, server)
assert (model.unsupported is None) == bool(task)
sep = acm.resolve("someone/new_sep-GGUF", network = False)
- assert "Source separation" in sep.unsupported
+ assert list(sep.workflows) == ["separate"] and sep.separation is not None
+ acm.require_runnable(sep, "tts")
+ vad = acm.resolve("someone/new_vad-GGUF", network = False)
+ assert "Voice activity detection" in vad.unsupported
clone_only = acm.resolve("someone/new_clone_only-GGUF", network = False)
assert list(clone_only.workflows) == ["clone"] and clone_only.clone.reference_text == "optional"
@@ -446,13 +450,40 @@ def test_a_spec_fallback_family_the_runtime_lacks_is_refused(hub, monkeypatch, t
def test_known_task_families_are_refused_with_a_reason(hub):
- _put(_snapshot(hub), "HTDemucs-GGUF/htdemucs-q8_0.gguf", _gguf_bytes(family = "htdemucs"))
- model = acm.resolve(f"{AUDIO_CPP_REPO}/HTDemucs-GGUF", network = False)
- assert "Source separation" in model.unsupported
+ _put(
+ _snapshot(hub),
+ "Sortformer-Diar-GGUF/sortformer-diar-q8_0.gguf",
+ _gguf_bytes(family = "sortformer_diar"),
+ )
+ model = acm.resolve(f"{AUDIO_CPP_REPO}/Sortformer-Diar-GGUF", network = False)
+ assert "Speaker diarization" in model.unsupported
with pytest.raises(acm.AudioCppModelError):
acm.require_runnable(model, "tts")
+ "folder, family",
+ [
+ ("HTDemucs-GGUF", "htdemucs"),
+ ("HTDemucs-6stems-GGUF", "htdemucs_6stems"),
+ ("BS-RoFormer-ep368-GGUF", "bs_roformer"),
+ ("Mel-Band-RoFormer-GGUF", "mel_band_roformer"),
+ ],
+)
+def test_separation_families_resolve_runnable(hub, folder, family):
+ _put(_snapshot(hub), f"{folder}/{folder.lower()}-q8_0.gguf", _gguf_bytes(family = family))
+ model = acm.resolve(f"{AUDIO_CPP_REPO}/{folder}", network = False)
+ assert (model.family, model.task, model.server_task) == (family, "sep", "sep")
+ assert model.unsupported is None and model.options == ()
+ assert (model.audio_type, model.hub_task) == ("audiocpp_sep", "audio-to-audio")
+ assert model.workflows == {"separate": acm.WorkflowBinding("sep", "tasks", None, ("audio",))}
+ assert model.separation == acm.FAMILIES[family].separation
+ acm.require_runnable(model, "tts")
+ with pytest.raises(acm.AudioCppModelError, match = "not a speech-to-text model"):
+ acm.require_runnable(model, "asr")
+ assert acm.family_from_names([folder]) == family
+
+
def test_qwen3_tts_package_kind_comes_from_its_name(hub):
snap = _snapshot(hub)
for folder in (
@@ -496,8 +527,13 @@ _CLONE_TABLE = {
def test_every_task_family_binds_its_workflows():
- workflow_for = {"tts": "speak", "music": "music", "asr": "transcribe"}
- endpoint_for = {"speak": "speech", "music": "tasks", "transcribe": "transcriptions"}
+ workflow_for = {"tts": "speak", "music": "music", "asr": "transcribe", "sep": "separate"}
+ endpoint_for = {
+ "speak": "speech",
+ "music": "tasks",
+ "transcribe": "transcriptions",
+ "separate": "tasks",
+ }
for family in acm.FAMILIES.values():
bindings = family.workflows
if not family.task:
@@ -535,7 +571,9 @@ def test_every_task_family_binds_its_workflows():
assert acm.FAMILIES["moonshine_asr"].workflows["transcribe"].inputs == ("audio",)
design = acm.family_policy("qwen3_tts", names = ["Qwen3-TTS-12Hz-1.7B-VoiceDesign-GGUF"])
assert design.workflows["speak"].server_task == "vdes"
- assert acm.family_policy("htdemucs").workflows == {}
+ assert acm.family_policy("htdemucs").workflows == {
+ "separate": acm.WorkflowBinding("sep", "tasks", None, ("audio",))
+ }
def test_a_resolved_model_carries_its_workflow_binding(hub):
@@ -545,7 +583,9 @@ def test_a_resolved_model_carries_its_workflow_binding(hub):
model = acm.resolve(f"{AUDIO_CPP_REPO}/{folder}", network = False)
assert list(model.workflows) == ["speak"] and model.workflows["speak"].server_task == "vdes"
_put(snap, "HTDemucs-GGUF/htdemucs-q8_0.gguf", _gguf_bytes(family = "htdemucs"))
- assert acm.resolve(f"{AUDIO_CPP_REPO}/HTDemucs-GGUF", network = False).workflows == {}
+ assert list(acm.resolve(f"{AUDIO_CPP_REPO}/HTDemucs-GGUF", network = False).workflows) == [
+ "separate"
+ ]
def test_sub_folders_and_same_quant_files_become_named_variants(hub):
@@ -737,12 +777,12 @@ def test_windows_refuses_a_model_path_the_server_cannot_open(hub, monkeypatch):
def test_an_unsupported_model_is_a_runtime_problem(hub):
from core.inference import audio_cpp_server as srv
refused = _model(
- "HTDemucs-GGUF",
- "htdemucs",
+ "Silero-VAD-GGUF",
+ "silero_vad",
"",
- unsupported = "Source separation models are not supported in Studio yet.",
+ unsupported = "Voice activity detection models are not supported in Studio yet.",
)
- assert "Source separation" in srv.model_runtime_problem(refused, "x")
+ assert "Voice activity detection" in srv.model_runtime_problem(refused, "x")
def test_files_split_across_snapshots_are_found_in_the_one_that_holds_them(hub):
@@ -913,6 +953,21 @@ def test_materialize_refuses_a_repo_file_name_that_climbs_out_of_the_farm(
assert not victim.exists()
+def test_v1_models_lists_speech_but_not_separation_models(hub, monkeypatch):
+ from core.inference import audio_cpp_server
+ from routes import inference as ri
+
+ snap = _kokoro(hub)
+ _put(snap, "HTDemucs-GGUF/htdemucs-q8_0.gguf", _gguf_bytes(family = "htdemucs"))
+ assert {m.id: m.task for m in acm.downloaded_models()}[
+ f"{AUDIO_CPP_REPO}/HTDemucs-GGUF"
+ ] == "sep"
+ monkeypatch.setattr(audio_cpp_server, "find_audio_cpp_server_binary", lambda: "audiocpp_server")
+ monkeypatch.setattr(audio_cpp_server, "model_runtime_problem", lambda model, binary = None: None)
+ listed = [o["id"] for o in ri._audio_cpp_speech_model_objects(0)]
+ assert listed == [f"{AUDIO_CPP_REPO}/Kokoro-82M-GGUF"]
+
+
def test_downloaded_models_are_found_by_header(hub):
_kokoro(hub)
snap = _snapshot(hub)
@@ -1203,6 +1258,9 @@ def test_model_config_answers_umbrella_ids_without_the_hub(hub, monkeypatch):
# A dictation model is not a main-slot model.
with pytest.raises(ValueError, match = "speech-to-text"):
model_config.ModelConfig.from_identifier("audiocpp-canary-180m-flash")
+ _put(snap, "HTDemucs-GGUF/htdemucs-q8_0.gguf", _gguf_bytes(family = "htdemucs"))
+ sep = model_config.ModelConfig.from_identifier(f"{AUDIO_CPP_REPO}/HTDemucs-GGUF")
+ assert sep.is_audio and sep.audio_type == "audiocpp_sep" and sep.audio_cpp.task == "sep"
def test_model_config_routes_a_hub_repo_by_its_gguf_header(hub, monkeypatch):
@@ -1271,6 +1329,8 @@ def test_audio_cpp_ggufs_are_classified_off_chat(hub):
assert cc._gguf_path_audio_type(music) == "audiocpp_music"
assert cc._gguf_path_task(asr) == "automatic-speech-recognition"
assert cc._gguf_path_task(sep) == "audio-to-audio"
+ assert cc._gguf_path_audio_type(sep) == "audiocpp_sep"
+ assert cc._gguf_path_audio_workflows(sep) == ["separate"]
# Architecture alone (a remote prefix): the family comes from the names.
assert cc._arch_to_task("audiocpp", ("audio-cpp/MiniMax-Music3-GGUF",)) == "text-to-audio"
assert cc._arch_to_audio_type("audiocpp", ("Kokoro-82M-GGUF",)) == "audiocpp_tts"
diff --git a/studio/backend/tests/test_audio_cpp_outputs.py b/studio/backend/tests/test_audio_cpp_outputs.py
new file mode 100644
index 00000000000..807035c6dce
--- /dev/null
+++ b/studio/backend/tests/test_audio_cpp_outputs.py
@@ -0,0 +1,219 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Stems out of a saved ``/v1/tasks/run`` separation answer, decoded from disk."""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import struct
+import tracemalloc
+import wave
+
+import pytest
+
+from core.inference.audio_cpp_outputs import (
+ SeparationOutputError,
+ extract_named_outputs,
+ riff_info,
+)
+
+RATE = 44100
+
+
+def _wav(frames = 4410, fill = b"\x01\x00") -> bytes:
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as w:
+ w.setnchannels(2)
+ w.setsampwidth(2)
+ w.setframerate(RATE)
+ w.writeframes(fill * 2 * frames)
+ return buf.getvalue()
+
+
+def _float_wav(frames, channels = 2) -> bytes:
+ data = b"\x00\x00\x00\x00" * frames * channels
+ fmt = struct.pack("<HHIIHH", 3, channels, RATE, RATE * channels * 4, channels * 4, 32)
+ fmt += struct.pack("<H", 0) # cbSize: an 18-byte fmt chunk
+ body = b"WAVE" + b"fmt " + struct.pack("<I", len(fmt)) + fmt
+ body += b"data" + struct.pack("<I", len(data)) + data
+ return b"RIFF" + struct.pack("<I", len(body)) + body
+
+
+def _b64(data: bytes) -> str:
+ return base64.b64encode(data).decode("ascii")
+
+
+def _answer(tmp_path, payload):
+ """``payload`` is raw bytes or a JSON-able answer; returns (answer path, out dir)."""
+ path = tmp_path / "response.json"
+ path.write_bytes(payload if isinstance(payload, bytes) else json.dumps(payload).encode())
+ out = tmp_path / "out"
+ out.mkdir(exist_ok = True)
+ return path, out
+
+
+def _stems(ids):
+ return [{"id": i, "audio": _b64(_wav()), "sample_rate": RATE, "channels": 2} for i in ids]
+
+
+ "ids",
+ [
+ ["drums", "bass", "other", "vocals"],
+ ["drums", "bass", "other", "vocals", "guitar", "piano"],
+ ],
+)
+def test_every_named_stem_lands_in_its_own_wav(tmp_path, ids):
+ path, out = _answer(
+ tmp_path, {"named_audio_outputs": _stems(ids), "timing": {"wall_ms": 365.5}}
+ )
+ outputs = extract_named_outputs(path, out)
+ assert [o["id"] for o in outputs] == ids
+ for output in outputs:
+ assert set(output) == {"id", "path", "sample_rate", "channels", "duration_s"}
+ assert output["path"] == str(out / f"{output['id']}.wav")
+ assert (output["sample_rate"], output["channels"], output["duration_s"]) == (RATE, 2, 0.1)
+ with wave.open(output["path"]) as w:
+ assert (w.getframerate(), w.getnchannels(), w.getnframes()) == (RATE, 2, 4410)
+
+
+_FF = _wav(fill = b"\xff\xff") # base64 holds "////", which a JSON writer may escape as "\/"
+_PRETTY = {"timing": {}, "named_audio_outputs": [{"audio": _b64(_wav()), "channels": 2, "id": "v"}]}
+
+
+ "raw, stem, data, duration",
+ [
+ pytest.param(json.dumps(_PRETTY, indent = 4), "v", _wav(), 0.1, id = "key_order_and_indent"),
+ pytest.param(
+ '{ "named_audio_outputs" :\n [ { "id" : "drums" ,\n "audio" : "'
+ + _b64(_wav())
+ + '" } ] }',
+ "drums",
+ _wav(),
+ 0.1,
+ id = "spaced",
+ ),
+ pytest.param(
+ '{"named_audio_outputs":[{"id":"v","audio":"' + _b64(_FF).replace("/", "\\/") + '"}]}',
+ "v",
+ _FF,
+ 0.1,
+ id = "escaped_slashes",
+ ),
+ pytest.param(
+ json.dumps(
+ {
+ "named_audio_outputs": [
+ {"id": "v", "audio": "data:audio/wav;base64," + _b64(_wav())}
+ ]
+ }
+ ),
+ "v",
+ _wav(),
+ 0.1,
+ id = "data_uri_prefix",
+ ),
+ pytest.param(
+ json.dumps({"named_audio_outputs": [{"id": "v", "audio": _b64(_float_wav(22050))}]}),
+ "v",
+ _float_wav(22050),
+ 0.5,
+ id = "float_wav_duration",
+ ),
+ pytest.param(
+ json.dumps(
+ {
+ "audio": _b64(_wav()),
+ "named_audio_outputs": [{"id": "audio_0", "audio": _b64(_wav())}],
+ }
+ ),
+ "audio_0",
+ _wav(),
+ 0.1,
+ id = "top_level_audio_not_counted_twice",
+ ),
+ ],
+)
+def test_answer_shapes_decode_to_one_stem(tmp_path, raw, stem, data, duration):
+ path, out = _answer(tmp_path, raw.encode())
+ (output,) = extract_named_outputs(path, out, chunk = 1001)
+ assert (output["id"], output["sample_rate"], output["channels"]) == (stem, RATE, 2)
+ assert output["duration_s"] == duration
+ assert [p.name for p in out.iterdir()] == [f"{stem}.wav"]
+ assert (out / f"{stem}.wav").read_bytes() == data
+
+
+ "payload, match",
+ [
+ ({"audio": _b64(_wav())}, "returned no stems"),
+ ({"named_audio_outputs": []}, "returned no stems"),
+ (
+ {"named_audio_outputs": [{"id": "v", "audio": _b64(b"not a wav at all")}]},
+ "returned no stems",
+ ),
+ ({"named_audio_outputs": [{"id": "vocals"}]}, "returned no stems"),
+ ({"named_audio_outputs": [{"audio": _b64(_wav())}]}, "returned no stems"),
+ (b"", None),
+ (b"{not json", None),
+ (b'{"named_audio_outputs":[{"id":"x","audio":"QUJD', None),
+ pytest.param(
+ {
+ "named_audio_outputs": [
+ {"id": "v", "audio": _b64(_wav())},
+ {"id": "d", "audio": _b64(b"garbage!")},
+ ]
+ },
+ None,
+ id = "a_bad_stem_after_a_good_one_removes_both",
+ ),
+ ],
+)
+def test_a_broken_answer_is_an_error_and_leaves_nothing(tmp_path, payload, match):
+ path, out = _answer(tmp_path, payload)
+ with pytest.raises(SeparationOutputError, match = match):
+ extract_named_outputs(path, out)
+ assert list(out.iterdir()) == []
+
+
+def test_duplicate_and_unsafe_ids_become_safe_unique_names(tmp_path):
+ payload = {"named_audio_outputs": _stems(["vocals", "vocals", "../x", "a b/c"])}
+ path, out = _answer(tmp_path, payload)
+ ids = [o["id"] for o in extract_named_outputs(path, out)]
+ assert ids == ["vocals", "vocals_2", "___x", "a_b_c"]
+ assert sorted(p.name for p in out.iterdir()) == sorted(f"{i}.wav" for i in ids)
+ assert not (tmp_path / "x.wav").exists()
+
+
+def test_a_large_answer_decodes_in_bounded_memory(tmp_path):
+ # Two ~30 MB base64 stems (~46 MB of audio): never held whole in memory.
+ frames = 5_700_000
+ stem = _wav(frames, fill = b"\x12\x34")
+ encoded = base64.b64encode(stem)
+ path = tmp_path / "response.json"
+ with open(path, "wb") as f:
+ f.write(b'{"named_audio_outputs":[')
+ for index, name in enumerate((b"vocals", b"instrumental")):
+ if index:
+ f.write(b",")
+ f.write(b'{"id":"' + name + b'","audio":"')
+ f.write(encoded)
+ f.write(b'","sample_rate":44100,"channels":2}')
+ f.write(b'],"timing":{"wall_ms":1}}')
+ assert path.stat().st_size > 60_000_000
+ del stem, encoded
+ out = tmp_path / "out"
+ out.mkdir()
+ tracemalloc.start()
+ try:
+ outputs = extract_named_outputs(path, out)
+ _current, peak = tracemalloc.get_traced_memory()
+ finally:
+ tracemalloc.stop()
+ assert peak < 16 * 1024 * 1024, peak
+ assert [o["id"] for o in outputs] == ["vocals", "instrumental"]
+ assert riff_info(out / "vocals.wav")["frames"] == frames
diff --git a/studio/backend/tests/test_audio_cpp_separation.py b/studio/backend/tests/test_audio_cpp_separation.py
new file mode 100644
index 00000000000..103aaad8d43
--- /dev/null
+++ b/studio/backend/tests/test_audio_cpp_separation.py
@@ -0,0 +1,303 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The exact request audio.cpp receives for a separation, against a recording fake server."""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import threading
+import wave
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+import pytest
+
+from core.inference import audio_cpp_backend as acb
+from core.inference import audio_cpp_files
+from core.inference import audio_cpp_models as acm
+from core.inference import audio_cpp_server as srv
+from core.inference.audio_cpp_models import AUDIO_CPP_REPO, AudioCppModel, AudioCppVariant, RepoFile
+from core.inference.audio_errors import AudioGenerationCancelledError, AudioRuntimeError
+
+SOURCE = "/srv/accounts/a/audio/inputs/0123.44100.stereo.wav"
+_REAL_START = srv.AudioCppServer.start
+
+
+def _wav(frames = 441) -> bytes:
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as w:
+ w.setnchannels(2)
+ w.setsampwidth(2)
+ w.setframerate(44100)
+ w.writeframes(b"\x01\x00" * 2 * frames)
+ return buf.getvalue()
+
+
+def _answer(ids) -> bytes:
+ stems = [
+ {
+ "id": stem,
+ "audio": base64.b64encode(_wav()).decode(),
+ "sample_rate": 44100,
+ "channels": 2,
+ }
+ for stem in ids
+ ]
+ return json.dumps({"named_audio_outputs": stems, "timing": {"wall_ms": 1.0}}).encode()
+
+
+def _model(family, display = None) -> AudioCppModel:
+ policy = acm.family_policy(family)
+ variant = AudioCppVariant("Q8_0", (RepoFile("m-q8_0.gguf", 1),), "m-q8_0.gguf")
+ return AudioCppModel(
+ id = f"{AUDIO_CPP_REPO}/{display or family}",
+ repo_id = AUDIO_CPP_REPO,
+ folder = display or family,
+ display_name = display or family,
+ family = family,
+ task = policy.task,
+ server_task = policy.default_server_task,
+ variant = variant,
+ variants = (variant,),
+ default_variant = "Q8_0",
+ model_options = dict(policy.model_options),
+ separation = policy.separation,
+ )
+
+
+class _Recorder:
+ model_id = "studio-test"
+ backend = "cpu"
+
+ def __init__(
+ self,
+ model,
+ answer = None,
+ error = None,
+ ):
+ self.model = model
+ self.answer = answer
+ self.error = error
+ self.calls: list[tuple[str, dict]] = []
+ self.stopped = False
+
+ def alive(self):
+ return not self.stopped
+
+ def stop(self):
+ self.stopped = True
+
+ def post_json(self, *_args, **_kwargs):
+ raise AssertionError("a separation streams its answer to disk")
+
+ def post_json_to_file(self, path, payload, dest, **_kwargs):
+ self.calls.append((path, json.loads(json.dumps(payload))))
+ Path(dest).write_bytes(b'{"named_audio_outputs": [') # a partial answer on disk
+ if self.error is not None:
+ raise self.error
+ Path(dest).write_bytes(self.answer)
+ return "application/json"
+
+
+def started(monkeypatch, tmp_path):
+ """Every server start, as ``(served model, recorder)``; the answer comes from the family."""
+ starts: list[tuple[AudioCppModel, _Recorder]] = []
+ answers = {
+ "htdemucs": ["drums", "bass", "other", "vocals"],
+ "bs_roformer": ["vocals", "instrumental"],
+ }
+
+ def start(served, path, **_kwargs):
+ server = _Recorder(served, _answer(answers[served.family]))
+ starts.append((served, server))
+ return server
+
+ monkeypatch.setattr(audio_cpp_files, "materialize", lambda m: str(tmp_path / "m.gguf"))
+ monkeypatch.setattr(acb.AudioCppServer, "start", start)
+ return starts
+
+
+def _backend(model):
+ backend = acb.AudioCppBackend()
+ backend._model = model
+ backend.models = {model.id: {"is_audio": True}}
+ backend.active_model_name = model.id
+ return backend
+
+
+def _out(tmp_path) -> Path:
+ out = tmp_path / ".separate-x"
+ out.mkdir(exist_ok = True)
+ return out
+
+
+def test_the_body_is_exactly_the_model_and_the_track_and_htdemucs_never_restarts(started, tmp_path):
+ backend = _backend(_model("htdemucs", "HTDemucs-GGUF"))
+ out = _out(tmp_path)
+ outputs = backend.separate_audio(SOURCE, str(out), {"num_overlap": 2})
+ ((_served, server),) = started
+ ((path, body),) = server.calls
+ assert path == "/v1/tasks/run"
+ assert body == {"model": "studio-test", "audio": SOURCE}
+ assert [o["id"] for o in outputs] == ["drums", "bass", "other", "vocals"]
+ for output in outputs:
+ assert Path(output["path"]).parent == out and Path(output["path"]).is_file()
+ assert (output["sample_rate"], output["channels"], output["duration_s"]) == (44100, 2, 0.01)
+ assert not (out / ".response.json").exists()
+ backend.separate_audio(SOURCE, str(_out(tmp_path)))
+ backend.separate_audio(SOURCE, str(_out(tmp_path)), {"num_overlap": 1})
+ assert len(started) == 1
+ assert started[0][0].model_options.get("session_options") in (None, {})
+
+
+def test_roformer_overlap_is_a_session_option_that_restarts_only_on_change(started, tmp_path):
+ backend = _backend(_model("bs_roformer", "BS-RoFormer-ep368-GGUF"))
+ backend._start_server(backend._model)
+ assert len(started) == 1
+ backend.separate_audio(SOURCE, str(_out(tmp_path)))
+ assert len(started) == 1
+ backend.separate_audio(SOURCE, str(_out(tmp_path)), {"num_overlap": 1})
+ assert len(started) == 2
+ assert started[1][0].model_options["session_options"] == {"num_overlap": "1"}
+ backend.separate_audio(SOURCE, str(_out(tmp_path)), {"num_overlap": 1})
+ assert len(started) == 2
+ backend.separate_audio(SOURCE, str(_out(tmp_path)))
+ assert len(started) == 3
+ assert started[2][0].model_options.get("session_options") == {}
+ for _served, server in started[1:]:
+ for _path, body in server.calls:
+ assert set(body) == {"model", "audio"}
+
+
+def test_the_server_entry_runs_the_sep_task_with_the_overlap(monkeypatch, tmp_path):
+ seen = {}
+ binary = tmp_path / srv.BINARY_NAME
+ binary.write_bytes(b"")
+
+ def spy(command, *_args, **_kwargs):
+ with open(command[command.index("--config") + 1], encoding = "utf-8") as f:
+ seen.update(json.load(f))
+ raise OSError("not launched in tests")
+
+ monkeypatch.setattr(srv, "ensure_binary", lambda: str(binary))
+ monkeypatch.setattr(srv, "model_runtime_problem", lambda model, binary = None: None)
+ monkeypatch.setattr(srv, "select_backend", lambda binary, force_cpu: "cpu")
+ monkeypatch.setattr(srv.subprocess, "Popen", spy)
+ from dataclasses import replace
+
+ model = replace(
+ _model("mel_band_roformer", "Mel-Band-RoFormer-GGUF"),
+ model_options = {"session_options": {"num_overlap": "1"}},
+ )
+ with pytest.raises(srv.AudioCppUnavailableError):
+ _REAL_START(model, "/models/m-q8_0.gguf")
+ (entry,) = seen["models"]
+ assert (entry["family"], entry["task"], entry["mode"]) == (
+ "mel_band_roformer",
+ "sep",
+ "offline",
+ )
+ assert entry["session_options"] == {"num_overlap": "1"}
+
+
+def test_a_cancel_stops_the_server_and_removes_the_answer(started, tmp_path, monkeypatch):
+ backend = _backend(_model("htdemucs", "HTDemucs-GGUF"))
+ backend._start_server(backend._model)
+ server = started[0][1]
+ cancel = threading.Event()
+
+ def cancelled(path, payload, dest, **_kwargs):
+ Path(dest).write_bytes(b"{")
+ cancel.set()
+ raise srv.AudioCppRequestCancelledError("Request cancelled.")
+
+ server.post_json_to_file = cancelled
+ out = _out(tmp_path)
+ with pytest.raises(AudioGenerationCancelledError):
+ backend.separate_audio(SOURCE, str(out), cancel_event = cancel)
+ assert server.stopped and backend._server is None
+ assert list(out.iterdir()) == []
+
+
+def test_a_runtime_refusal_keeps_its_reason(started, tmp_path):
+ backend = _backend(_model("htdemucs", "HTDemucs-GGUF"))
+ backend._start_server(backend._model)
+ started[0][1].error = srv.AudioCppRequestError(
+ 400, "HTDemucs prepare() sample rate mismatch: expected 44100, got 48000"
+ )
+ with pytest.raises(AudioRuntimeError) as caught:
+ backend.separate_audio(SOURCE, str(_out(tmp_path)))
+ assert caught.value.status == 400
+ assert caught.value.detail == (
+ "The audio runtime could not separate the track: "
+ "HTDemucs prepare() sample rate mismatch: expected 44100, got 48000"
+ )
+
+
+def test_an_answer_without_stems_is_an_empty_list(started, tmp_path):
+ backend = _backend(_model("htdemucs", "HTDemucs-GGUF"))
+ backend._start_server(backend._model)
+ started[0][1].answer = json.dumps({"audio": base64.b64encode(b"nope").decode()}).encode()
+ out = _out(tmp_path)
+ assert backend.separate_audio(SOURCE, str(out)) == []
+ assert list(out.iterdir()) == []
+
+
+class _Handler(BaseHTTPRequestHandler):
+ body = b""
+ status = 200
+
+ def log_message(self, *_args):
+ pass
+
+ def do_POST(self):
+ self.rfile.read(int(self.headers.get("Content-Length") or 0))
+ self.send_response(self.status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(self.body)))
+ self.end_headers()
+ self.wfile.write(self.body)
+
+
+def http_server():
+ httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
+ thread = threading.Thread(target = httpd.serve_forever, daemon = True)
+ thread.start()
+
+ class _Process:
+ pid = 0
+
+ def poll(self):
+ return None
+
+ server = srv.AudioCppServer(
+ _Process(), httpd.server_address[1], None, "m", "cpu", Path("/nonexistent")
+ )
+ yield server
+ httpd.shutdown()
+
+
+def test_post_json_to_file_streams_a_large_answer(http_server, tmp_path):
+ _Handler.status, _Handler.body = 200, b'{"named_audio_outputs":[]}' + b" " * (3 << 20)
+ dest = tmp_path / "answer.json"
+ ctype = http_server.post_json_to_file(
+ "/v1/tasks/run", {"model": "m"}, dest, cancel_event = threading.Event()
+ )
+ assert ctype == "application/json"
+ assert dest.read_bytes() == _Handler.body
+
+
+def test_post_json_to_file_raises_the_runtime_error(http_server, tmp_path):
+ _Handler.status = 400
+ _Handler.body = json.dumps(
+ {"error": {"message": "unknown htdemucs request option: seed"}}
+ ).encode()
+ with pytest.raises(srv.AudioCppRequestError) as caught:
+ http_server.post_json_to_file("/v1/tasks/run", {"model": "m"}, tmp_path / "a.json")
+ assert caught.value.status == 400
+ assert caught.value.detail == "unknown htdemucs request option: seed"
diff --git a/studio/backend/tests/test_audio_gallery.py b/studio/backend/tests/test_audio_gallery.py
index 5cb0115281a..22f6c3ff51d 100644
--- a/studio/backend/tests/test_audio_gallery.py
+++ b/studio/backend/tests/test_audio_gallery.py
@@ -793,3 +793,90 @@ def test_the_inputs_and_voices_folders_never_list_as_clips():
assert len(gallery.list_audio()) == 1
assert gallery.clear() == 1
assert (gallery.gallery_dir() / "inputs" / "abc.wav").is_file()
+
+
+def _stem_meta(**over):
+ return _meta(
+ audio_type = "audiocpp_sep",
+ workflow = "separate",
+ sample_rate = 44100,
+ role = "vocals",
+ group_id = "g" * 32,
+ settings = {"stems": ["vocals", "instrumental"], "num_overlap": None},
+ **over,
+ )
+
+
+def test_save_file_copies_across_filesystems(tmp_path, monkeypatch):
+ import errno
+
+ src = tmp_path / "vocals.wav"
+ src.write_bytes(_wav())
+ real_replace = os.replace
+
+ def cross_device(a, b):
+ if Path(a) == src:
+ raise OSError(errno.EXDEV, "Invalid cross-device link")
+ return real_replace(a, b)
+
+ monkeypatch.setattr(gallery.os, "replace", cross_device)
+ record = gallery.save_file(src, _stem_meta())
+ assert gallery.audio_path(record["id"]).read_bytes() == _wav()
+ assert not list(gallery.gallery_dir().glob(".*.tmp"))
+
+
+def test_save_file_rolls_back_when_the_sidecar_fails(tmp_path, monkeypatch):
+ src = tmp_path / "vocals.wav"
+ src.write_bytes(_wav())
+ gallery.gallery_dir()
+
+ def broken(self, *_args, **_kwargs):
+ raise OSError(28, "No space left on device")
+
+ monkeypatch.setattr(Path, "write_text", broken)
+ with pytest.raises(OSError):
+ gallery.save_file(src, _stem_meta())
+ monkeypatch.undo()
+ monkeypatch.setattr(gallery, "studio_root", lambda: tmp_path)
+ assert gallery.list_audio() == []
+ assert not [p for p in gallery.gallery_dir().iterdir() if p.suffix in (".wav", ".json", ".tmp")]
+
+
+def test_save_file_prunes_only_when_asked(tmp_path, monkeypatch):
+ pruned = []
+ monkeypatch.setattr(gallery, "_prune_to_cap", lambda: pruned.append(1) or 0)
+ for prune in (False, True):
+ src = tmp_path / f"{prune}.wav"
+ src.write_bytes(_wav())
+ gallery.save_file(src, _stem_meta(), prune = prune)
+ assert pruned == [1]
+
+
+def test_prune_keeps_or_drops_a_group_whole(tmp_path, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_AUDIO_GALLERY_MAX_CLIPS", "3")
+
+ def group(gid, t):
+ ids = []
+ for i, role in enumerate(("vocals", "drums", "bass", "other")):
+ src = tmp_path / f"{gid}{role}.wav"
+ src.write_bytes(_wav())
+ meta = {**_stem_meta(), "group_id": gid, "role": role}
+ record = gallery.save_file(src, meta, prune = False)
+ os.utime(gallery.gallery_dir() / f"{record['id']}.wav", (t + i, t + i))
+ ids.append(record["id"])
+ return ids
+
+ old = group("a" * 32, 100.0)
+ new = group("b" * 32, 200.0)
+ gallery._prune_to_cap()
+ assert all(gallery.audio_path(i) is not None for i in new)
+ assert all(gallery.audio_path(i) is None for i in old)
+
+
+def test_a_separate_scoped_clear_spares_speak_and_clone():
+ speech = gallery.save(_wav(), _meta())
+ clone = gallery.save(_wav(), _meta(audio_type = "audiocpp_tts", workflow = "clone"))
+ stem = gallery.save(_wav(), _stem_meta())
+ assert gallery.clear(workflow = "separate") == 1
+ assert gallery.audio_path(stem["id"]) is None
+ assert {r["id"] for r in gallery.list_audio()} == {speech["id"], clone["id"]}
diff --git a/studio/backend/tests/test_audio_run_route.py b/studio/backend/tests/test_audio_run_route.py
index dba276d82af..81b16c5bd79 100644
--- a/studio/backend/tests/test_audio_run_route.py
+++ b/studio/backend/tests/test_audio_run_route.py
@@ -888,6 +888,463 @@ def test_the_orchestrator_sends_music_and_merges_the_reload_status(monkeypatch):
assert orchestrator.models["m"] == {"is_audio": True, "audio_music": reloaded}
+HTDEMUCS_6 = "audio-cpp/audio.cpp-gguf/HTDemucs-6stems-GGUF"
+BS_ROFORMER = "audio-cpp/audio.cpp-gguf/BS-RoFormer-ep368-GGUF"
+SIX_STEMS = ["drums", "bass", "other", "vocals", "guitar", "piano"]
+
+
+def _sep_info(family = "htdemucs_6stems"):
+ return {
+ "is_audio": True,
+ "audio_type": "audiocpp_sep",
+ "audio_family": family,
+ "audio_workflows": ["separate"],
+ "audio_options": [],
+ "audio_clone": None,
+ }
+
+
+def _stem(
+ path,
+ stem = "vocals",
+ data = None,
+ duration_s = 0.01,
+) -> dict:
+ """A worker stem entry; writes ``data`` (a WAV by default) to ``path``."""
+ Path(path).write_bytes(_wav() if data is None else data)
+ return {
+ "id": stem,
+ "path": str(path),
+ "sample_rate": 44100,
+ "channels": 2,
+ "duration_s": duration_s,
+ }
+
+
+class _SepBackend(_Backend):
+ def __init__(
+ self,
+ name,
+ info,
+ stems = SIX_STEMS,
+ ):
+ super().__init__(name, info)
+ self.stems = stems
+ self.separations: list[dict] = []
+ self.outputs = None
+
+ def separate_audio_response(
+ self,
+ source_path,
+ output_dir,
+ audio_options = None,
+ cancel_event = None,
+ ):
+ self.separations.append(
+ {"source_path": source_path, "output_dir": output_dir, "audio_options": audio_options}
+ )
+ if self.outputs is not None:
+ return self.outputs(output_dir)
+ return [_stem(Path(output_dir) / f"{s}.wav", s) for s in self.stems]
+
+
+def sep(stub):
+ backend = _SepBackend(HTDEMUCS_6, _sep_info())
+ stub["backend"] = backend
+ return backend
+
+
+def _separate(
+ client,
+ source = None,
+ **body,
+):
+ if source is not None:
+ body["inputs"] = {"source": source}
+ return client.post("/api/inference/audio/run", json = {"workflow": "separate", **body})
+
+
+def _wav_input(
+ account,
+ seconds,
+ rate = 8000,
+ layout = "mono",
+) -> str:
+ data = encode("wav", "pcm_s16le", rate, layout, seconds)
+ save = lambda: asyncio.run(audio_inputs.save_stream(_chunks([data]), "track.wav"))[0]["id"]
+ return run_as(account, save)
+
+
+def _gallery_root(tmp_path) -> Path:
+ return tmp_path / "accounts" / ALICE.account_id / "audio"
+
+
+def _wav_info(path) -> tuple:
+ with wave.open(str(path), "rb") as w:
+ return w.getframerate(), w.getnchannels(), w.getnframes()
+
+
+def _staging_and_stems(tmp_path):
+ root = _gallery_root(tmp_path)
+ return list(root.glob(".separate-*")), sorted(p.name for p in root.glob("*.wav"))
+
+
+def test_a_separation_prepares_44k_and_saves_every_stem_as_one_group(sep, tmp_path):
+ input_id = _input(ALICE, 2.0) # 48 kHz stereo
+ with _client(ALICE) as client:
+ response = _separate(client, {"input_id": input_id}, seed = 3)
+ assert response.status_code == 200, response.text
+ body = response.json()
+ stem_bytes = client.get(body["clips"][0]["url"])
+ listed = client.get("/api/inference/audio/gallery").text
+ (call,) = sep.separations
+ assert sep.calls == []
+ source = Path(call["source_path"])
+ assert source.is_absolute() and source.parent == _gallery_root(tmp_path) / "inputs"
+ assert source.name.startswith(f"{input_id}.44100.stereo")
+ rate, channels, frames = _wav_info(source)
+ assert (rate, channels) == (44100, 2) and abs(frames - 2 * 44100) <= 2
+ staging = Path(call["output_dir"])
+ assert staging.parent == _gallery_root(tmp_path) and staging.name.startswith(".separate-")
+ assert not staging.exists()
+ assert call["audio_options"] is None
+ order = ["vocals", "drums", "bass", "guitar", "piano", "other"]
+ assert [clip["role"] for clip in body["clips"]] == order
+ assert body["audio"] is None and body["model"] and len(body["group_id"]) == 32
+ assert stem_bytes.status_code == 200 and stem_bytes.content[:4] == b"RIFF"
+ for clip in body["clips"]:
+ assert set(clip) == {"id", "role", "url", "sample_rate", "duration_s", "workflow"}
+ assert (clip["workflow"], clip["sample_rate"], clip["duration_s"]) == (
+ "separate",
+ 44100,
+ 0.01,
+ )
+ sidecar = (_gallery_root(tmp_path) / f"{clip['id']}.json").read_text(encoding = "utf-8")
+ meta = json.loads(sidecar)
+ assert meta["workflow"] == "separate" and meta["audio_type"] == "audiocpp_sep"
+ assert meta["role"] == clip["role"] and meta["group_id"] == body["group_id"]
+ assert meta["settings"] == {"stems": order, "num_overlap": None}
+ assert meta["prompt"] == "me.webm"
+ assert not any(isinstance(v, str) and v.startswith("/") for v in meta.values())
+ assert str(tmp_path) not in sidecar and ".separate-" not in sidecar
+ for text in (response.text, listed):
+ assert str(tmp_path) not in text and ".separate-" not in text
+ # The listing keeps the group, so history shows the run as one item.
+ stems = [c for c in json.loads(listed)["audio"] if c["workflow"] == "separate"]
+ assert {c["group_id"] for c in stems} == {body["group_id"]}
+ assert sorted(c["role"] for c in stems) == sorted(SIX_STEMS)
+
+
+def test_a_mono_track_reaches_the_runtime_as_44k_mono(sep):
+ input_id = _wav_input(ALICE, 1.0, rate = 22050, layout = "mono")
+ with _client(ALICE) as client:
+ assert _separate(client, {"input_id": input_id}).status_code == 200
+ rate, channels, frames = _wav_info(sep.separations[0]["source_path"])
+ assert (rate, channels) == (44100, 1) and abs(frames - 44100) <= 1
+
+
+def test_a_history_clip_can_be_separated_and_is_recorded(sep, tmp_path):
+ clip_id = _alice_sources()["clip_id"]
+ with _client(ALICE) as client:
+ response = _separate(client, {"clip_id": clip_id})
+ assert response.status_code == 200, response.text
+ sidecar = _gallery_root(tmp_path) / f"{response.json()['clips'][0]['id']}.json"
+ meta = json.loads(sidecar.read_text(encoding = "utf-8"))
+ assert meta["source_clip_id"] == clip_id and meta["prompt"] == "alice said"
+
+
+ "body, detail",
+ [
+ ({"text": "hello"}, "Separate takes no text."),
+ ({}, "Add a track to separate."),
+ (
+ {"inputs": {"source": {"voice_id": "a" * 32}}},
+ "Pick a track to separate, not a saved voice.",
+ ),
+ (
+ {"inputs": {"source": {"input_id": "a" * 32}, "reference": {"input_id": "a" * 32}}},
+ "Separate takes only a track.",
+ ),
+ (
+ {"inputs": {"source": {"input_id": "a" * 32}}, "mode": "song"},
+ "A music mode, lyrics, variations and edits apply to Music only.",
+ ),
+ ],
+)
+def test_a_separation_request_without_a_track_is_a_400(sep, body, detail):
+ with _client(ALICE) as client:
+ response = _separate(client, **body)
+ assert (response.status_code, response.json()["detail"]) == (400, detail)
+ assert sep.separations == []
+
+
+def test_a_clone_or_speak_run_still_needs_text(stub):
+ with _client(ALICE) as client:
+ response = client.post("/api/inference/audio/run", json = {"workflow": "speak"})
+ assert response.status_code == 422
+ assert stub["backend"].calls == []
+
+
+def test_separation_needs_a_separation_model(stub):
+ input_id = _input(ALICE)
+ stub["use"](*_KOKORO)
+ with _client(ALICE) as client:
+ response = _separate(client, {"input_id": input_id})
+ assert (response.status_code, response.json()["detail"]) == (
+ 400,
+ "Load a model that can separate audio.",
+ )
+ assert stub["backend"].calls == []
+
+
+def test_options_only_for_a_family_that_takes_them(sep, stub):
+ source = {"input_id": _input(ALICE)}
+ with _client(ALICE) as client:
+ refused = _separate(client, source, options = {"num_overlap": 1})
+ assert (refused.status_code, refused.json()["detail"]) == (
+ 400,
+ "HTDemucs-6stems has no separation options.",
+ )
+ roformer = _SepBackend(BS_ROFORMER, _sep_info("bs_roformer"), ["vocals", "instrumental"])
+ stub["backend"] = roformer
+ with _client(ALICE) as client:
+ unknown = _separate(client, source, options = {"shifts": 2})
+ assert (unknown.status_code, unknown.json()["detail"]) == (400, "Unknown option 'shifts'.")
+ assert _separate(client, source, options = {"num_overlap": 9}).status_code == 400
+ ok = _separate(client, source, options = {"num_overlap": 1})
+ assert ok.status_code == 200, ok.text
+ (call,) = roformer.separations
+ assert call["audio_options"] == {"num_overlap": 1}
+ assert [clip["role"] for clip in ok.json()["clips"]] == ["vocals", "instrumental"]
+ assert sep.separations == []
+
+
+def test_a_track_over_ten_minutes_is_refused(sep):
+ input_id = _wav_input(ALICE, 601.0)
+ with _client(ALICE) as client:
+ long = _separate(client, {"input_id": input_id})
+ assert (long.status_code, long.json()["detail"]) == (
+ 400,
+ "Separate tracks up to 10 minutes long. This one is 10:01. Pick a shorter track.",
+ )
+ ten = _separate(client, {"input_id": _wav_input(ALICE, 600.0)})
+ assert ten.status_code == 200, ten.text
+ (call,) = sep.separations
+ assert abs(_wav_info(call["source_path"])[2] - 600 * 44100) <= 600
+
+
+def test_another_accounts_track_is_404(sep):
+ sources = _alice_sources()
+ with _client(BOB) as client:
+ for kind in ("input_id", "clip_id"):
+ response = _separate(client, {kind: sources[kind]})
+ assert response.status_code == 404, response.text
+ expired = _separate(client, {"input_id": "c" * 32})
+ assert expired.json()["detail"] == "This track expired. Add it again."
+ assert sep.separations == []
+
+
+def test_another_account_cannot_fetch_a_stem(sep):
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ url = _separate(client, {"input_id": input_id}).json()["clips"][0]["url"]
+ assert client.get(url).status_code == 200
+ with _client(BOB) as client:
+ assert client.get(url).status_code == 404
+
+
+ "outputs",
+ [
+ pytest.param(lambda tmp, out: [], id = "no_stems"),
+ pytest.param(lambda tmp, out: [_stem(tmp / "elsewhere.wav")], id = "outside_staging"),
+ pytest.param(lambda tmp, out: [_stem(Path(out) / "v.wav", data = b"junk")], id = "not_a_wav"),
+ ],
+)
+def test_unusable_stems_are_a_502(sep, tmp_path, outputs):
+ sep.outputs = lambda out: outputs(tmp_path, out)
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ response = _separate(client, {"input_id": input_id})
+ assert (response.status_code, response.json()["detail"]) == (
+ 502,
+ "The audio runtime returned no stems.",
+ )
+ assert _staging_and_stems(tmp_path) == ([], [])
+
+
+def test_a_cancel_is_a_499_and_leaves_no_staging(sep, tmp_path):
+ from core.inference.audio_errors import AudioGenerationCancelledError
+
+ def cancelled(out):
+ _stem(Path(out) / "drums.wav")
+ raise AudioGenerationCancelledError("Audio generation cancelled")
+
+ sep.outputs = cancelled
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ assert _separate(client, {"input_id": input_id}).status_code == 499
+ assert _staging_and_stems(tmp_path) == ([], [])
+
+
+def test_a_failed_save_rolls_back_the_whole_group(sep, tmp_path, monkeypatch):
+ real = audio_gallery.save_file
+ saved = []
+
+ def flaky(
+ src,
+ meta,
+ *,
+ prune = True,
+ ):
+ assert prune is False
+ if len(saved) == 2:
+ raise OSError(28, "No space left on device")
+ record = real(src, meta, prune = prune)
+ saved.append(record["id"])
+ return record
+
+ monkeypatch.setattr(audio_gallery, "save_file", flaky)
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ response = _separate(client, {"input_id": input_id})
+ assert (response.status_code, response.json()["detail"]) == (
+ 500,
+ "Could not save the stems to history: [Errno 28] No space left on device",
+ )
+ assert len(saved) == 2
+ assert _staging_and_stems(tmp_path) == ([], [])
+ assert not list(_gallery_root(tmp_path).glob("*.json"))
+
+
+def test_a_separation_model_refuses_speech_and_points_at_separate(sep):
+ with _client(ALICE) as client:
+ generate = client.post(_GENERATE[0], json = _GENERATE[1])
+ speech = client.post(_SPEECH[0], json = _SPEECH[1])
+ speak = _run(client, workflow = "speak")
+ assert (generate.status_code, generate.json()["detail"]) == (
+ 400,
+ "HTDemucs-6stems separates audio into stems. Open Audio, then Separate.",
+ )
+ for response in (speech, speak):
+ assert response.status_code == 400 and "Separate" in response.json()["detail"]
+ assert sep.calls == [] and sep.separations == []
+
+
+def test_the_separate_scope_clears_only_separations(sep, tmp_path):
+ _alice_sources()
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ assert _separate(client, {"input_id": input_id}).status_code == 200
+ cleared = client.delete("/api/inference/audio/gallery", params = {"workflow": "separate"})
+ assert cleared.status_code == 200 and cleared.json()["removed"] == 6
+ assert len(_staging_and_stems(tmp_path)[1]) == 1
+
+
+def _orchestrator(monkeypatch, reply):
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ sent = []
+
+ def read_one(*, timeout):
+ return {"type": "audio_done", "request_id": sent[-1]["request_id"], **reply}
+
+ for name, value in {
+ "_ensure_subprocess_alive": lambda: True,
+ "_send_cmd": sent.append,
+ "_direct_reader": lambda _rid, _cancel = None: (read_one, lambda **_k: None, lambda: None),
+ "_reserve_worker": lambda _why: contextlib.nullcontext(),
+ "_wait_worker_idle": lambda **_k: True,
+ "_claim_worker": lambda _cancel: None,
+ "_release_worker": lambda *_a, **_k: None,
+ }.items():
+ monkeypatch.setattr(orchestrator, name, value, raising = False)
+ orchestrator.active_model_name = "m"
+ orchestrator.models = {"m": {"audio_type": "audiocpp_sep"}}
+ orchestrator._gen_lock = threading.Lock()
+ orchestrator._send_order_lock = threading.Lock()
+ orchestrator._unload_pending = False
+ return orchestrator, sent
+
+
+_VOCALS = {
+ "id": "vocals",
+ "path": "/x/.separate-1/vocals.wav",
+ "sample_rate": 44100,
+ "channels": 2,
+ "duration_s": 1.0,
+}
+
+
+def test_the_orchestrator_sends_paths_and_returns_the_stem_list(monkeypatch):
+ from core.inference.orchestrator import _audio_generation_timeout
+
+ orchestrator, sent = _orchestrator(monkeypatch, {"outputs": [_VOCALS], "sample_rate": 44100})
+ result = orchestrator.separate_audio_response(
+ "/x/inputs/a.44100.stereo.wav", "/x/.separate-1", {"num_overlap": 1}
+ )
+ assert result == [_VOCALS]
+ (cmd,) = sent
+ assert cmd["workflow"] == "separate" and cmd["text"] == ""
+ assert cmd["audio_inputs"] == {"source": "/x/inputs/a.44100.stereo.wav"}
+ assert cmd["output_dir"] == "/x/.separate-1"
+ assert cmd["audio_options"] == {"num_overlap": 1}
+ assert cmd["max_new_tokens"] == 8192 and "seed" not in cmd
+ assert _audio_generation_timeout(8192) == 3600.0
+ orchestrator, sent = _orchestrator(
+ monkeypatch, {"wav_base64": base64.b64encode(_wav()).decode(), "sample_rate": 24000}
+ )
+ orchestrator.generate_audio_response("plain")
+ assert "output_dir" not in sent[0]
+
+
+def test_the_worker_separates_into_paths_not_bytes():
+ from core.inference.worker import _handle_generate_audio
+
+ seen = []
+
+ class _Separator:
+ def separate_audio(self, **kwargs):
+ seen.append(kwargs)
+ return [_VOCALS]
+
+ def generate_audio_response(self, **kwargs):
+ raise AssertionError("a separation is not speech")
+
+ class _Speech:
+ def generate_audio_response(self, **kwargs):
+ return _wav(), 24000
+
+ responses: queue.Queue = queue.Queue()
+ cancel = threading.Event()
+ cmd = {
+ "request_id": "r1",
+ "text": "",
+ "workflow": "separate",
+ "audio_inputs": {"source": "/abs/in.wav"},
+ "output_dir": "/x/.separate-1",
+ "audio_options": {"num_overlap": 1},
+ }
+ _handle_generate_audio(_Separator(), cmd, responses, cancel)
+ reply = responses.get_nowait()
+ assert reply["type"] == "audio_done" and "wav_base64" not in reply
+ assert reply["outputs"] == [_VOCALS]
+ assert seen == [
+ {
+ "source_path": "/abs/in.wav",
+ "output_dir": "/x/.separate-1",
+ "options": {"num_overlap": 1},
+ "cancel_event": cancel,
+ }
+ ]
+ _handle_generate_audio(_Speech(), {**cmd, "request_id": "r2"}, responses, threading.Event())
+ refused = responses.get_nowait()
+ assert refused["type"] == "audio_error" and refused["code"] == "audio_unsupported_backend"
+
+
def test_the_worker_keeps_run_fields_off_a_backend_without_them():
from core.inference.worker import _handle_generate_audio
@@ -923,6 +1380,32 @@ def test_the_worker_keeps_run_fields_off_a_backend_without_them():
assert seen == ["hi"]
+def test_a_float_wav_separation_source_is_probed_through_ffmpeg(tmp_path):
+ import struct
+
+ import numpy as np
+
+ from routes.inference import _probe_audio_with_av
+
+ frames = np.zeros((22050, 2), dtype = np.float32).tobytes()
+ fmt = struct.pack("<HHIIHH", 3, 2, 44100, 44100 * 8, 8, 32)
+ path = tmp_path / "stem.wav"
+ path.write_bytes(
+ b"RIFF"
+ + struct.pack("<I", 36 + len(frames))
+ + b"WAVEfmt "
+ + struct.pack("<I", 16)
+ + fmt
+ + b"data"
+ + struct.pack("<I", len(frames))
+ + frames
+ )
+ channels, seconds = _probe_audio_with_av(path)
+ assert channels == 2 and abs(seconds - 0.5) < 0.01
+ (tmp_path / "junk.wav").write_bytes(b"not audio")
+ assert _probe_audio_with_av(tmp_path / "junk.wav") is None
+
+
def test_a_repaint_past_the_end_sizes_its_work_to_the_range_end(stub):
backend = stub["use"](ACE_STEP, _music_info("ace_step", "ACE-Step1.5-GGUF"))
backend.__class__ = _MusicBackend
diff --git a/studio/backend/tests/test_audio_workflows.py b/studio/backend/tests/test_audio_workflows.py
index 79c26eb673a..29786d98194 100644
--- a/studio/backend/tests/test_audio_workflows.py
+++ b/studio/backend/tests/test_audio_workflows.py
@@ -11,7 +11,11 @@ from pathlib import Path
import pytest
from core.inference import audio_workflows as aw
-from core.inference.audio_cpp_models import AUDIO_CPP_MUSIC_AUDIO_TYPE, AUDIO_CPP_TTS_AUDIO_TYPE
+from core.inference.audio_cpp_models import (
+ AUDIO_CPP_MUSIC_AUDIO_TYPE,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
+ AUDIO_CPP_TTS_AUDIO_TYPE,
+)
from hub.schemas.inventory import CachedGgufRepo, CachedModelRepo, LocalModelInfo
from models.inference import InferenceStatusResponse, LoadResponse
@@ -45,6 +49,7 @@ def test_a_status_reports_no_workflows_for_a_model_that_is_not_audio():
("snac", ["speak"]),
("higgs_tts2", ["speak"]),
("whisper", ["transcribe"]),
+ (AUDIO_CPP_SEP_AUDIO_TYPE, ["separate"]),
(None, ["speak"]),
],
)
@@ -77,6 +82,8 @@ def test_a_load_response_derives_workflows_and_keeps_an_explicit_value():
("text-to-speech", "minimax_music3", ["music"]),
("text-to-speech", AUDIO_CPP_MUSIC_AUDIO_TYPE, ["music"]),
("text-to-speech", None, ["speak"]),
+ ("audio-to-audio", AUDIO_CPP_SEP_AUDIO_TYPE, ["separate"]),
+ ("audio-to-audio", None, None),
("text-generation", None, None),
(None, None, None),
],
@@ -98,7 +105,7 @@ def test_workflow_for_audio_type_is_speak_or_music():
assert aw.workflow_for_audio_type("snac") == "speak"
assert aw.workflow_for_audio_type("unknown") == "speak"
assert aw.workflow_for_audio_type(None) == "speak"
- assert aw.AUDIO_WORKFLOW_IDS == ("speak", "clone", "music", "transcribe")
+ assert aw.AUDIO_WORKFLOW_IDS == ("speak", "clone", "music", "separate", "transcribe")
def test_workflow_ids_match_the_frontend_order():
@@ -161,3 +168,22 @@ def test_an_umbrella_snapshot_lists_every_downloaded_familys_workflows(tmp_path)
"speak",
"clone",
]
+
+
+def test_a_local_separation_gguf_is_classified_for_separate(monkeypatch):
+ from types import SimpleNamespace
+
+ from hub.services.models import catalog_classification as cc
+
+ model = SimpleNamespace(path = "/m/sep.gguf")
+ monkeypatch.setattr(cc, "_local_model_audio_type", lambda _m: "audiocpp_sep")
+ assert cc._local_model_classification_for_task(model, "audio-to-audio") == (
+ "audio-to-audio",
+ "audiocpp_sep",
+ )
+ # Other audio-to-audio kinds keep no audio type, so they stay off every page.
+ monkeypatch.setattr(cc, "_local_model_audio_type", lambda _m: "audiocpp_tts")
+ assert cc._local_model_classification_for_task(model, "audio-to-audio") == (
+ "audio-to-audio",
+ None,
+ )
diff --git a/studio/frontend/src/features/audio/api.ts b/studio/frontend/src/features/audio/api.ts
index 9d5352c8e9f..90cf8123418 100644
--- a/studio/frontend/src/features/audio/api.ts
+++ b/studio/frontend/src/features/audio/api.ts
@@ -107,9 +107,13 @@ export interface AudioGalleryClip {
order_at?: number | null;
/** The Audio workflow that made the clip. Older servers omit it; read it through clipWorkflow. */
workflow?: string | null;
- /** Clips one run made together share a group. */
- group_id?: string | null;
reference_name?: string | null;
+ /** Clips one run made together share it (a separation's stems, music takes); null for one clip. */
+ group_id?: string | null;
+ /** The stem a clip holds (vocals, drums, ...). */
+ role?: string | null;
+ /** The run's settings, e.g. a separation's stem list. */
+ settings?: Record<string, unknown> | null;
}
export interface AudioGalleryListResponse {
@@ -214,7 +218,7 @@ export async function deleteAudioClip(id: string): Promise<void> {
}
export async function clearAudioGallery(
- workflow?: "speak" | "clone" | "music",
+ workflow?: "speak" | "clone" | "music" | "separate",
): Promise<number> {
const query = workflow ? `?workflow=${workflow}` : "";
const response = await authFetch(`/api/inference/audio/gallery${query}`, {
@@ -422,6 +426,8 @@ export interface AudioRunResponse {
duration_s: number;
workflow: string;
}[];
+ /** One separation's stems share it; null for a single clip. */
+ group_id: string | null;
model: string;
audio: GeneratedAudio | null;
}
diff --git a/studio/frontend/src/features/audio/audio-cpp-catalog.ts b/studio/frontend/src/features/audio/audio-cpp-catalog.ts
index 3dcd4a8b5eb..18ae931ce93 100644
--- a/studio/frontend/src/features/audio/audio-cpp-catalog.ts
+++ b/studio/frontend/src/features/audio/audio-cpp-catalog.ts
@@ -8,15 +8,23 @@ export const AUDIO_CPP_REPO = "audio-cpp/audio.cpp-gguf";
export const AUDIO_CPP_TTS_AUDIO_TYPE = "audiocpp_tts";
export const AUDIO_CPP_MUSIC_AUDIO_TYPE = "audiocpp_music";
+/** Source separation (HTDemucs, RoFormers). Mirrors AUDIO_CPP_SEP_AUDIO_TYPE in audio_cpp_models.py. */
+export const AUDIO_CPP_SEP_AUDIO_TYPE = "audiocpp_sep";
export const AUDIO_CPP_AUDIO_TYPES: ReadonlySet<string> = new Set([
AUDIO_CPP_TTS_AUDIO_TYPE,
AUDIO_CPP_MUSIC_AUDIO_TYPE,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
]);
-export type AudioCppTask = "tts" | "music" | "asr";
+export type AudioCppTask = "tts" | "music" | "asr" | "sep";
/** Mirrors AudioWorkflowId; spelled out to keep this file import-free. */
-export type AudioCppWorkflow = "speak" | "clone" | "music" | "transcribe";
+export type AudioCppWorkflow =
+ | "speak"
+ | "clone"
+ | "music"
+ | "separate"
+ | "transcribe";
export interface AudioCppModel {
/** Hub repo id, or `${AUDIO_CPP_REPO}/<folder>` for a package in the shared repo. */
@@ -27,6 +35,7 @@ export interface AudioCppModel {
languages?: readonly string[];
/** Phonemizes with eSpeak-ng, which upstream runtime bundles lack (backend needs_espeak). */
needsEspeak?: boolean;
+ stems?: readonly string[];
}
/** The `audio_cpp_runtime` block of /api/inference/audio/stt/status. */
@@ -64,6 +73,30 @@ export const AUDIO_CPP_MODELS: readonly AudioCppModel[] = [
{ id: folder("Stable-Audio-3-Small-Music-GGUF"), task: "music" },
{ id: folder("Stable-Audio-3-Small-SFX-GGUF"), task: "music" },
{ id: folder("ControlFoley-GGUF"), task: "music" },
+ {
+ id: folder("HTDemucs-GGUF"),
+ task: "sep",
+ workflows: ["separate"],
+ stems: ["vocals", "drums", "bass", "other"],
+ },
+ {
+ id: folder("BS-RoFormer-ep368-GGUF"),
+ task: "sep",
+ workflows: ["separate"],
+ stems: ["vocals", "instrumental"],
+ },
+ {
+ id: folder("HTDemucs-6stems-GGUF"),
+ task: "sep",
+ workflows: ["separate"],
+ stems: ["vocals", "drums", "bass", "guitar", "piano", "other"],
+ },
+ {
+ id: folder("Mel-Band-RoFormer-GGUF"),
+ task: "sep",
+ workflows: ["separate"],
+ stems: ["vocals", "instrumental"],
+ },
{ id: folder("Qwen3-ASR-0.6B-GGUF"), task: "asr" },
{ id: folder("Qwen3-ASR-1.7B-GGUF"), task: "asr" },
{ id: folder("Parakeet-TDT-0.6B-v3-GGUF"), task: "asr" },
@@ -185,6 +218,7 @@ const TASK_WORKFLOW: Record<AudioCppTask, AudioCppWorkflow> = {
tts: "speak",
music: "music",
asr: "transcribe",
+ sep: "separate",
};
export function audioCppWorkflowsFor(
diff --git a/studio/frontend/src/features/audio/audio-page-policy.ts b/studio/frontend/src/features/audio/audio-page-policy.ts
index 72d2d2bd308..d11dd1e8015 100644
--- a/studio/frontend/src/features/audio/audio-page-policy.ts
+++ b/studio/frontend/src/features/audio/audio-page-policy.ts
@@ -7,6 +7,7 @@ import {
AUDIO_CPP_MUSIC_AUDIO_TYPE,
AUDIO_CPP_MUSIC_MAX_SECONDS,
AUDIO_CPP_MUSIC_MIN_SECONDS,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
AUDIO_CPP_TTS_AUDIO_TYPE,
type AudioCppRuntimeStatus,
audioCppDisplayName,
@@ -130,6 +131,7 @@ const TTS_AUDIO_TYPES = new Set([
"minimax_music3",
AUDIO_CPP_TTS_AUDIO_TYPE,
AUDIO_CPP_MUSIC_AUDIO_TYPE,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
]);
// The GGUF runtime's speech and music load from a GGUF too, so a status may call them one.
const GGUF_TTS_AUDIO_TYPES = new Set([
@@ -138,6 +140,7 @@ const GGUF_TTS_AUDIO_TYPES = new Set([
"dac",
AUDIO_CPP_TTS_AUDIO_TYPE,
AUDIO_CPP_MUSIC_AUDIO_TYPE,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
]);
const NATIVE_TTS_AUDIO_TYPES = new Set([
"higgs_tts2",
@@ -147,6 +150,7 @@ const NATIVE_TTS_AUDIO_TYPES = new Set([
"minimax_music3",
AUDIO_CPP_TTS_AUDIO_TYPE,
AUDIO_CPP_MUSIC_AUDIO_TYPE,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
]);
export const MOSS_TTS_FRAMES_PER_SECOND = 12.5;
export const MOSS_TTS_DEFAULT_SECONDS = 15;
diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsx
index b58af08426c..6602fbfacd4 100644
--- a/studio/frontend/src/features/audio/audio-page.tsx
+++ b/studio/frontend/src/features/audio/audio-page.tsx
@@ -55,6 +55,7 @@ import { galleryCache, useAudioGallery, useWorkflowHistory } from "./hooks/use-a
import { useAudioHandoff } from "./hooks/use-audio-handoff";
import { useAudioModelSlot } from "./hooks/use-audio-model-slot";
import { useCloneGeneration } from "./hooks/use-clone-generation";
+import { useSeparateGeneration } from "./hooks/use-separate-generation";
import { useSpeechGeneration } from "./hooks/use-speech-generation";
import { useSttSidecar } from "./hooks/use-stt-sidecar";
import { useTranscription } from "./hooks/use-transcription";
@@ -67,9 +68,16 @@ import {
} from "./pages/clone-page";
import { useMusicGeneration } from "./hooks/use-music-generation";
import { MusicOutput, MusicRail, musicPageModels } from "./pages/music-page";
+import {
+ SeparateFooter,
+ SeparateOutput,
+ SeparateRail,
+ separatePageModels,
+} from "./pages/separate-page";
import { SpeakOutput, SpeakRail, speakPageModels } from "./pages/speak-page";
import { type GenerateBlocker, TtsFooter } from "./pages/tts-workspace";
import { TranscribeFooter, TranscribeOutput, TranscribeRail } from "./pages/transcribe-page";
+import type { SendTarget } from "./components/stem-mixer-types";
import { AUDIO_CPP_REPO, audioCppModelFor } from "./audio-cpp-catalog";
import { selectionExpired } from "./audio-run-request";
import type { AudioSourceInputHandle } from "./components/audio-source-input";
@@ -502,6 +510,24 @@ export function AudioPage({
audioOptionValues,
sttRepo: cloneSttRepo,
});
+ const separate = useSeparateGeneration({
+ status,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ generateAbort,
+ setMode,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ audioDevice,
+ refreshGallery,
+ selectClip: selectGeneratedClip,
+ setFallbackClip,
+ setSelectedId,
+ pendingTranscribeRelease,
+ replayQueuedTtsPick,
+ });
const { navigateSelf } = useAudioHandoff({
active,
@@ -603,6 +629,41 @@ export function AudioPage({
},
[transitionWorkflow, setPrompt],
);
+ const handleSendStem = useCallback(
+ async (target: SendTarget, clip: AudioGalleryClip, name: string) => {
+ if (target.workflow === "clone") {
+ if (!transitionWorkflow("clone")) return;
+ // Adopted like any other clip, so the old reference's transcript does not stay attached.
+ useAudioCloneStore.getState().adoptReference({
+ kind: "clip",
+ id: clip.id,
+ name,
+ durationS: clip.duration_s,
+ });
+ return;
+ }
+ if (target.workflow === "transcribe") {
+ // As in sendHandlersFor: switching mid-run would stop the run in progress.
+ if (busyRef.current !== null) {
+ toast.info("Wait for the current audio task to finish, then send the stem.");
+ return;
+ }
+ if (!transitionWorkflow("transcribe")) return;
+ // A stem is a history clip, so it goes in by id like any other.
+ useAudioTranscribeStore.setState({
+ source: {
+ kind: "clip",
+ id: clip.id,
+ name,
+ durationS: clip.duration_s,
+ transcript: null,
+ language: null,
+ },
+ });
+ }
+ },
+ [transitionWorkflow, busyRef],
+ );
const pageModelLoaded =
ttsLoaded &&
@@ -639,6 +700,17 @@ export function AudioPage({
label: `use ${model.name}`,
onClick: () => void pickRecommendedModel(model.id),
}));
+ const recommendedSeparateActions = separatePageModels(
+ MODELS_BY_MODE.speak,
+ isMac,
+ )
+ .slice(0, 2)
+ .map((model) => ({
+ label: `use ${model.name}`,
+ onClick: () => void pickRecommendedModel(model.id),
+ }));
+ const loadedSeparates =
+ status?.audio_workflows?.includes("separate") === true;
const generateBlocker: GenerateBlocker | null =
busy === "loading"
? { reason: "Waiting for the model to finish loading." }
@@ -651,14 +723,37 @@ export function AudioPage({
? "Load a music model to generate."
: ttsWorkflow === "clone"
? "Load a model that can clone a voice."
- : "Load a speech model to generate.",
+ : ttsWorkflow === "separate"
+ ? "Load a model that can separate audio."
+ : "Load a speech model to generate.",
actions:
ttsWorkflow === "clone"
? [chooseModelAction, ...recommendedCloneActions]
- : ttsWorkflow === "music"
- ? [chooseModelAction, ...recommendedMusicActions]
+ : ttsWorkflow === "separate"
+ ? [chooseModelAction, ...recommendedSeparateActions]
+ : ttsWorkflow === "music"
+ ? [chooseModelAction, ...recommendedMusicActions]
: [chooseModelAction],
}
+ : !pageModelLoaded && ttsWorkflow === "separate"
+ ? {
+ reason: "The loaded model cannot separate audio.",
+ actions: [
+ { label: "Choose a model that separates", onClick: openSelector },
+ ...recommendedSeparateActions,
+ ],
+ }
+ : !pageModelLoaded && loadedSeparates
+ ? {
+ reason: "The loaded model separates audio.",
+ actions: [
+ { label: "Choose a different model", onClick: openSelector },
+ {
+ label: "open Separate",
+ onClick: () => transitionWorkflow("separate"),
+ },
+ ],
+ }
: !pageModelLoaded && ttsWorkflow === "clone"
? {
reason: "The loaded model cannot clone a voice.",
@@ -705,6 +800,8 @@ export function AudioPage({
},
],
}
+ : ttsWorkflow === "separate"
+ ? separate.blocker
: ttsWorkflow === "clone"
? clone.blocker
: musicStudio
@@ -725,10 +822,12 @@ export function AudioPage({
? { reason: toolBlocker }
: null;
const pageGenerationError =
- ttsWorkflow === "clone"
- ? clone.generationError
- : musicStudio
- ? music.generationError
+ ttsWorkflow === "separate"
+ ? separate.generationError
+ : ttsWorkflow === "clone"
+ ? clone.generationError
+ : musicStudio
+ ? music.generationError
: generationError;
const generateFailure: GenerateBlocker | null = pageGenerationError
? {
@@ -739,22 +838,34 @@ export function AudioPage({
}
: null;
const setCloneGenerationError = clone.setGenerationError;
+ const setSeparateGenerationError = separate.setGenerationError;
const setMusicGenerationError = music.setGenerationError;
useEffect(() => {
setGenerationError(null);
setCloneGenerationError(null);
+ setSeparateGenerationError(null);
setMusicGenerationError(null);
}, [
status?.active_model,
setGenerationError,
setCloneGenerationError,
+ setSeparateGenerationError,
setMusicGenerationError,
]);
useEffect(() => {
if (busy === "generating") setGenerationError(null);
if (busy === "generating") setCloneGenerationError(null);
- if (busy === "generating") setMusicGenerationError(null);
- }, [busy, setGenerationError, setCloneGenerationError, setMusicGenerationError]);
+ if (busy === "generating") {
+ setSeparateGenerationError(null);
+ setMusicGenerationError(null);
+ }
+ }, [
+ busy,
+ setGenerationError,
+ setCloneGenerationError,
+ setSeparateGenerationError,
+ setMusicGenerationError,
+ ]);
const [elapsedSeconds, setElapsedSeconds] = useState<number | null>(null);
useEffect(() => {
if (busy !== "generating") {
@@ -925,6 +1036,7 @@ export function AudioPage({
const pageRootRef = useRef<HTMLDivElement | null>(null);
const generateShortcut = useRef<() => void>(() => {});
const handlePageGenerate =
+ ttsWorkflow === "separate" ? separate.handleGenerate :
ttsWorkflow === "clone" ? clone.handleGenerate : handleGenerate;
generateShortcut.current = () => {
if (canTranscribe) handleTranscribe();
@@ -953,6 +1065,8 @@ export function AudioPage({
? musicPageModels(MODELS_BY_MODE.speak, isMac)
: ttsWorkflow === "clone"
? clonePageModels(MODELS_BY_MODE.speak, isMac)
+ : ttsWorkflow === "separate"
+ ? separatePageModels(MODELS_BY_MODE.speak, isMac)
: speakPageModels(MODELS_BY_MODE.speak, isMac)
: MODELS_BY_MODE[mode];
const selectorRowFilter = useCallback(
@@ -1002,9 +1116,17 @@ export function AudioPage({
: ttsLoaded
? pageModelLoaded
? audioCapabilityLine(
- musicGeneration ? "music" : ttsWorkflow === "clone" ? "clone" : "tts",
+ musicGeneration
+ ? "music"
+ : ttsWorkflow === "separate"
+ ? "separate"
+ : ttsWorkflow === "clone" ? "clone" : "tts",
status?.audio_type,
)
+ : ttsWorkflow === "separate"
+ ? "The loaded model cannot separate audio."
+ : loadedSeparates
+ ? "The loaded model separates audio. Open Separate to use it."
: ttsWorkflow === "clone"
? "The loaded model cannot clone a voice."
: musicGeneration
@@ -1016,6 +1138,8 @@ export function AudioPage({
? "The loaded model is not a TTS audio model."
: ttsWorkflow === "music"
? "No music model loaded."
+ : ttsWorkflow === "separate"
+ ? "No separation model loaded."
: ttsWorkflow === "clone"
? "No voice cloning model loaded."
: "No TTS model loaded."
@@ -1058,7 +1182,7 @@ export function AudioPage({
additionalOnDeviceModels={
mode === "transcribe"
? sttOnDeviceModels
- : ttsWorkflow === "clone"
+ : ttsWorkflow === "clone" || ttsWorkflow === "separate"
? []
: trainedTtsModels.filter(
(model) =>
@@ -1239,6 +1363,16 @@ export function AudioPage({
setDescription={setAudioInstructions}
/>
</AudioActiveProvider>
+ ) : ttsWorkflow === "separate" ? (
+ <AudioActiveProvider value={active}>
+ <SeparateRail
+ {...railProps}
+ separate={separate}
+ historyClips={clips}
+ pageModelLoaded={pageModelLoaded}
+ pageModelId={lastPageModel}
+ />
+ </AudioActiveProvider>
) : ttsWorkflow === "clone" ? (
<AudioActiveProvider value={active}>
<CloneRail
@@ -1271,7 +1405,20 @@ export function AudioPage({
{mode === "speak" ? (
/* The scroll mask provides the fade; leave the footer unpainted to avoid dark-mode banding. */
<div className="relative z-10 flex shrink-0 justify-center px-10 pt-0.5 pb-4">
- {ttsWorkflow === "clone" ? (
+ {ttsWorkflow === "separate" ? (
+ <SeparateFooter
+ busy={busy}
+ generationPresentation={generationPresentation}
+ generationPhase={generationPhase}
+ handleStopGeneration={handleStopGeneration}
+ ttsLoaded={pageModelLoaded}
+ blocker={generateBlocker}
+ shortcutLabel={shortcutLabel}
+ error={generateFailure}
+ elapsedSeconds={elapsedSeconds}
+ separate={separate}
+ />
+ ) : ttsWorkflow === "clone" ? (
<CloneFooter
busy={busy}
generationPresentation={generationPresentation}
@@ -1411,6 +1558,15 @@ export function AudioPage({
};
return ttsWorkflow === "music" ? (
<MusicOutput {...outputProps} modelReady={pageModelLoaded} />
+ ) : ttsWorkflow === "separate" ? (
+ <SeparateOutput
+ {...outputProps}
+ modelReady={pageModelLoaded}
+ recommendedModels={selectorModels}
+ onPickModel={pickRecommendedModel}
+ separate={separate}
+ onSendStem={handleSendStem}
+ />
) : ttsWorkflow === "clone" ? (
<CloneOutput
{...outputProps}
diff --git a/studio/frontend/src/features/audio/audio-run-request.ts b/studio/frontend/src/features/audio/audio-run-request.ts
index 5f9e5051425..c93ef08bdb7 100644
--- a/studio/frontend/src/features/audio/audio-run-request.ts
+++ b/studio/frontend/src/features/audio/audio-run-request.ts
@@ -101,13 +101,15 @@ export interface AudioMusicRunFields {
}
export interface AudioRunRequest {
- workflow: "clone" | "speak" | "music";
+ workflow: "clone" | "speak" | "music" | "separate";
music?: AudioMusicRunFields;
- text: string;
+ /** Required for clone, speak and music; separate takes none. */
+ text?: string;
language?: string | null;
instructions?: string | null;
inputs?: {
reference?: AudioSourceRef | null;
+ source?: AudioSourceRef | null;
reference_text?: string | null;
emotion?: AudioSourceRef | null;
};
@@ -133,10 +135,8 @@ function cleanRef(
export function buildAudioRunBody(
request: AudioRunRequest,
): Record<string, unknown> {
- const body: Record<string, unknown> = {
- workflow: request.workflow,
- text: request.text,
- };
+ const body: Record<string, unknown> = { workflow: request.workflow };
+ if (request.workflow !== "separate") body.text = request.text ?? "";
const language = request.language?.trim();
if (language) body.language = language;
const instructions = request.instructions?.trim();
@@ -144,6 +144,8 @@ export function buildAudioRunBody(
const inputs: Record<string, unknown> = {};
const reference = cleanRef(request.inputs?.reference);
if (reference) inputs.reference = reference;
+ const source = cleanRef(request.inputs?.source);
+ if (source) inputs.source = source;
const rawReferenceText = request.inputs?.reference_text;
// A cleared field is sent blank, so the server does not refill it from a saved voice.
if (typeof rawReferenceText === "string") {
diff --git a/studio/frontend/src/features/audio/audio-workspace-utils.ts b/studio/frontend/src/features/audio/audio-workspace-utils.ts
index f001aa06c16..164e3c200a4 100644
--- a/studio/frontend/src/features/audio/audio-workspace-utils.ts
+++ b/studio/frontend/src/features/audio/audio-workspace-utils.ts
@@ -3,6 +3,7 @@
import {
AUDIO_CPP_MUSIC_AUDIO_TYPE,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
AUDIO_CPP_TTS_AUDIO_TYPE,
audioCppDisplayName,
isAudioCppFolderId,
@@ -18,6 +19,7 @@ export function audioModelLabel(id: string): string {
export function loadedAudioKind(audioType: string | null | undefined): string {
if (audioType === AUDIO_CPP_TTS_AUDIO_TYPE) return "speech";
if (audioType === AUDIO_CPP_MUSIC_AUDIO_TYPE) return "music";
+ if (audioType === AUDIO_CPP_SEP_AUDIO_TYPE) return "separation";
return audioType ?? "audio";
}
diff --git a/studio/frontend/src/features/audio/catalog.ts b/studio/frontend/src/features/audio/catalog.ts
index 11d30e56061..1f63089cb42 100644
--- a/studio/frontend/src/features/audio/catalog.ts
+++ b/studio/frontend/src/features/audio/catalog.ts
@@ -147,15 +147,19 @@ export function audioModelsForTask(task: AudioTask): ModelOption[] {
return AUDIO_MODEL_OPTIONS.filter(matches);
}
-const CAPABILITY_LABEL: Record<AudioTask | "music" | "clone", string> = {
+const CAPABILITY_LABEL: Record<
+ AudioTask | "music" | "clone" | "separate",
+ string
+> = {
music: "Music generation",
+ separate: "Source separation",
clone: "Voice cloning",
tts: "Text-to-speech",
stt: "Speech-to-text",
};
export function audioCapabilityLine(
- task: AudioTask | "music" | "clone",
+ task: AudioTask | "music" | "clone" | "separate",
detail?: string | null,
): string {
const base = CAPABILITY_LABEL[task];
diff --git a/studio/frontend/src/features/audio/components/stem-mixer-state.ts b/studio/frontend/src/features/audio/components/stem-mixer-state.ts
new file mode 100644
index 00000000000..ff46e3c301e
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/stem-mixer-state.ts
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Free of app imports so the node test runner can load it.
+
+export interface StemMix {
+ volume: number;
+ muted: boolean;
+ solo: boolean;
+}
+
+export interface StemMixerState {
+ byStem: Readonly<Record<string, StemMix>>;
+}
+
+export type StemMixerAction =
+ | { type: "toggleSolo"; role: string }
+ | { type: "toggleMute"; role: string }
+ | { type: "setVolume"; role: string; volume: number }
+ | { type: "reset" };
+
+export const DEFAULT_STEM_MIX: StemMix = {
+ volume: 1,
+ muted: false,
+ solo: false,
+};
+
+export const INITIAL_STEM_MIXER_STATE: StemMixerState = { byStem: {} };
+
+export function stemMix(state: StemMixerState, role: string): StemMix {
+ return state.byStem[role] ?? DEFAULT_STEM_MIX;
+}
+
+export function anySolo(state: StemMixerState): boolean {
+ return Object.values(state.byStem).some((mix) => mix.solo);
+}
+
+export function clampVolume(volume: number, fallback = 1): number {
+ if (!Number.isFinite(volume)) return fallback;
+ return Math.min(1, Math.max(0, volume));
+}
+
+function withStem(
+ state: StemMixerState,
+ role: string,
+ patch: Partial<StemMix>,
+): StemMixerState {
+ const current = stemMix(state, role);
+ const next = { ...current, ...patch };
+ if (
+ next.volume === current.volume &&
+ next.muted === current.muted &&
+ next.solo === current.solo
+ )
+ return state;
+ return { byStem: { ...state.byStem, [role]: next } };
+}
+
+export function stemMixerReducer(
+ state: StemMixerState,
+ action: StemMixerAction,
+): StemMixerState {
+ switch (action.type) {
+ case "toggleSolo":
+ return withStem(state, action.role, {
+ solo: !stemMix(state, action.role).solo,
+ });
+ case "toggleMute":
+ return withStem(state, action.role, {
+ muted: !stemMix(state, action.role).muted,
+ });
+ case "setVolume": {
+ const current = stemMix(state, action.role).volume;
+ return withStem(state, action.role, {
+ volume: clampVolume(action.volume, current),
+ });
+ }
+ case "reset":
+ return Object.keys(state.byStem).length === 0
+ ? state
+ : INITIAL_STEM_MIXER_STATE;
+ default:
+ return state;
+ }
+}
+
+/** Solo beats mute: while any stem is soloed only soloed stems sound. */
+export function effectiveGain(state: StemMixerState, role: string): number {
+ const mix = stemMix(state, role);
+ if (anySolo(state)) return mix.solo ? mix.volume : 0;
+ return mix.muted ? 0 : mix.volume;
+}
diff --git a/studio/frontend/src/features/audio/components/stem-mixer-types.ts b/studio/frontend/src/features/audio/components/stem-mixer-types.ts
new file mode 100644
index 00000000000..b6d27435b47
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/stem-mixer-types.ts
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+export interface SendTarget {
+ id: string;
+ workflow: string;
+ label: string;
+}
+
+export interface StemMixerStem {
+ clipId: string;
+ role: string;
+ label: string;
+ src: string | null;
+ /** Its audio could not be fetched: left out of playback instead of waiting on it. */
+ failed?: boolean;
+ durationS: number;
+ peaks: readonly number[] | null;
+}
+
+export interface StemMixerProps {
+ groupId: string;
+ title: string;
+ subtitle?: string;
+ stems: readonly StemMixerStem[];
+ autoFocus?: boolean;
+ onDownloadStem(clipId: string): void;
+ onDownloadAll(): void;
+ sendTargets: readonly SendTarget[];
+ onSend(target: SendTarget, clipId: string): void;
+ active: boolean;
+}
diff --git a/studio/frontend/src/features/audio/components/stem-mixer.tsx b/studio/frontend/src/features/audio/components/stem-mixer.tsx
new file mode 100644
index 00000000000..b1088f3a681
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/stem-mixer.tsx
@@ -0,0 +1,316 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Button } from "@/components/ui/button";
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuItem,
+ DropdownMenuSub,
+ DropdownMenuSubContent,
+ DropdownMenuSubTrigger,
+ DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu";
+import { Slider } from "@/components/ui/slider";
+import { cn } from "@/lib/utils";
+import {
+ ArrowDown01Icon,
+ Download01Icon,
+ HeadphonesIcon,
+ PauseIcon,
+ PlayIcon,
+ SentIcon,
+ VolumeHighIcon,
+ VolumeOffIcon,
+} from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import {
+ type KeyboardEvent,
+ useEffect,
+ useMemo,
+ useReducer,
+ useRef,
+} from "react";
+import { useStemTransport } from "../hooks/use-stem-transport";
+import {
+ INITIAL_STEM_MIXER_STATE,
+ effectiveGain,
+ stemMix,
+ stemMixerReducer,
+} from "./stem-mixer-state";
+import type { StemMixerProps } from "./stem-mixer-types";
+import { WaveformBars } from "./waveform";
+import { formatSeconds } from "./waveform-peaks";
+
+const SEEK_STEP_SECONDS = 5;
+
+export function StemMixer({
+ groupId,
+ title,
+ subtitle,
+ stems,
+ autoFocus = false,
+ onDownloadStem,
+ onDownloadAll,
+ sendTargets,
+ onSend,
+ active,
+}: StemMixerProps) {
+ const [mix, dispatch] = useReducer(
+ stemMixerReducer,
+ INITIAL_STEM_MIXER_STATE,
+ );
+ const playRef = useRef<HTMLButtonElement | null>(null);
+
+ // biome-ignore lint/correctness/useExhaustiveDependencies: groupId is the trigger, not a value read inside.
+ useEffect(() => {
+ dispatch({ type: "reset" });
+ }, [groupId]);
+
+ // A stem that failed to load would hold the shared clock forever; the rest play without it.
+ const playable = useMemo(() => stems.filter((stem) => !stem.failed), [stems]);
+ const gains = playable.map((stem) => effectiveGain(mix, stem.role));
+ const sources = useMemo(
+ () => playable.map((stem) => ({ id: stem.clipId, src: stem.src })),
+ [playable],
+ );
+ const durationHint = Math.max(0, ...stems.map((stem) => stem.durationS || 0));
+ const transport = useStemTransport({
+ sources,
+ gains,
+ active,
+ durationHint,
+ });
+ const { ready, playing, position, duration, toggle, seek } = transport;
+ const fraction = duration > 0 ? Math.min(1, position / duration) : 0;
+ const loadingCount = playable.filter((stem) => !stem.src).length;
+
+ const focusedGroup = useRef<string | null>(null);
+ useEffect(() => {
+ if (!(autoFocus && ready) || focusedGroup.current === groupId) return;
+ focusedGroup.current = groupId;
+ playRef.current?.focus();
+ }, [autoFocus, ready, groupId]);
+
+ const handleKeyDown = (event: KeyboardEvent<HTMLElement>) => {
+ if (event.metaKey || event.ctrlKey || event.altKey) return;
+ const target = event.target as HTMLElement;
+ const ours =
+ target === event.currentTarget ||
+ target.closest("[data-stem-bars]") !== null;
+ if (!ours) return;
+ if (event.key === " " || event.key === "k" || event.key === "K") {
+ event.preventDefault();
+ toggle();
+ } else if (event.key === "ArrowRight") {
+ event.preventDefault();
+ seek(position + SEEK_STEP_SECONDS);
+ } else if (event.key === "ArrowLeft") {
+ event.preventDefault();
+ seek(position - SEEK_STEP_SECONDS);
+ } else if (event.key === "Home") {
+ event.preventDefault();
+ seek(0);
+ }
+ };
+
+ const status = loadingCount > 0 ? "Loading stems…" : "";
+
+ return (
+ <section
+ // biome-ignore lint/a11y/noNoninteractiveTabindex: the mixer is a keyboard target for Space/K and the arrows.
+ tabIndex={0}
+ aria-label="Stem mixer"
+ onKeyDown={handleKeyDown}
+ className="@container corner-squircle grid gap-3 rounded-4xl bg-card p-4 ring-1 ring-[color-mix(in_oklab,var(--foreground)_calc(10%*var(--contrast-edge-gain,1)),transparent)] outline-none focus-visible:ring-2 focus-visible:ring-ring"
+ >
+ <output aria-live="polite" aria-atomic="true" className="sr-only">
+ {status}
+ </output>
+ <header className="flex flex-wrap items-center gap-2">
+ <div className="min-w-0 flex-1">
+ <h3 className="truncate text-ui-13 font-medium text-foreground">
+ {title}
+ </h3>
+ <p className="truncate text-ui-11p5 text-muted-foreground">
+ {subtitle ? `${subtitle} · ` : ""}
+ <span className="font-mono tabular-nums">
+ {formatSeconds(duration)}
+ </span>
+ </p>
+ </div>
+ <Button
+ ref={playRef}
+ type="button"
+ variant="muted"
+ size="icon"
+ className="size-[calc(32px*var(--ui-space-scale,1))] shrink-0"
+ aria-label={playing ? "Pause stems" : "Play stems"}
+ disabled={!ready}
+ onClick={toggle}
+ >
+ <HugeiconsIcon
+ icon={playing ? PauseIcon : PlayIcon}
+ className="size-3.5"
+ />
+ </Button>
+ <span className="shrink-0 font-mono text-ui-11p5 tabular-nums text-muted-foreground">
+ {formatSeconds(position)} / {formatSeconds(duration)}
+ </span>
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ disabled={stems.some((stem) => !stem.src)}
+ onClick={onDownloadAll}
+ >
+ <HugeiconsIcon
+ icon={Download01Icon}
+ strokeWidth={1.75}
+ data-icon="inline-start"
+ />
+ Download all
+ </Button>
+ {sendTargets.length > 0 ? (
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild={true}>
+ <Button type="button" variant="outline" size="sm">
+ <HugeiconsIcon
+ icon={SentIcon}
+ strokeWidth={1.75}
+ data-icon="inline-start"
+ />
+ Send to
+ <HugeiconsIcon
+ icon={ArrowDown01Icon}
+ strokeWidth={1.75}
+ data-icon="inline-end"
+ />
+ </Button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent align="end">
+ {stems.map((stem) => (
+ <DropdownMenuSub key={stem.clipId}>
+ <DropdownMenuSubTrigger disabled={!stem.src}>
+ <span>{stem.label}</span>
+ </DropdownMenuSubTrigger>
+ <DropdownMenuSubContent>
+ {sendTargets.map((target) => (
+ <DropdownMenuItem
+ key={target.id}
+ onSelect={() => onSend(target, stem.clipId)}
+ >
+ {target.label}
+ </DropdownMenuItem>
+ ))}
+ </DropdownMenuSubContent>
+ </DropdownMenuSub>
+ ))}
+ </DropdownMenuContent>
+ </DropdownMenu>
+ ) : null}
+ </header>
+ <ul className="grid gap-2">
+ {stems.map((stem) => {
+ const level = stemMix(mix, stem.role);
+ const silent = effectiveGain(mix, stem.role) === 0;
+ const percent = Math.round(level.volume * 100);
+ return (
+ <li
+ key={stem.clipId}
+ className="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-x-3 gap-y-1 @[30rem]:grid-cols-[calc(88px*var(--ui-space-scale,1))_minmax(0,1fr)_auto]"
+ >
+ <span
+ className={cn(
+ "col-start-1 row-start-1 truncate text-ui-13 font-medium",
+ silent ? "text-muted-foreground" : "text-foreground",
+ )}
+ >
+ {stem.label}
+ </span>
+ <div
+ data-stem-bars={true}
+ aria-busy={!(stem.src || stem.failed)}
+ className={cn(
+ "col-span-2 row-start-2 flex min-w-0 @[30rem]:col-span-1 @[30rem]:col-start-2 @[30rem]:row-start-1",
+ !(stem.src || stem.failed) &&
+ "animate-pulse motion-reduce:animate-none",
+ silent && "opacity-35",
+ )}
+ >
+ <WaveformBars
+ peaks={stem.src ? stem.peaks : null}
+ fraction={fraction}
+ label={stem.label}
+ valueNow={position}
+ valueMax={duration}
+ disabled={!ready}
+ onSeek={(next) => seek(next * duration)}
+ className="h-[calc(28px*var(--ui-space-scale,1))]"
+ />
+ </div>
+ <div className="col-start-2 row-start-1 flex items-center gap-1 @[30rem]:col-start-3">
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-xs"
+ aria-label={`Solo ${stem.label}`}
+ aria-pressed={level.solo}
+ className="size-[calc(24px*var(--ui-space-scale,1))] text-muted-foreground aria-pressed:bg-foreground aria-pressed:text-background"
+ onClick={() =>
+ dispatch({ type: "toggleSolo", role: stem.role })
+ }
+ >
+ <HugeiconsIcon icon={HeadphonesIcon} strokeWidth={1.75} />
+ </Button>
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-xs"
+ aria-label={`Mute ${stem.label}`}
+ aria-pressed={level.muted}
+ className="size-[calc(24px*var(--ui-space-scale,1))] text-muted-foreground aria-pressed:bg-foreground aria-pressed:text-background"
+ onClick={() =>
+ dispatch({ type: "toggleMute", role: stem.role })
+ }
+ >
+ <HugeiconsIcon
+ icon={level.muted ? VolumeOffIcon : VolumeHighIcon}
+ strokeWidth={1.75}
+ />
+ </Button>
+ <Slider
+ className="panel-slider w-[calc(88px*var(--ui-space-scale,1))] px-1"
+ min={0}
+ max={100}
+ step={1}
+ value={[percent]}
+ aria-label={`${stem.label} volume`}
+ thumbValueText={(value) => `${stem.label} volume ${value} %`}
+ onValueChange={([value]) =>
+ dispatch({
+ type: "setVolume",
+ role: stem.role,
+ volume: (value ?? percent) / 100,
+ })
+ }
+ />
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-xs"
+ aria-label={`Download ${stem.label}`}
+ className="size-[calc(24px*var(--ui-space-scale,1))] text-muted-foreground"
+ disabled={!stem.src}
+ onClick={() => onDownloadStem(stem.clipId)}
+ >
+ <HugeiconsIcon icon={Download01Icon} strokeWidth={1.75} />
+ </Button>
+ </div>
+ </li>
+ );
+ })}
+ </ul>
+ </section>
+ );
+}
diff --git a/studio/frontend/src/features/audio/components/stem-zip.ts b/studio/frontend/src/features/audio/components/stem-zip.ts
new file mode 100644
index 00000000000..8618fd00d29
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/stem-zip.ts
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Stored (uncompressed) zip: WAV barely compresses. Free of app imports for the node test runner.
+
+import { Zip, ZipPassThrough } from "fflate";
+
+const INVALID_CHARS = new Set('<>:"/\\|?*');
+const MAX_TITLE_CHARS = 120;
+const FALLBACK_TITLE = "Separated track";
+
+export function sanitizeFileNamePart(part: string, fallback: string): string {
+ let out = "";
+ for (const char of part) {
+ const code = char.codePointAt(0) ?? 0;
+ out += INVALID_CHARS.has(char) || code < 0x20 || code === 0x7f ? "_" : char;
+ }
+ const cleaned = out
+ .replace(/\s+/g, " ")
+ .trim()
+ .replace(/[. ]+$/, "");
+ return cleaned || fallback;
+}
+
+/** The title is kept short so the stem label always survives. */
+export function stemFileName(title: string, label: string): string {
+ const bare = title.replace(/\.(wav|mp3|flac|ogg|m4a|aac|opus|webm)$/i, "");
+ const safeTitle = [...sanitizeFileNamePart(bare, FALLBACK_TITLE)]
+ .slice(0, MAX_TITLE_CHARS)
+ .join("")
+ .trim();
+ const safeLabel = sanitizeFileNamePart(label, "Stem");
+ return `${safeTitle || FALLBACK_TITLE} - ${safeLabel}.wav`;
+}
+
+export function stemZipName(title: string): string {
+ return stemFileName(title, "stems").replace(/\.wav$/, ".zip");
+}
+
+export function uniqueStemNames(names: readonly string[]): string[] {
+ const taken = new Set<string>();
+ return names.map((name) => {
+ let candidate = name;
+ const dot = name.lastIndexOf(".");
+ const stem = dot > 0 ? name.slice(0, dot) : name;
+ const ext = dot > 0 ? name.slice(dot) : "";
+ for (let n = 2; taken.has(candidate.toLowerCase()); n += 1)
+ candidate = `${stem} (${n})${ext}`;
+ taken.add(candidate.toLowerCase());
+ return candidate;
+ });
+}
+
+// Zip output is moved into Blob parts at this size, so it leaves the JS heap as it is written.
+const ZIP_FLUSH_BYTES = 32 * 1024 * 1024;
+
+/** Blobs are streamed in one at a time (a loader is called only when its turn comes), and the
+ * output is flushed into Blob parts, so neither all stems nor the whole archive sit in the heap. */
+export async function zipStems(
+ files: readonly { name: string; blob: Blob | (() => Promise<Blob>) }[],
+): Promise<Blob> {
+ const names = uniqueStemNames(files.map((file) => file.name));
+ const parts: Blob[] = [];
+ let pending: Uint8Array[] = [];
+ let pendingBytes = 0;
+ const flush = () => {
+ if (pending.length === 0) return;
+ parts.push(new Blob(pending as BlobPart[]));
+ pending = [];
+ pendingBytes = 0;
+ };
+ await new Promise<void>((resolve, reject) => {
+ const zip = new Zip((error, data, final) => {
+ if (error) {
+ reject(error);
+ return;
+ }
+ pending.push(data);
+ pendingBytes += data.byteLength;
+ if (pendingBytes >= ZIP_FLUSH_BYTES) flush();
+ if (final) resolve();
+ });
+ const write = async () => {
+ for (const [index, file] of files.entries()) {
+ const blob =
+ typeof file.blob === "function" ? await file.blob() : file.blob;
+ const entry = new ZipPassThrough(names[index]);
+ zip.add(entry);
+ const reader = blob.stream().getReader();
+ for (;;) {
+ const { done: end, value } = await reader.read();
+ if (end) break;
+ entry.push(value, false);
+ }
+ entry.push(new Uint8Array(0), true);
+ }
+ zip.end();
+ };
+ write().catch(reject);
+ });
+ flush();
+ return new Blob(parts, { type: "application/zip" });
+}
diff --git a/studio/frontend/src/features/audio/components/waveform.tsx b/studio/frontend/src/features/audio/components/waveform.tsx
index 24f5acbc0ff..fdf9fdbc228 100644
--- a/studio/frontend/src/features/audio/components/waveform.tsx
+++ b/studio/frontend/src/features/audio/components/waveform.tsx
@@ -6,6 +6,8 @@ import { cn } from "@/lib/utils";
import { PauseIcon, PlayIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import {
+ type CSSProperties,
+ type KeyboardEvent,
type ReactNode,
type Ref,
useCallback,
@@ -53,8 +55,6 @@ export function Waveform({
const duration =
durationS && durationS > 0 ? durationS : (mediaDuration ?? 0);
const fraction = duration > 0 ? Math.min(1, position / duration) : 0;
- const bars = peaks && peaks.length > 0 ? peaks : null;
- const count = bars?.length ?? WAVEFORM_BARS;
useEffect(() => {
setPlaying(false);
@@ -106,17 +106,15 @@ export function Waveform({
}, [onPositionChange, position, playing]);
const slider = (
- <div
- role="slider"
- tabIndex={src ? 0 : -1}
- aria-label={`${label} playback position`}
- aria-valuemin={0}
- aria-valuemax={Math.round(duration)}
- aria-valuenow={Math.round(position)}
- aria-valuetext={`${formatSeconds(position)} of ${formatSeconds(duration)}`}
- aria-disabled={!src}
+ <WaveformBars
+ peaks={peaks}
+ fraction={fraction}
+ label={label}
+ valueNow={position}
+ valueMax={duration}
+ disabled={!src}
style={tailS > 0 ? { flexGrow: duration > 0 ? duration : 1 } : undefined}
- className="relative h-[calc(32px*var(--ui-space-scale,1))] min-w-0 flex-1 cursor-pointer rounded-md focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
+ onSeek={(next) => seekTo(next * duration)}
onKeyDown={(event) => {
if (event.key === " ") {
event.preventDefault();
@@ -132,43 +130,7 @@ export function Waveform({
seekTo(0);
}
}}
- onClick={(event) => {
- const box = event.currentTarget.getBoundingClientRect();
- if (box.width <= 0) return;
- seekTo(((event.clientX - box.left) / box.width) * duration);
- }}
- >
- <svg
- aria-hidden="true"
- className="size-full"
- viewBox={`0 0 ${count * 3} 32`}
- preserveAspectRatio="none"
- >
- {Array.from({ length: count }, (_, index) => {
- const peak = bars ? (bars[index] ?? 0) : 0;
- const height = Math.max(2, peak * 30);
- const played = (index + 0.5) / count <= fraction;
- return (
- <rect
- // biome-ignore lint/suspicious/noArrayIndexKey: bars are positional and never reorder.
- key={index}
- x={index * 3 + 0.5}
- y={(32 - height) / 2}
- width={2}
- height={height}
- rx={1}
- className={cn(
- "fill-current",
- played && position > 0
- ? "text-foreground"
- : "text-muted-foreground",
- bars ? "opacity-100" : "opacity-40",
- )}
- />
- );
- })}
- </svg>
- </div>
+ />
);
return (
@@ -229,3 +191,86 @@ export function Waveform({
</div>
);
}
+
+/** Seekable bars; keys are left to the caller. */
+export function WaveformBars({
+ peaks,
+ fraction,
+ label,
+ valueNow,
+ valueMax,
+ disabled = false,
+ onSeek,
+ onKeyDown,
+ className,
+ style,
+}: {
+ peaks: readonly number[] | null;
+ fraction: number;
+ label: string;
+ valueNow: number;
+ valueMax: number;
+ disabled?: boolean;
+ onSeek?: (fraction: number) => void;
+ onKeyDown?: (event: KeyboardEvent<HTMLDivElement>) => void;
+ className?: string;
+ style?: CSSProperties;
+}) {
+ const bars = peaks && peaks.length > 0 ? peaks : null;
+ const count = bars?.length ?? WAVEFORM_BARS;
+ return (
+ <div
+ role="slider"
+ tabIndex={disabled ? -1 : 0}
+ aria-label={`${label} playback position`}
+ aria-valuemin={0}
+ aria-valuemax={Math.round(valueMax)}
+ aria-valuenow={Math.round(valueNow)}
+ aria-valuetext={`${formatSeconds(valueNow)} of ${formatSeconds(valueMax)}`}
+ aria-disabled={disabled}
+ style={style}
+ className={cn(
+ "relative h-[calc(32px*var(--ui-space-scale,1))] min-w-0 flex-1 cursor-pointer rounded-md focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring",
+ className,
+ )}
+ onKeyDown={onKeyDown}
+ onClick={(event) => {
+ if (disabled || !onSeek) return;
+ const box = event.currentTarget.getBoundingClientRect();
+ if (box.width <= 0) return;
+ onSeek(
+ Math.min(1, Math.max(0, (event.clientX - box.left) / box.width)),
+ );
+ }}
+ >
+ <svg
+ aria-hidden="true"
+ className="size-full"
+ viewBox={`0 0 ${count * 3} 32`}
+ preserveAspectRatio="none"
+ >
+ {Array.from({ length: count }, (_, index) => {
+ const peak = bars ? (bars[index] ?? 0) : 0;
+ const height = Math.max(2, peak * 30);
+ const played = fraction > 0 && (index + 0.5) / count <= fraction;
+ return (
+ <rect
+ // biome-ignore lint/suspicious/noArrayIndexKey: bars are positional and never reorder.
+ key={index}
+ x={index * 3 + 0.5}
+ y={(32 - height) / 2}
+ width={2}
+ height={height}
+ rx={1}
+ className={cn(
+ "fill-current",
+ played ? "text-foreground" : "text-muted-foreground",
+ bars ? "opacity-100" : "opacity-40",
+ )}
+ />
+ );
+ })}
+ </svg>
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx b/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
index 07247b7e01b..52ba8945157 100644
--- a/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
+++ b/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
@@ -64,7 +64,7 @@ export function useAudioGallery({
prompt: string;
model: string;
saved: boolean;
- workflow: "speak" | "music" | "clone";
+ workflow: "speak" | "music" | "clone" | "separate";
} | null>(null);
const fallbackClipRef = useRef(fallbackClip);
fallbackClipRef.current = fallbackClip;
@@ -425,7 +425,7 @@ export function useAudioGallery({
[refreshGallery],
);
- const handleClearGallery = useCallback(async (workflow?: "speak" | "clone" | "music") => {
+ const handleClearGallery = useCallback(async (workflow?: "speak" | "clone" | "music" | "separate") => {
try {
await clearAudioGallery(workflow);
if (workflow) {
@@ -554,7 +554,10 @@ export function useWorkflowHistory({
loadMore,
loadingMoreRef,
selectClip,
-}: { workflow: "speak" | "clone" | "music"; enabled: boolean } & Pick<
+}: {
+ workflow: "speak" | "clone" | "music" | "separate";
+ enabled: boolean;
+} & Pick<
AudioGallery,
| "clips"
| "hasMore"
diff --git a/studio/frontend/src/features/audio/hooks/use-clone-generation.ts b/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
index c27df39901c..9992733e5a2 100644
--- a/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
+++ b/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
@@ -55,7 +55,7 @@ export async function showRunResult({
}: {
response: AudioRunResponse;
text: string;
- workflow: "speak" | "clone" | "music";
+ workflow: "speak" | "clone" | "music" | "separate";
} & Pick<
AudioGallery,
"refreshGallery" | "selectClip" | "setFallbackClip" | "setSelectedId"
diff --git a/studio/frontend/src/features/audio/hooks/use-separate-generation.ts b/studio/frontend/src/features/audio/hooks/use-separate-generation.ts
new file mode 100644
index 00000000000..cf3c747020a
--- /dev/null
+++ b/studio/frontend/src/features/audio/hooks/use-separate-generation.ts
@@ -0,0 +1,326 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { toast } from "@/lib/toast";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { AudioApiError, runAudio } from "../api";
+import { selectionExpired, sourceRefOf } from "../audio-run-request";
+import type { AudioSourceInputHandle } from "../components/audio-source-input";
+import type { GenerateBlocker } from "../pages/tts-workspace";
+import {
+ type OverlapValue,
+ type SeparateBlockerKind,
+ estimateSeparateSeconds,
+ overlapReloads,
+ separateBlocker,
+ separateCpuWarning,
+} from "../separate-policy";
+import { useAudioCloneStore } from "../stores/audio-clone-store";
+import { useAudioSeparateStore } from "../stores/audio-separate-store";
+import { audioToolPanelsFor } from "../tools/registry";
+import {
+ audioModelContextFor,
+ collectToolRequest,
+ panelValue,
+ toolValueKey,
+} from "../tools/select";
+import type { AudioHostState } from "./audio-host-state";
+import type { AudioSourceStatus } from "./audio-source-state";
+import type { AudioGallery } from "./use-audio-gallery";
+import type { AudioModelSlot } from "./use-audio-model-slot";
+import { showRunResult } from "./use-clone-generation";
+
+export const SEPARATE_TRACK_EXPIRED_MESSAGE =
+ "This track expired. Add it again.";
+
+/** Mirrors useCloneGeneration so Stop, phases and errors behave the same. */
+export function useSeparateGeneration({
+ status,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ generateAbort,
+ setMode,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ audioDevice,
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ pendingTranscribeRelease,
+ replayQueuedTtsPick,
+}: Pick<
+ AudioHostState,
+ | "status"
+ | "busyRef"
+ | "setBusy"
+ | "updateGenerationPhase"
+ | "generateAbort"
+ | "setMode"
+ | "refreshStatus"
+ | "activeRef"
+ | "modeRef"
+ | "audioDevice"
+> &
+ Pick<
+ AudioGallery,
+ "refreshGallery" | "selectClip" | "setFallbackClip" | "setSelectedId"
+ > &
+ Pick<AudioModelSlot, "pendingTranscribeRelease" | "replayQueuedTtsPick">) {
+ const source = useAudioSeparateStore((state) => state.source);
+ const lastOverlapByModel = useAudioSeparateStore(
+ (state) => state.lastOverlapByModel,
+ );
+ const storedToolValues = useAudioCloneStore((state) => state.toolValues);
+ const model = status?.active_model ?? null;
+ const family = status?.audio_family ?? null;
+ const [generationError, setGenerationError] = useState<string | null>(null);
+ const [sourceStatus, setSourceStatus] = useState<AudioSourceStatus>({
+ phase: "idle",
+ });
+ const sourceHandle = useRef<AudioSourceInputHandle | null>(null);
+ const [expiredSourceId, setExpiredSourceId] = useState<string | null>(null);
+ const [lastResult, setLastResult] = useState<{
+ groupId: string | null;
+ stems: number;
+ } | null>(null);
+ const [reloading, setReloading] = useState(false);
+ const [pendingRun, setPendingRun] = useState<{
+ title: string;
+ model: string | null;
+ } | null>(null);
+
+ const toolContext = useMemo(
+ () =>
+ audioModelContextFor(status, {
+ musicGeneration: false,
+ cudaMusicGeneration: false,
+ musicNeedsDescription: false,
+ }),
+ [status],
+ );
+ const toolPanels = useMemo(
+ () => audioToolPanelsFor("separate", toolContext),
+ [toolContext],
+ );
+ const toolValues = useMemo(() => {
+ const values: Record<string, unknown> = {};
+ for (const panel of toolPanels) {
+ values[panel.id] =
+ storedToolValues[toolValueKey(model, "separate", panel.id)];
+ }
+ return values;
+ }, [toolPanels, storedToolValues, model]);
+ const handleToolValueChange = useCallback(
+ (panelId: string, value: unknown) =>
+ useAudioCloneStore
+ .getState()
+ .setToolValue(toolValueKey(model, "separate", panelId), value),
+ [model],
+ );
+ const toolRequest = useMemo(
+ () => collectToolRequest(toolPanels, toolValues, { text: "" }, toolContext),
+ [toolPanels, toolValues, toolContext],
+ );
+ const overlapPanel = toolPanels.find(
+ (panel) => panel.id === "roformer-overlap",
+ );
+ const overlap = overlapPanel
+ ? panelValue<OverlapValue>(overlapPanel, toolValues, []).overlap
+ : true;
+ const willReload =
+ overlapPanel !== undefined &&
+ model !== null &&
+ overlapReloads(lastOverlapByModel[model], overlap);
+
+ const [now, setNow] = useState(() => Date.now());
+ useEffect(() => {
+ if (source?.kind !== "input" || !source.expiresAt) return;
+ const timer = window.setInterval(() => setNow(Date.now()), 60_000);
+ return () => window.clearInterval(timer);
+ }, [source]);
+ const sourceExpired =
+ sourceStatus.phase === "expired" ||
+ selectionExpired(source, now) ||
+ (source !== null && source.id === expiredSourceId);
+
+ const inputBlocker = separateBlocker({
+ source,
+ sourceBusy:
+ sourceStatus.phase === "uploading" || sourceStatus.phase === "recording",
+ sourceExpired,
+ sourceError: sourceStatus.phase === "error" ? sourceStatus.message : null,
+ });
+ const addTrack = {
+ label: "Add a track",
+ onClick: () => sourceHandle.current?.focus(),
+ };
+ const blockerActions: Partial<
+ Record<SeparateBlockerKind, GenerateBlocker["actions"]>
+ > = {
+ source: [addTrack],
+ "source-error": [addTrack],
+ "source-expired": [
+ {
+ label: "Add it again",
+ onClick: () => {
+ useAudioSeparateStore.getState().setSource(null);
+ sourceHandle.current?.browse();
+ },
+ },
+ ],
+ "too-long": [
+ {
+ label: "Pick a shorter track",
+ onClick: () => sourceHandle.current?.browse(),
+ },
+ ],
+ };
+ const blocker: GenerateBlocker | null = inputBlocker
+ ? {
+ reason: inputBlocker.reason,
+ actions: blockerActions[inputBlocker.kind],
+ }
+ : null;
+
+ const estimateSeconds = estimateSeparateSeconds(
+ family,
+ source?.durationS,
+ overlap,
+ );
+ const cpuWarning = separateCpuWarning(family, audioDevice);
+
+ const handleGenerate = useCallback(async () => {
+ const state = useAudioSeparateStore.getState();
+ if (!state.source || inputBlocker) return;
+ if (busyRef.current) return;
+ busyRef.current = "generating";
+ setBusy("generating");
+ setLastResult(null);
+ updateGenerationPhase("preparing");
+ const releaseInFlight = pendingTranscribeRelease.current;
+ if (releaseInFlight && !(await releaseInFlight)) {
+ updateGenerationPhase(null);
+ busyRef.current = null;
+ setBusy(null);
+ setMode("transcribe");
+ return;
+ }
+ const runModel = model;
+ const controller = new AbortController();
+ generateAbort.current = controller;
+ setReloading(willReload);
+ setPendingRun({ title: state.source.name, model: runModel });
+ updateGenerationPhase("generating");
+ try {
+ const response = await runAudio(
+ {
+ workflow: "separate",
+ inputs: { source: sourceRefOf(state.source) },
+ options: toolRequest.patch.options,
+ },
+ controller.signal,
+ );
+ if (runModel && overlapPanel) {
+ useAudioSeparateStore.getState().setLastOverlap(runModel, overlap);
+ }
+ updateGenerationPhase("finishing");
+ await showRunResult({
+ response,
+ text: state.source.name,
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ workflow: "separate",
+ });
+ setLastResult({
+ groupId: response.group_id,
+ stems: response.clips.length,
+ });
+ } catch (error) {
+ if (!controller.signal.aborted) {
+ updateGenerationPhase("finishing");
+ const expired =
+ error instanceof AudioApiError &&
+ error.status === 404 &&
+ state.source.kind === "input";
+ if (expired) {
+ setExpiredSourceId(state.source.id);
+ sourceHandle.current?.markExpired();
+ }
+ const message = expired
+ ? SEPARATE_TRACK_EXPIRED_MESSAGE
+ : error instanceof Error
+ ? error.message
+ : "Separation failed.";
+ setGenerationError(message);
+ if (!expired) toast.error(message);
+ await refreshStatus();
+ }
+ } finally {
+ generateAbort.current = null;
+ setReloading(false);
+ setPendingRun(null);
+ updateGenerationPhase(null);
+ busyRef.current = null;
+ setBusy(null);
+ if (activeRef.current && modeRef.current === "speak")
+ replayQueuedTtsPick();
+ }
+ }, [
+ inputBlocker,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ pendingTranscribeRelease,
+ setMode,
+ model,
+ generateAbort,
+ willReload,
+ toolRequest,
+ overlapPanel,
+ overlap,
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ replayQueuedTtsPick,
+ ]);
+
+ useEffect(() => {
+ if (source) setGenerationError(null);
+ }, [source]);
+
+ return {
+ source,
+ sourceHandle,
+ sourceStatus,
+ setSourceStatus,
+ sourceExpired,
+ toolContext,
+ toolPanels,
+ toolValues,
+ handleToolValueChange,
+ family,
+ overlap,
+ willReload,
+ reloading,
+ pendingRun,
+ estimateSeconds,
+ cpuWarning,
+ blocker,
+ handleGenerate,
+ generationError,
+ setGenerationError,
+ lastResult,
+ clearLastResult: () => setLastResult(null),
+ };
+}
+
+export type SeparateGeneration = ReturnType<typeof useSeparateGeneration>;
diff --git a/studio/frontend/src/features/audio/hooks/use-stem-sources.ts b/studio/frontend/src/features/audio/hooks/use-stem-sources.ts
new file mode 100644
index 00000000000..c885c0dbc19
--- /dev/null
+++ b/studio/frontend/src/features/audio/hooks/use-stem-sources.ts
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { BlobUrlCache } from "@/lib/blob-url-cache";
+import { useEffect, useState } from "react";
+import { fetchAudioBlob } from "../api";
+import { computePeaks } from "../components/waveform-peaks";
+
+// Outlives groups so going back redraws at once; insertion order is the LRU order.
+const PEAKS_CAP = 64;
+const peaksByClip = new Map<string, readonly number[]>();
+
+function rememberPeaks(clipId: string, peaks: readonly number[]): void {
+ peaksByClip.delete(clipId);
+ peaksByClip.set(clipId, peaks);
+ while (peaksByClip.size > PEAKS_CAP) {
+ const oldest = peaksByClip.keys().next().value;
+ if (oldest === undefined) break;
+ peaksByClip.delete(oldest);
+ }
+}
+
+function cachedPeaks(clipId: string): readonly number[] | undefined {
+ const peaks = peaksByClip.get(clipId);
+ if (peaks) rememberPeaks(clipId, peaks);
+ return peaks;
+}
+
+// Peaks only need the shape: decoding at 22.05 kHz halves the float buffer.
+const PEAKS_DECODE_RATE = 22_050;
+
+// From the Blob, not its object URL: the page's CSP (connect-src) refuses fetch() on blob: URLs.
+async function decodePeaks(blob: Blob): Promise<number[]> {
+ const bytes = await blob.arrayBuffer();
+ const context = new OfflineAudioContext(1, 1, PEAKS_DECODE_RATE);
+ const buffer = await context.decodeAudioData(bytes);
+ const channels: Float32Array[] = [];
+ for (let index = 0; index < buffer.numberOfChannels; index += 1)
+ channels.push(buffer.getChannelData(index));
+ return computePeaks(channels);
+}
+
+export interface StemSourceInput {
+ clipId: string;
+ url: string;
+}
+
+export interface StemSources {
+ srcById: Readonly<Record<string, string>>;
+ peaksById: Readonly<Record<string, readonly number[]>>;
+ failedIds: readonly string[];
+}
+
+const EMPTY: StemSources = { srcById: {}, peaksById: {}, failedIds: [] };
+
+function groupKey(stems: readonly StemSourceInput[]): string {
+ return stems.map((stem) => `${stem.clipId}\u0000${stem.url}`).join("\u0001");
+}
+
+/** Own cache, not the gallery's budgeted LRU, which would evict stems mid-playback. */
+export function useStemSources(
+ stems: readonly StemSourceInput[],
+ attempt = 0,
+): StemSources {
+ const key = groupKey(stems);
+ const stateKey = `${attempt}\u0002${key}`;
+ const [state, setState] = useState<{ key: string } & StemSources>({
+ key: "",
+ ...EMPTY,
+ });
+
+ useEffect(() => {
+ if (!key) return;
+ const group: StemSourceInput[] = key.split("\u0001").map((entry) => {
+ const [clipId, url] = entry.split("\u0000");
+ return { clipId, url };
+ });
+ let cancelled = false;
+ // Sized to the group, so prune is never called.
+ const cache = new BlobUrlCache(Number.POSITIVE_INFINITY);
+ const peaksById: Record<string, readonly number[]> = {};
+ const failedIds: string[] = [];
+ const blobs = new Map<string, Blob>();
+ for (const { clipId } of group) {
+ const peaks = cachedPeaks(clipId);
+ if (peaks) peaksById[clipId] = peaks;
+ }
+ const publish = () => {
+ if (cancelled) return;
+ setState({
+ key: stateKey,
+ srcById: cache.toRecord(),
+ peaksById: { ...peaksById },
+ failedIds: [...failedIds],
+ });
+ };
+ publish();
+
+ const loadSrc = async ({ clipId, url }: StemSourceInput) => {
+ try {
+ const blob = await fetchAudioBlob(url);
+ if (cancelled) return;
+ cache.set(clipId, URL.createObjectURL(blob), blob.size);
+ blobs.set(clipId, blob);
+ } catch {
+ if (cancelled) return;
+ failedIds.push(clipId);
+ }
+ publish();
+ };
+
+ void (async () => {
+ const loads = new Map(
+ group.map((stem) => [stem.clipId, loadSrc(stem)] as const),
+ );
+ for (const { clipId } of group) {
+ await loads.get(clipId);
+ if (cancelled) return;
+ const blob = blobs.get(clipId);
+ blobs.delete(clipId);
+ if (peaksById[clipId] || !blob) continue;
+ try {
+ const peaks = await decodePeaks(blob);
+ rememberPeaks(clipId, peaks);
+ if (cancelled) return;
+ peaksById[clipId] = peaks;
+ publish();
+ } catch {
+ // Undecodable: the row keeps its flat placeholder and still plays.
+ }
+ }
+ })();
+
+ return () => {
+ cancelled = true;
+ cache.clear();
+ };
+ }, [key, stateKey]);
+
+ return state.key === stateKey ? state : EMPTY;
+}
diff --git a/studio/frontend/src/features/audio/hooks/use-stem-transport.ts b/studio/frontend/src/features/audio/hooks/use-stem-transport.ts
new file mode 100644
index 00000000000..36b0fac3abf
--- /dev/null
+++ b/studio/frontend/src/features/audio/hooks/use-stem-transport.ts
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useCallback, useEffect, useRef, useState } from "react";
+
+const DRIFT_TOLERANCE_S = 0.04;
+const DRIFT_CHECK_MS = 250;
+const CANPLAY_TIMEOUT_MS = 10_000;
+
+export interface StemTransportSource {
+ id: string;
+ src: string | null;
+}
+
+export interface StemTransport {
+ ready: boolean;
+ playing: boolean;
+ position: number;
+ duration: number;
+ play(): void;
+ pause(): void;
+ toggle(): void;
+ seek(seconds: number): void;
+}
+
+interface StemVoice {
+ element: HTMLAudioElement;
+ source: MediaElementAudioSourceNode | null;
+ gain: GainNode | null;
+}
+
+function waitForCanPlay(element: HTMLAudioElement): Promise<void> {
+ const canPlay = () =>
+ !element.seeking && element.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA;
+ if (canPlay()) return Promise.resolve();
+ const events = ["canplay", "canplaythrough", "seeked", "error"] as const;
+ return new Promise((resolve) => {
+ const finish = () => {
+ window.clearTimeout(timer);
+ for (const name of events) element.removeEventListener(name, check);
+ resolve();
+ };
+ const check = () => {
+ if (canPlay() || element.error) finish();
+ };
+ const timer = window.setTimeout(finish, CANPLAY_TIMEOUT_MS);
+ for (const name of events) element.addEventListener(name, check);
+ });
+}
+
+function setGain(voice: StemVoice, value: number, context: AudioContext) {
+ voice.gain?.gain.setTargetAtTime(value, context.currentTime, 0.015);
+}
+
+/** Media elements through gain nodes, not decoded buffers, so memory stays near the blob size. */
+export function useStemTransport({
+ sources,
+ gains,
+ active,
+ durationHint = 0,
+}: {
+ sources: readonly StemTransportSource[];
+ gains: readonly number[];
+ active: boolean;
+ durationHint?: number;
+}): StemTransport {
+ const contextRef = useRef<AudioContext | null>(null);
+ const voicesRef = useRef<StemVoice[]>([]);
+ const gainsRef = useRef(gains);
+ gainsRef.current = gains;
+ const positionRef = useRef(0);
+ const playingRef = useRef(false);
+ // Lets a start still waiting on `canplay` know it is stale.
+ const startToken = useRef(0);
+ const [playing, setPlaying] = useState(false);
+ const [position, setPosition] = useState(0);
+ const [mediaDuration, setMediaDuration] = useState(0);
+
+ const ready = sources.length > 0 && sources.every((source) => source.src);
+ const srcKey = ready
+ ? sources.map((source) => source.src ?? "").join("\u0000")
+ : "";
+ const duration =
+ mediaDuration > 0 ? mediaDuration : Math.max(0, durationHint);
+ const durationRef = useRef(duration);
+ durationRef.current = duration;
+
+ const updatePosition = useCallback((seconds: number) => {
+ positionRef.current = seconds;
+ setPosition(seconds);
+ }, []);
+
+ const stopAll = useCallback(() => {
+ startToken.current += 1;
+ playingRef.current = false;
+ setPlaying(false);
+ for (const voice of voicesRef.current) voice.element.pause();
+ }, []);
+
+ useEffect(() => {
+ if (!srcKey) return;
+ const voices: StemVoice[] = srcKey.split("\u0000").map((src) => {
+ const element = new Audio();
+ element.preload = "auto";
+ element.src = src;
+ return { element, source: null, gain: null };
+ });
+ voicesRef.current = voices;
+ updatePosition(0);
+ setMediaDuration(0);
+ const first = voices[0]?.element;
+ const onMetadata = () => {
+ const lengths = voices
+ .map((voice) => voice.element.duration)
+ .filter((value) => Number.isFinite(value) && value > 0);
+ if (lengths.length > 0) setMediaDuration(Math.max(...lengths));
+ };
+ const onEnded = () => {
+ startToken.current += 1;
+ playingRef.current = false;
+ setPlaying(false);
+ for (const voice of voices) voice.element.pause();
+ updatePosition(0);
+ for (const voice of voices) voice.element.currentTime = 0;
+ };
+ for (const voice of voices)
+ voice.element.addEventListener("loadedmetadata", onMetadata);
+ first?.addEventListener("ended", onEnded);
+ return () => {
+ startToken.current += 1;
+ playingRef.current = false;
+ setPlaying(false);
+ first?.removeEventListener("ended", onEnded);
+ for (const voice of voices) {
+ voice.element.removeEventListener("loadedmetadata", onMetadata);
+ voice.element.pause();
+ voice.source?.disconnect();
+ voice.gain?.disconnect();
+ voice.element.removeAttribute("src");
+ voice.element.load();
+ }
+ if (voicesRef.current === voices) voicesRef.current = [];
+ };
+ }, [srcKey, updatePosition]);
+
+ useEffect(
+ () => () => {
+ const context = contextRef.current;
+ contextRef.current = null;
+ void context?.close().catch(() => {});
+ },
+ [],
+ );
+
+ // Created on the first Play, a user gesture, so the browser lets the context run.
+ const ensureGraph = useCallback((): AudioContext => {
+ let context = contextRef.current;
+ if (!context) {
+ context = new AudioContext();
+ contextRef.current = context;
+ }
+ for (const [index, voice] of voicesRef.current.entries()) {
+ if (voice.source) continue;
+ voice.source = context.createMediaElementSource(voice.element);
+ voice.gain = context.createGain();
+ voice.gain.gain.value = gainsRef.current[index] ?? 1;
+ voice.source.connect(voice.gain).connect(context.destination);
+ }
+ return context;
+ }, []);
+
+ const gainKey = gains.join(",");
+ useEffect(() => {
+ const context = contextRef.current;
+ if (!context) return;
+ const values = gainKey.split(",").map(Number);
+ for (const [index, voice] of voicesRef.current.entries())
+ setGain(voice, values[index] ?? 1, context);
+ }, [gainKey]);
+
+ const start = useCallback(
+ async (from: number) => {
+ const voices = voicesRef.current;
+ if (voices.length === 0) return;
+ const token = ++startToken.current;
+ const context = ensureGraph();
+ for (const voice of voices) voice.element.pause();
+ for (const voice of voices) voice.element.currentTime = from;
+ updatePosition(from);
+ playingRef.current = true;
+ setPlaying(true);
+ try {
+ await context.resume();
+ await Promise.all(voices.map((voice) => waitForCanPlay(voice.element)));
+ if (token !== startToken.current) return;
+ await Promise.all(voices.map((voice) => voice.element.play()));
+ } catch {
+ if (token === startToken.current) stopAll();
+ }
+ },
+ [ensureGraph, stopAll, updatePosition],
+ );
+
+ const play = useCallback(() => {
+ if (playingRef.current) return;
+ const from =
+ positionRef.current >= durationRef.current - 0.05
+ ? 0
+ : positionRef.current;
+ void start(from);
+ }, [start]);
+
+ const pause = useCallback(() => {
+ if (!playingRef.current) return;
+ const current = voicesRef.current[0]?.element.currentTime;
+ stopAll();
+ if (current !== undefined && Number.isFinite(current))
+ updatePosition(current);
+ }, [stopAll, updatePosition]);
+
+ const toggle = useCallback(() => {
+ if (playingRef.current) pause();
+ else play();
+ }, [pause, play]);
+
+ const seek = useCallback(
+ (seconds: number) => {
+ const end = durationRef.current;
+ const next = Math.min(end > 0 ? end : 0, Math.max(0, seconds));
+ if (playingRef.current) {
+ void start(next);
+ return;
+ }
+ startToken.current += 1;
+ updatePosition(next);
+ for (const voice of voicesRef.current) voice.element.currentTime = next;
+ },
+ [start, updatePosition],
+ );
+
+ useEffect(() => {
+ if (!playing) return;
+ const timer = window.setInterval(() => {
+ const voices = voicesRef.current;
+ const lead = voices[0]?.element;
+ if (!lead || lead.paused || lead.seeking) return;
+ const now = lead.currentTime;
+ updatePosition(now);
+ for (const voice of voices.slice(1)) {
+ const element = voice.element;
+ if (element.seeking || element.ended) continue;
+ if (Math.abs(element.currentTime - now) > DRIFT_TOLERANCE_S)
+ element.currentTime = now;
+ }
+ }, DRIFT_CHECK_MS);
+ return () => window.clearInterval(timer);
+ }, [playing, updatePosition]);
+
+ useEffect(() => {
+ if (!active) pause();
+ }, [active, pause]);
+
+ return {
+ ready: Boolean(srcKey),
+ playing,
+ position,
+ duration,
+ play,
+ pause,
+ toggle,
+ seek,
+ };
+}
diff --git a/studio/frontend/src/features/audio/pages/separate-page.tsx b/studio/frontend/src/features/audio/pages/separate-page.tsx
new file mode 100644
index 00000000000..6453f36adfa
--- /dev/null
+++ b/studio/frontend/src/features/audio/pages/separate-page.tsx
@@ -0,0 +1,674 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import {
+ GalleryItemMenu,
+ GalleryPinBadge,
+} from "@/components/gallery-item-menu";
+import { Button } from "@/components/ui/button";
+import type { ModelOption } from "@/features/model-picker/components/model-selector/types";
+import { toast } from "@/lib/toast";
+import { cn } from "@/lib/utils";
+import { AudioWave01Icon, Download01Icon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import { type ComponentProps, useEffect, useMemo, useState } from "react";
+import {
+ type AudioGalleryClip,
+ addAudioClipToProject,
+ fetchAudioBlob,
+ setAudioClipFlags,
+} from "../api";
+import { audioCppModelFor, audioCppWorkflowsFor } from "../audio-cpp-catalog";
+import { audioModelLabel, formatClipDuration } from "../audio-workspace-utils";
+import { macTtsCatalogChoiceIsRunnable } from "../catalog";
+import {
+ AudioHistoryProvider,
+ AudioSourceInput,
+} from "../components/audio-source-input";
+import { StemMixer } from "../components/stem-mixer";
+import type { SendTarget } from "../components/stem-mixer-types";
+import { stemFileName, stemZipName, zipStems } from "../components/stem-zip";
+import type { AudioHostState } from "../hooks/audio-host-state";
+import type { AudioGallery } from "../hooks/use-audio-gallery";
+import {
+ SEPARATE_TRACK_EXPIRED_MESSAGE,
+ type SeparateGeneration,
+} from "../hooks/use-separate-generation";
+import { useStemSources } from "../hooks/use-stem-sources";
+import {
+ SEPARATE_MAX_SECONDS,
+ separatePresentation,
+} from "../separate-policy";
+import {
+ SEPARATION_STEMS_BY_FAMILY,
+ type SeparationGroup,
+ groupSeparationClips,
+ stemLabel,
+} from "../separation-stems";
+import { useAudioSeparateStore } from "../stores/audio-separate-store";
+import { AudioToolPanels } from "../tools/tool-panel-host";
+import { TtsFooter, TtsRailFields } from "./tts-workspace";
+
+const SEPARATE_MODEL_ORDER = [
+ "HTDemucs-GGUF",
+ "BS-RoFormer-ep368-GGUF",
+ "HTDemucs-6stems-GGUF",
+ "Mel-Band-RoFormer-GGUF",
+];
+
+function separateRank(id: string): number {
+ const index = SEPARATE_MODEL_ORDER.findIndex((name) =>
+ id.endsWith(`/${name}`),
+ );
+ return index === -1 ? SEPARATE_MODEL_ORDER.length : index;
+}
+
+export function separatePageModels(
+ models: ModelOption[],
+ isMac: boolean,
+): ModelOption[] {
+ return models
+ .filter((model) => {
+ const entry = audioCppModelFor(model.id);
+ return (
+ entry !== null &&
+ audioCppWorkflowsFor(entry).includes("separate") &&
+ (!isMac || macTtsCatalogChoiceIsRunnable(model.id))
+ );
+ })
+ .map((model, index) => ({ model, index }))
+ .sort(
+ (a, b) =>
+ separateRank(a.model.id) - separateRank(b.model.id) ||
+ a.index - b.index,
+ )
+ .map(({ model }) => model);
+}
+
+export function separateOutputStems(
+ family: string | null | undefined,
+ modelId: string | null | undefined,
+): readonly string[] | null {
+ const known = family ? SEPARATION_STEMS_BY_FAMILY[family] : undefined;
+ if (known) return known.stems;
+ const stems = audioCppModelFor(modelId)?.stems;
+ return stems ?? null;
+}
+
+type RailProps = Omit<
+ ComponentProps<typeof TtsRailFields>,
+ | "musicGeneration"
+ | "prompt"
+ | "setPrompt"
+ | "toolPanels"
+ | "inputs"
+ | "claimedOptions"
+ | "samplingControls"
+ | "audioOptionSpecs"
+>;
+
+function StemChips({ stems }: { stems: readonly string[] }) {
+ return (
+ <div className="grid gap-1">
+ <span className="text-ui-13 font-medium text-foreground">Outputs</span>
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {stems.map(stemLabel).join(" · ")}
+ </p>
+ </div>
+ );
+}
+
+function SeparateInputs({
+ separate,
+ historyClips,
+ disabled,
+ pageModelLoaded,
+ pageModelId,
+}: {
+ separate: SeparateGeneration;
+ historyClips: readonly AudioGalleryClip[];
+ disabled: boolean;
+ pageModelLoaded: boolean;
+ pageModelId: string | null;
+}) {
+ const setSource = useAudioSeparateStore((state) => state.setSource);
+ const outputs = separateOutputStems(
+ pageModelLoaded ? separate.family : null,
+ pageModelId,
+ );
+ return (
+ <AudioHistoryProvider value={historyClips}>
+ <div data-tour="audio-separate-source">
+ <AudioSourceInput
+ id="separate-source"
+ label="Track"
+ hint="Converted to 44.1 kHz automatically. Up to 10 minutes."
+ value={separate.source}
+ onChange={setSource}
+ disabled={disabled}
+ allowSavedVoice={false}
+ usesFirstSeconds={null}
+ expiredMessage={SEPARATE_TRACK_EXPIRED_MESSAGE}
+ recordHint="Play the track near the mic, up to 10 minutes."
+ // A second under the cap: the timer fires late, and the server allows no overrun.
+ maxRecordSeconds={SEPARATE_MAX_SECONDS - 1}
+ handleRef={separate.sourceHandle}
+ onStatusChange={separate.setSourceStatus}
+ />
+ </div>
+ {outputs ? <StemChips stems={outputs} /> : null}
+ {pageModelLoaded ? null : (
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ Loading a separation model replaces the model in the main slot.
+ </p>
+ )}
+ {pageModelLoaded && separate.cpuWarning ? (
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {separate.cpuWarning}
+ </p>
+ ) : null}
+ </AudioHistoryProvider>
+ );
+}
+
+export function SeparateRail({
+ separate,
+ historyClips,
+ pageModelLoaded,
+ pageModelId,
+ ...props
+}: RailProps & {
+ separate: SeparateGeneration;
+ historyClips: readonly AudioGalleryClip[];
+ pageModelLoaded: boolean;
+ pageModelId: string | null;
+}) {
+ const disabled = props.busy === "generating";
+ return (
+ <TtsRailFields
+ {...props}
+ musicGeneration={false}
+ prompt=""
+ setPrompt={() => {}}
+ samplingControls={false}
+ audioOptionSpecs={[]}
+ inputs={
+ <SeparateInputs
+ separate={separate}
+ historyClips={historyClips}
+ disabled={disabled}
+ pageModelLoaded={pageModelLoaded}
+ pageModelId={pageModelId}
+ />
+ }
+ toolPanels={
+ <AudioToolPanels
+ workflow="separate"
+ ctx={separate.toolContext}
+ values={separate.toolValues}
+ onChange={separate.handleToolValueChange}
+ specs={[]}
+ disabled={disabled}
+ core={{ text: "" }}
+ />
+ }
+ />
+ );
+}
+
+export function SeparateFooter({
+ separate,
+ generationPresentation,
+ generationPhase,
+ ...props
+}: Omit<
+ ComponentProps<typeof TtsFooter>,
+ | "prompt"
+ | "lyricsOptional"
+ | "musicNeedsDescription"
+ | "audioInstructions"
+ | "handleGenerate"
+ | "secondaryAction"
+> & {
+ separate: SeparateGeneration;
+ generationPhase: string | null;
+}) {
+ const presentation = separatePresentation(
+ generationPresentation,
+ generationPhase,
+ separate.reloading,
+ );
+ const ready =
+ props.ttsLoaded && props.blocker === null && props.busy === null;
+ return (
+ <div className="flex w-full max-w-sm flex-col items-center gap-1.5">
+ <TtsFooter
+ {...props}
+ generationPresentation={presentation}
+ prompt=""
+ lyricsOptional={true}
+ musicNeedsDescription={false}
+ audioInstructions=""
+ handleGenerate={separate.handleGenerate}
+ />
+ {ready && !presentation && !props.error ? (
+ <p className="text-center text-ui-11p5 leading-snug text-muted-foreground">
+ {separate.willReload
+ ? "Reloads the model with the new overlap first (a few seconds). "
+ : ""}
+ {separate.estimateSeconds !== null && !separate.cpuWarning ? (
+ <>
+ About{" "}
+ <span className="font-mono tabular-nums">
+ {separate.estimateSeconds}
+ </span>{" "}
+ s on the GPU.
+ </>
+ ) : null}
+ </p>
+ ) : null}
+ </div>
+ );
+}
+
+function saveBlob(blob: Blob, name: string) {
+ const url = URL.createObjectURL(blob);
+ const anchor = document.createElement("a");
+ anchor.href = url;
+ anchor.download = name;
+ anchor.click();
+ window.setTimeout(() => URL.revokeObjectURL(url), 0);
+}
+
+async function downloadGroup(group: SeparationGroup) {
+ try {
+ // Each stem is fetched when the zip reaches it, so a long song never holds every stem at once.
+ const files = group.stems.map((clip) => ({
+ name: stemFileName(group.title, stemLabel(clip.role ?? "")),
+ blob: () => fetchAudioBlob(clip.url),
+ }));
+ saveBlob(await zipStems(files), stemZipName(group.title));
+ } catch {
+ toast.error("Could not download the stems.");
+ }
+}
+
+const SEND_TARGETS: readonly SendTarget[] = [
+ { id: "transcribe", workflow: "transcribe", label: "Transcribe" },
+ { id: "clone", workflow: "clone", label: "Clone (as reference)" },
+];
+
+/** Keyed by group so each one starts from its own clock. */
+function SelectedSeparation({
+ group,
+ autoFocus,
+ active,
+ onSendStem,
+}: {
+ group: SeparationGroup;
+ autoFocus: boolean;
+ active: boolean;
+ onSendStem: (
+ target: SendTarget,
+ clip: AudioGalleryClip,
+ name: string,
+ ) => void;
+}) {
+ const inputs = useMemo(
+ () => group.stems.map((clip) => ({ clipId: clip.id, url: clip.url })),
+ [group.stems],
+ );
+ const [attempt, setAttempt] = useState(0);
+ const sources = useStemSources(inputs, attempt);
+ // Stable while the group and its fetched audio are, so the transport keeps its players.
+ const stems = useMemo(
+ () =>
+ group.stems.map((clip) => ({
+ clipId: clip.id,
+ role: clip.role ?? clip.id,
+ label: stemLabel(clip.role ?? ""),
+ src: sources.srcById[clip.id] ?? null,
+ failed: sources.failedIds.includes(clip.id),
+ durationS: clip.duration_s,
+ peaks: sources.peaksById[clip.id] ?? null,
+ })),
+ [group.stems, sources.srcById, sources.peaksById, sources.failedIds],
+ );
+ return (
+ <>
+ <StemMixer
+ groupId={group.groupId}
+ title={group.title}
+ subtitle={audioModelLabel(group.model)}
+ stems={stems}
+ autoFocus={autoFocus}
+ active={active}
+ sendTargets={SEND_TARGETS}
+ onDownloadStem={(clipId) => {
+ const clip = group.stems.find((item) => item.id === clipId);
+ const src = sources.srcById[clipId];
+ if (!clip || !src) return;
+ const anchor = document.createElement("a");
+ anchor.href = src;
+ anchor.download = stemFileName(
+ group.title,
+ stemLabel(clip.role ?? ""),
+ );
+ anchor.click();
+ }}
+ onDownloadAll={() => void downloadGroup(group)}
+ onSend={(target, clipId) => {
+ const clip = group.stems.find((item) => item.id === clipId);
+ if (!clip) return;
+ onSendStem(
+ target,
+ clip,
+ `${group.title.replace(/\.[a-z0-9]{2,4}$/i, "")} - ${stemLabel(clip.role ?? "")}`,
+ );
+ }}
+ />
+ {sources.failedIds.length > 0 ? (
+ <p role="alert" className="mt-2 text-ui-11p5 text-muted-foreground">
+ Could not load {sources.failedIds.length} of {group.stems.length}{" "}
+ stems.{" "}
+ <Button
+ variant="link"
+ size="sm"
+ className="h-auto p-0"
+ onClick={() => setAttempt((n) => n + 1)}
+ >
+ Try again
+ </Button>
+ </p>
+ ) : null}
+ </>
+ );
+}
+
+export function SeparateOutput({
+ clips,
+ selectedId,
+ selectClip,
+ fallbackClip,
+ handleDownloadFallbackClip,
+ handleDeleteClip,
+ handleArchiveClip,
+ handleTogglePin,
+ handleClearGallery,
+ hasMore,
+ loadMore,
+ active,
+ modelReady,
+ recommendedModels = [],
+ onPickModel,
+ separate,
+ onSendStem,
+}: Pick<
+ AudioGallery,
+ | "selectedId"
+ | "selectClip"
+ | "fallbackClip"
+ | "handleDownloadFallbackClip"
+ | "handleDeleteClip"
+ | "handleArchiveClip"
+ | "handleTogglePin"
+ | "hasMore"
+ | "loadMore"
+> &
+ Pick<AudioHostState, "active"> & {
+ clips: AudioGalleryClip[];
+ handleClearGallery: () => Promise<void>;
+ modelReady: boolean;
+ recommendedModels?: readonly ModelOption[];
+ onPickModel?: (id: string) => void;
+ separate: SeparateGeneration;
+ onSendStem: (
+ target: SendTarget,
+ clip: AudioGalleryClip,
+ name: string,
+ ) => void;
+ }) {
+ const groups = useMemo(
+ () => groupSeparationClips(clips, hasMore),
+ [clips, hasMore],
+ );
+ // The hidden oldest run waits on the next page; a short list never scrolls, so fetch it here.
+ const tailHidden = useMemo(
+ () => hasMore && groupSeparationClips(clips).length > groups.length,
+ [clips, hasMore, groups],
+ );
+ // biome-ignore lint/correctness/useExhaustiveDependencies: each loaded page re-checks, in case the run is still cut.
+ useEffect(() => {
+ if (tailHidden) void loadMore();
+ }, [tailHidden, clips, loadMore]);
+ const selected =
+ groups.find((group) =>
+ group.stems.some((clip) => clip.id === selectedId),
+ ) ?? null;
+ const { lastResult, clearLastResult, pendingRun } = separate;
+ const fresh =
+ lastResult !== null &&
+ selected !== null &&
+ (lastResult.groupId === null || selected.groupId === lastResult.groupId);
+ // Once only; later re-renders keep focus where the user put it.
+ const [focusedGroup, setFocusedGroup] = useState<string | null>(null);
+ useEffect(() => {
+ if (fresh && selected) setFocusedGroup(selected.groupId);
+ }, [fresh, selected]);
+ const announcement = lastResult
+ ? `Separated into ${lastResult.stems} stem${lastResult.stems === 1 ? "" : "s"}.`
+ : "";
+ const picks = modelReady || !onPickModel ? [] : recommendedModels.slice(0, 3);
+
+ const pinGroup = (group: SeparationGroup, pinned: boolean) => {
+ for (const clip of group.stems) void handleTogglePin(clip.id, pinned);
+ };
+ const archiveGroup = async (group: SeparationGroup) => {
+ const [last, ...rest] = [...group.stems].reverse();
+ try {
+ await Promise.all(
+ rest.map((clip) => setAudioClipFlags(clip.id, { archived: true })),
+ );
+ } catch (error) {
+ toast.error(
+ error instanceof Error ? error.message : "Could not archive the stems.",
+ );
+ return;
+ }
+ if (last) await handleArchiveClip(last.id);
+ };
+ const deleteGroup = async (group: SeparationGroup) => {
+ for (const clip of group.stems) await handleDeleteClip(clip.id);
+ };
+
+ return (
+ <>
+ <output aria-live="polite" aria-atomic="true" className="sr-only">
+ {announcement}
+ </output>
+ <div
+ className={cn(
+ "flex min-h-0 flex-1 flex-col items-center gap-4",
+ selected || pendingRun ? "justify-start" : "justify-center",
+ )}
+ >
+ {pendingRun ? (
+ <output className="corner-squircle grid w-full gap-1 rounded-4xl bg-card p-4 ring-1 ring-[color-mix(in_oklab,var(--foreground)_calc(10%*var(--contrast-edge-gain,1)),transparent)]">
+ <span className="block truncate text-ui-13 font-medium text-foreground">
+ {pendingRun.title}
+ </span>
+ <span className="block text-ui-11p5 text-muted-foreground">
+ {pendingRun.model
+ ? `Separating with ${audioModelLabel(pendingRun.model)}…`
+ : "Separating…"}{" "}
+ The stems appear here when they are ready.
+ </span>
+ </output>
+ ) : selected ? (
+ <div className="w-full">
+ <SelectedSeparation
+ key={selected.groupId}
+ group={selected}
+ autoFocus={focusedGroup === selected.groupId}
+ active={active}
+ onSendStem={(target, clip, name) => {
+ clearLastResult();
+ onSendStem(target, clip, name);
+ }}
+ />
+ {selected.complete ? null : (
+ <p className="mt-2 text-ui-11p5 text-muted-foreground">
+ {selected.stems.length} of {selected.expectedStems} stems; the
+ others were deleted.
+ </p>
+ )}
+ </div>
+ ) : fallbackClip ? (
+ <div className="flex w-full max-w-xl flex-col gap-3">
+ <p className="text-ui-13 text-muted-foreground">
+ Stems saved, waiting for the history to list them.
+ </p>
+ <Button
+ variant="outline"
+ size="sm"
+ className="self-start"
+ onClick={handleDownloadFallbackClip}
+ >
+ <HugeiconsIcon icon={Download01Icon} className="size-3.5" />
+ Download the first stem
+ </Button>
+ </div>
+ ) : (
+ <div className="grid justify-items-center gap-3">
+ <p className="text-ui-13 text-muted-foreground">
+ {modelReady
+ ? "Stems land here. Add a track and press Generate."
+ : "Stems land here. Load a separation model, add a track, and press Generate."}
+ </p>
+ {picks.length > 0 ? (
+ <div
+ className="flex flex-wrap justify-center gap-2"
+ aria-label="Recommended models"
+ >
+ {picks.map((model) => (
+ <Button
+ key={model.id}
+ type="button"
+ variant="outline"
+ size="sm"
+ onClick={() => onPickModel?.(model.id)}
+ >
+ Use {model.name}
+ </Button>
+ ))}
+ </div>
+ ) : null}
+ </div>
+ )}
+ </div>
+ {groups.length > 0 ? (
+ <div className="flex shrink-0 flex-col gap-2">
+ <div className="flex items-center justify-between">
+ <span className="text-ui-11p5 font-medium text-muted-foreground">
+ History
+ </span>
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={() => void handleClearGallery()}
+ >
+ Clear all
+ </Button>
+ </div>
+ <div
+ className="hover-scrollbar flex max-h-40 flex-col gap-1 overflow-y-auto py-1"
+ onScroll={(event) => {
+ const el = event.currentTarget;
+ if (
+ hasMore &&
+ el.scrollTop + el.clientHeight >= el.scrollHeight - 40
+ ) {
+ void loadMore();
+ }
+ }}
+ >
+ {groups.map((group) => {
+ const current = group.groupId === selected?.groupId;
+ return (
+ // Shell, not a button: the pin badge and dots menu are buttons and cannot nest.
+ <div
+ key={group.groupId}
+ className={cn(
+ "group relative flex items-center gap-1 rounded-md pr-1 transition-colors hover:bg-muted",
+ current && "bg-muted",
+ )}
+ >
+ <button
+ type="button"
+ onClick={() => {
+ const first = group.stems[0];
+ if (first) selectClip(first.id);
+ }}
+ aria-current={current ? "true" : undefined}
+ className="flex min-w-0 flex-1 items-center gap-2 px-2 py-1.5 text-left text-ui-13"
+ >
+ <HugeiconsIcon
+ icon={AudioWave01Icon}
+ className="size-3.5 shrink-0 text-muted-foreground"
+ />
+ <span className="min-w-0 flex-1 truncate">
+ {group.title}
+ </span>
+ <span
+ title={group.model}
+ className="max-w-[30%] shrink-0 truncate text-ui-11p5 text-muted-foreground"
+ >
+ {audioModelLabel(group.model)}
+ </span>
+ <span className="shrink-0 rounded-4xl bg-muted px-2 py-0.5 text-ui-11 text-muted-foreground group-hover:bg-background">
+ {group.complete
+ ? `${group.stems.length} stems`
+ : `${group.stems.length} of ${group.expectedStems} stems`}
+ </span>
+ <span className="shrink-0 font-mono text-ui-11p5 tabular-nums text-muted-foreground">
+ {formatClipDuration(group.durationS)}
+ </span>
+ </button>
+ {group.pinned && (
+ <GalleryPinBadge
+ noun="clip"
+ className="static shrink-0"
+ onUnpin={() => pinGroup(group, false)}
+ />
+ )}
+ <GalleryItemMenu
+ variant="row"
+ noun="clip"
+ active={active}
+ pinned={group.pinned}
+ archived={false}
+ onTogglePin={() => pinGroup(group, !group.pinned)}
+ onToggleArchive={() => void archiveGroup(group)}
+ onDelete={() => void deleteGroup(group)}
+ onDownload={() => void downloadGroup(group)}
+ onAddToProject={async (projectId) => {
+ let already = true;
+ for (const clip of group.stems) {
+ const result = await addAudioClipToProject(
+ clip.id,
+ projectId,
+ );
+ already = already && result.already;
+ }
+ return { already };
+ }}
+ />
+ </div>
+ );
+ })}
+ </div>
+ </div>
+ ) : null}
+ </>
+ );
+}
diff --git a/studio/frontend/src/features/audio/picker-filter.ts b/studio/frontend/src/features/audio/picker-filter.ts
index ba3b196248e..bba3c464533 100644
--- a/studio/frontend/src/features/audio/picker-filter.ts
+++ b/studio/frontend/src/features/audio/picker-filter.ts
@@ -4,6 +4,7 @@
// Free of app imports so the node test runner can load it directly.
import {
+ AUDIO_CPP_SEP_AUDIO_TYPE,
audioCppModelFor,
isCloneOnlyFamilyId,
isSpeakAndCloneFamilyId,
@@ -29,6 +30,9 @@ export function audioRowMatchesWorkflow(
if (row.audioWorkflows && row.audioWorkflows.length > 0) {
return row.audioWorkflows.includes(workflow);
}
+ if (row.audioType === AUDIO_CPP_SEP_AUDIO_TYPE) {
+ return workflow === "separate";
+ }
// MiniMax Music 3 is tagged text-to-speech on the Hub.
if (isMusicGenerationModel(row.id, row.audioType)) return workflow === "music";
const byTask = audioWorkflowForTask(row.task);
@@ -53,6 +57,9 @@ export function audioRowMatchesWorkflow(
if (catalogTask === "asr") {
return workflow === "transcribe";
}
+ if (workflow === "separate") {
+ return false;
+ }
if (byTask === "speak" || catalogTask === "tts" || row.audioType) {
// Speech models clone only when the backend says so; an older row stays on Speak.
return audioWorkflowForAudioType(row.audioType) === workflow;
diff --git a/studio/frontend/src/features/audio/separate-policy.ts b/studio/frontend/src/features/audio/separate-policy.ts
new file mode 100644
index 00000000000..8b4a48c9c0e
--- /dev/null
+++ b/studio/frontend/src/features/audio/separate-policy.ts
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Free of app imports so the node test runner can load it directly.
+
+import type { AudioGenerationPresentation } from "./audio-page-policy";
+import type { AudioSourceSelection } from "./audio-run-request";
+import type { AudioToolPanel } from "./tools/types";
+
+/** Mirrors SEPARATE_MAX_SECONDS in studio/backend/routes/inference.py. */
+export const SEPARATE_MAX_SECONDS = 600;
+
+export const ROFORMER_FAMILIES: ReadonlySet<string> = new Set([
+ "bs_roformer",
+ "mel_band_roformer",
+]);
+
+export type SeparateBlockerKind =
+ | "source"
+ | "source-busy"
+ | "source-expired"
+ | "source-error"
+ | "too-long";
+
+export interface SeparateBlocker {
+ kind: SeparateBlockerKind;
+ reason: string;
+}
+
+function clock(seconds: number): string {
+ const whole = Math.max(0, Math.round(seconds));
+ return `${Math.floor(whole / 60)}:${String(whole % 60).padStart(2, "0")}`;
+}
+
+export function separateBlocker({
+ source,
+ sourceBusy,
+ sourceExpired,
+ sourceError,
+}: {
+ source: AudioSourceSelection | null;
+ sourceBusy: boolean;
+ sourceExpired: boolean;
+ sourceError: string | null;
+}): SeparateBlocker | null {
+ if (sourceBusy) {
+ return {
+ kind: "source-busy",
+ reason: "Waiting for the track to finish uploading.",
+ };
+ }
+ if (sourceError) return { kind: "source-error", reason: sourceError };
+ if (!source) return { kind: "source", reason: "Add a track to separate." };
+ if (sourceExpired) {
+ return {
+ kind: "source-expired",
+ reason: "This track expired. Add it again.",
+ };
+ }
+ if (
+ source.durationS !== null &&
+ source.durationS > SEPARATE_MAX_SECONDS + 0.5
+ ) {
+ return {
+ kind: "too-long",
+ reason: `Separate tracks up to 10 minutes long. This one is ${clock(source.durationS)}.`,
+ };
+ }
+ return null;
+}
+
+// GPU seconds per second of audio, measured on the pinned runtime (180 s track).
+const GPU_SECONDS_PER_SECOND: Record<string, number> = {
+ htdemucs: 0.032,
+ htdemucs_6stems: 0.036,
+ bs_roformer: 0.056,
+ mel_band_roformer: 0.056,
+};
+const ROFORMER_NO_OVERLAP_SECONDS_PER_SECOND = 0.016;
+
+export function estimateSeparateSeconds(
+ family: string | null | undefined,
+ durationS: number | null | undefined,
+ overlap: boolean,
+): number | null {
+ if (!family || !durationS || durationS <= 0) return null;
+ const rate =
+ ROFORMER_FAMILIES.has(family) && !overlap
+ ? ROFORMER_NO_OVERLAP_SECONDS_PER_SECOND
+ : GPU_SECONDS_PER_SECOND[family];
+ if (rate === undefined) return null;
+ // Saving the stems adds about a second per minute of audio.
+ return Math.max(1, Math.round(rate * durationS + durationS / 60));
+}
+
+/** RoFormers on the CPU take minutes for seconds of audio (142 s for an 8 s clip). */
+export function separateCpuWarning(
+ family: string | null | undefined,
+ device: string | null | undefined,
+): string | null {
+ if (device !== "cpu" || !family) return null;
+ return ROFORMER_FAMILIES.has(family)
+ ? "RoFormer models are very slow on the CPU: expect minutes for a short clip. Load into GPU when you can."
+ : "Separating on the CPU is slower; expect about a second per second of audio.";
+}
+
+export interface OverlapValue {
+ overlap: boolean;
+}
+
+export function overlapRequest(value: OverlapValue | undefined): {
+ options?: { num_overlap: number };
+} {
+ return value?.overlap === false ? { options: { num_overlap: 1 } } : {};
+}
+
+/** A model this page has not run yet loaded with the default (overlap on). */
+export function overlapReloads(
+ lastOverlap: boolean | undefined,
+ overlap: boolean,
+): boolean {
+ return (lastOverlap ?? true) !== overlap;
+}
+
+export function separatePresentation(
+ presentation: AudioGenerationPresentation | null,
+ phase: string | null,
+ reloading: boolean,
+): AudioGenerationPresentation | null {
+ if (!presentation) return null;
+ switch (phase) {
+ case "preparing":
+ return { ...presentation, status: "Preparing the track…" };
+ case "generating":
+ return {
+ ...presentation,
+ status: reloading
+ ? "Reloading the model with the new overlap, then separating…"
+ : "Separating…",
+ };
+ case "finishing":
+ return { ...presentation, status: "Saving stems…" };
+ case "stopping":
+ return { ...presentation, status: "Stopping…" };
+ default:
+ return presentation;
+ }
+}
+
+/** Overlap is a load-time setting, so changing it reloads the model. */
+export const roformerOverlapLogic: Omit<
+ AudioToolPanel<OverlapValue>,
+ "Component"
+> = {
+ id: "roformer-overlap",
+ families: [...ROFORMER_FAMILIES],
+ workflows: ["separate"],
+ title: "Overlap",
+ claims: [],
+ initial: () => ({ overlap: true }),
+ toRequest: (value) => overlapRequest(value),
+};
diff --git a/studio/frontend/src/features/audio/separation-stems.ts b/studio/frontend/src/features/audio/separation-stems.ts
new file mode 100644
index 00000000000..744f0892038
--- /dev/null
+++ b/studio/frontend/src/features/audio/separation-stems.ts
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Free of app imports so the node test runner can load it directly.
+
+import type { AudioGalleryClip } from "./api";
+
+/** Display order. Mirrors _STEM_ORDER in studio/backend/routes/inference.py. */
+export const STEM_ORDER: readonly string[] = [
+ "vocals",
+ "drums",
+ "bass",
+ "guitar",
+ "piano",
+ "other",
+ "instrumental",
+];
+
+const STEM_LABELS: Record<string, string> = {
+ vocals: "Vocals",
+ drums: "Drums",
+ bass: "Bass",
+ guitar: "Guitar",
+ piano: "Piano",
+ other: "Other",
+ instrumental: "Instrumental",
+};
+
+export function stemLabel(id: string): string {
+ const known = STEM_LABELS[id];
+ if (known) return known;
+ const words = id.replace(/[_-]+/g, " ").trim();
+ return words ? words[0].toUpperCase() + words.slice(1) : "Stem";
+}
+
+export const SEPARATION_STEMS_BY_FAMILY: Readonly<
+ Record<string, { stems: readonly string[] }>
+> = {
+ htdemucs: { stems: ["vocals", "drums", "bass", "other"] },
+ htdemucs_6stems: {
+ stems: ["vocals", "drums", "bass", "guitar", "piano", "other"],
+ },
+ bs_roformer: { stems: ["vocals", "instrumental"] },
+ mel_band_roformer: { stems: ["vocals", "instrumental"] },
+};
+
+export function orderStems<T extends string>(ids: readonly T[]): T[] {
+ const rank = (id: string) => {
+ const index = STEM_ORDER.indexOf(id);
+ return index === -1 ? STEM_ORDER.length : index;
+ };
+ return ids
+ .map((id, index) => ({ id, index }))
+ .sort((a, b) => rank(a.id) - rank(b.id) || a.index - b.index)
+ .map(({ id }) => id);
+}
+
+export interface SeparationGroup {
+ groupId: string;
+ title: string;
+ model: string;
+ durationS: number;
+ createdAt: string;
+ stems: AudioGalleryClip[];
+ /** False when some of the run's stems were deleted. */
+ complete: boolean;
+ expectedStems: number;
+ pinned: boolean;
+}
+
+function settingsStems(clip: AudioGalleryClip): string[] | null {
+ const stems = clip.settings?.stems;
+ return Array.isArray(stems) && stems.every((s) => typeof s === "string")
+ ? (stems as string[])
+ : null;
+}
+
+/** A clip with no group is its own item. */
+export function groupSeparationClips(
+ clips: readonly AudioGalleryClip[],
+ hasMore = false,
+): SeparationGroup[] {
+ const byId = new Map<string, AudioGalleryClip[]>();
+ const order: string[] = [];
+ for (const clip of clips) {
+ const key = clip.group_id || `clip:${clip.id}`;
+ let list = byId.get(key);
+ if (!list) {
+ list = [];
+ byId.set(key, list);
+ order.push(key);
+ }
+ list.push(clip);
+ }
+ const groups = order.map((key) => {
+ const list = byId.get(key) ?? [];
+ const roleOf = (clip: AudioGalleryClip) => clip.role || clip.id;
+ const ordered = orderStems(list.map(roleOf)).map(
+ (role) => list.find((clip) => roleOf(clip) === role) as AudioGalleryClip,
+ );
+ const first = ordered[0];
+ const expected = Math.max(
+ ordered.length,
+ ...ordered.map((clip) => settingsStems(clip)?.length ?? 0),
+ );
+ return {
+ groupId: key,
+ title: first?.prompt || "Separated track",
+ model: first?.model ?? "",
+ durationS: Math.max(0, ...ordered.map((clip) => clip.duration_s || 0)),
+ createdAt: first?.created_at ?? "",
+ stems: ordered,
+ complete: ordered.length >= expected,
+ expectedStems: expected,
+ pinned: ordered.some((clip) => clip.pinned),
+ };
+ });
+ // The oldest group may continue on the next page: hide it until it has loaded, so its row
+ // never acts on part of the run.
+ const tail = groups[groups.length - 1];
+ return hasMore && tail && !tail.complete ? groups.slice(0, -1) : groups;
+}
diff --git a/studio/frontend/src/features/audio/stores/audio-separate-store.ts b/studio/frontend/src/features/audio/stores/audio-separate-store.ts
new file mode 100644
index 00000000000..acc8e9cfdc2
--- /dev/null
+++ b/studio/frontend/src/features/audio/stores/audio-separate-store.ts
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { create } from "zustand";
+import { persist } from "zustand/middleware";
+import type { AudioSourceSelection } from "../audio-run-request";
+
+export const AUDIO_SEPARATE_STORAGE_KEY = "unsloth_audio_separate_v1";
+
+interface AudioSeparateState {
+ source: AudioSourceSelection | null;
+ /** So a change is announced as a reload. */
+ lastOverlapByModel: Record<string, boolean>;
+ setSource: (source: AudioSourceSelection | null) => void;
+ setLastOverlap: (model: string, overlap: boolean) => void;
+}
+
+const MAX_MODELS = 20;
+
+export const useAudioSeparateStore = create<AudioSeparateState>()(
+ persist(
+ (set) => ({
+ source: null,
+ lastOverlapByModel: {},
+ setSource: (source) => set({ source }),
+ setLastOverlap: (model, overlap) =>
+ set((state) => {
+ const { [model]: _previous, ...rest } = state.lastOverlapByModel;
+ const entries = Object.entries(rest);
+ const kept = entries.slice(
+ Math.max(0, entries.length - MAX_MODELS + 1),
+ );
+ return {
+ lastOverlapByModel: {
+ ...Object.fromEntries(kept),
+ [model]: overlap,
+ },
+ };
+ }),
+ }),
+ {
+ name: AUDIO_SEPARATE_STORAGE_KEY,
+ version: 1,
+ partialize: (state) => ({
+ source: state.source,
+ lastOverlapByModel: state.lastOverlapByModel,
+ }),
+ },
+ ),
+);
diff --git a/studio/frontend/src/features/audio/tools/registry.tsx b/studio/frontend/src/features/audio/tools/registry.tsx
index 7f54f011e2d..8cef957063a 100644
--- a/studio/frontend/src/features/audio/tools/registry.tsx
+++ b/studio/frontend/src/features/audio/tools/registry.tsx
@@ -9,6 +9,7 @@ import {
import type { AudioWorkflowId } from "../workflows";
import { CLONE_TOOL_PANELS } from "./clone-panels";
import { MUSIC_TOOL_PANELS } from "./music-panels";
+import { SEPARATE_TOOL_PANELS } from "./separate-panels";
import {
instructionsKindFor,
legacyMusicDescription,
@@ -115,6 +116,7 @@ export const AUDIO_TOOL_PANELS: readonly AnyAudioToolPanel[] = [
...SPEAK_TOOL_PANELS.slice(1),
...CLONE_TOOL_PANELS,
...MUSIC_TOOL_PANELS,
+ ...SEPARATE_TOOL_PANELS,
];
export function audioToolPanelsFor(
diff --git a/studio/frontend/src/features/audio/tools/separate-panels.tsx b/studio/frontend/src/features/audio/tools/separate-panels.tsx
new file mode 100644
index 00000000000..ffa93bd813e
--- /dev/null
+++ b/studio/frontend/src/features/audio/tools/separate-panels.tsx
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Switch } from "@/components/ui/switch";
+import { type OverlapValue, roformerOverlapLogic } from "../separate-policy";
+import type { AudioToolPanel } from "./types";
+
+export const roformerOverlapPanel: AudioToolPanel<OverlapValue> = {
+ ...roformerOverlapLogic,
+ Component: ({ value, onChange, disabled }) => (
+ <div className="grid gap-1.5">
+ <label
+ htmlFor="separate-overlap"
+ className="flex items-center justify-between gap-3 text-ui-13 font-medium text-foreground"
+ >
+ Overlap (slower, cleaner)
+ <Switch
+ id="separate-overlap"
+ checked={value.overlap}
+ disabled={disabled}
+ onCheckedChange={(overlap) => onChange({ overlap })}
+ />
+ </label>
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ Runs the model over overlapping windows and blends them. Off is about
+ three times faster. Changing it reloads the model, which takes a few
+ seconds.
+ </p>
+ </div>
+ ),
+};
+
+export const SEPARATE_TOOL_PANELS = [roformerOverlapPanel] as const;
diff --git a/studio/frontend/src/features/audio/tour/steps.tsx b/studio/frontend/src/features/audio/tour/steps.tsx
index 73294c1db45..af12d21bfaf 100644
--- a/studio/frontend/src/features/audio/tour/steps.tsx
+++ b/studio/frontend/src/features/audio/tour/steps.tsx
@@ -12,8 +12,9 @@ const modeStep: TourStep = {
<>
Open the page title to switch pages. Speak turns text into speech, Clone
speaks in the voice from a short recording, Music makes songs and sound
- effects, and Transcribe turns a recording into text. Each lists only the
- models that can do it, so the picker above follows the page.
+ effects, Separate splits a track into vocals and instruments, and
+ Transcribe turns a recording into text. Each lists only the models that
+ can do it, so the picker above follows the page.
</>
),
};
@@ -24,6 +25,8 @@ const MODEL_STEP_BODY: Record<AudioWorkflowId, string> = {
clone:
"Models that can speak in the voice of a short recording you give them.",
music: "Music models. Loading one replaces the model in the main slot.",
+ separate:
+ "Source separation models. Loading one replaces the model in the main slot.",
transcribe:
"Speech recognition models. They run beside your chat model, not in its place.",
};
@@ -73,6 +76,35 @@ export function buildAudioTourSteps({
];
}
+ if (workflow === "separate") {
+ return [
+ modeStep,
+ modelStep(workflow),
+ {
+ id: "source",
+ target: "audio-separate-source",
+ title: "Track",
+ body: (
+ <>
+ Upload, record or pick the track to split. It is converted to 44.1
+ kHz for the model.
+ </>
+ ),
+ },
+ {
+ id: "output",
+ target: "audio-output",
+ title: "Stems",
+ body: (
+ <>
+ Each stem gets its own row with solo, mute and volume, all playing
+ in sync. Download one stem or all of them.
+ </>
+ ),
+ },
+ ];
+ }
+
if (workflow === "clone") {
return [
modeStep,
diff --git a/studio/frontend/src/features/audio/workflows.ts b/studio/frontend/src/features/audio/workflows.ts
index c8607f4b59a..3c5fa9549ff 100644
--- a/studio/frontend/src/features/audio/workflows.ts
+++ b/studio/frontend/src/features/audio/workflows.ts
@@ -7,12 +7,18 @@ import {
AiVoiceIcon,
MusicNote03Icon,
SpeechToTextIcon,
+ SplitIcon,
VoiceIdIcon,
} from "@hugeicons/core-free-icons";
import type { IconSvgElement } from "@hugeicons/react";
import { AUDIO_CPP_MUSIC_AUDIO_TYPE } from "./audio-cpp-catalog";
-export type AudioWorkflowId = "speak" | "clone" | "music" | "transcribe";
+export type AudioWorkflowId =
+ | "speak"
+ | "clone"
+ | "music"
+ | "separate"
+ | "transcribe";
export type AudioWorkflowSlot = "speak" | "transcribe";
@@ -52,6 +58,15 @@ export const AUDIO_WORKFLOWS: ReadonlyArray<{
slot: "speak",
createTrain: false,
},
+ {
+ id: "separate",
+ label: "Separate",
+ heading: "Separate audio",
+ icon: SplitIcon,
+ hint: "Split a track into vocals and instruments",
+ slot: "speak",
+ createTrain: false,
+ },
{
id: "transcribe",
label: "Transcribe",
@@ -105,11 +120,12 @@ export function audioWorkflowForAudioType(
export function clipWorkflow(clip: {
workflow?: string | null;
audio_type?: string | null;
-}): "speak" | "clone" | "music" {
+}): "speak" | "clone" | "music" | "separate" {
if (
clip.workflow === "speak" ||
clip.workflow === "clone" ||
- clip.workflow === "music"
+ clip.workflow === "music" ||
+ clip.workflow === "separate"
) {
return clip.workflow;
}
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
index 096dd9b80d2..61686a60b35 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
@@ -177,6 +177,7 @@ const AUDIO_GGUF_DESCRIPTIONS: Record<AudioCppTask, string> = {
tts: "Text-to-speech",
music: "Text-to-music",
asr: "Speech-to-text",
+ sep: "Source separation",
};
// Recommended audio GGUFs the backend runs on its audio runtime. They are plain GGUF rows, so the
@@ -187,7 +188,7 @@ const audioGgufGroups = (tasks: readonly AudioCppTask[]): CatalogGroup[] =>
canonicalId: model.id,
displayName: audioCppDisplayName(model.id),
description:
- model.workflows && !model.workflows.includes("speak")
+ model.workflows?.includes("clone") && !model.workflows.includes("speak")
? "Voice cloning"
: AUDIO_GGUF_DESCRIPTIONS[model.task],
scope: "audio",
@@ -661,7 +662,7 @@ export const AUDIO_CATALOG: CatalogGroup[] = [
}),
],
},
- ...audioGgufGroups(["tts", "music"]),
+ ...audioGgufGroups(["tts", "music", "sep"]),
// Llasa is deliberately absent: it speaks XCodec2 (65,536 <|s_N|> tokens), which is in neither
// _AUDIO_TOKEN_PATTERNS nor AudioCodecManager, so a curated row loaded then failed at generation.
// Training still works (unsloth_Llasa-3B.yaml). Re-add both rows with an xcodec2 decoder.
diff --git a/studio/frontend/tests/audio-handoff-intent.test.ts b/studio/frontend/tests/audio-handoff-intent.test.ts
index a88f7574f93..a11a01cf4b9 100644
--- a/studio/frontend/tests/audio-handoff-intent.test.ts
+++ b/studio/frontend/tests/audio-handoff-intent.test.ts
@@ -24,7 +24,7 @@ test("unknown tasks, workflows and audio types are dropped", () => {
assert.deepEqual(
validateAudioSearch({
task: "text-to-image",
- workflow: "separate",
+ workflow: "convert",
audioType: "not-a-type",
loadId: " ",
}),
@@ -34,6 +34,9 @@ test("unknown tasks, workflows and audio types are dropped", () => {
validateAudioSearch({ workflow: 3, task: ["text-to-speech"] }),
{},
);
+ assert.deepEqual(validateAudioSearch({ workflow: "separate" }), {
+ workflow: "separate",
+ });
});
test("the params the route accepted before pass through unchanged", () => {
diff --git a/studio/frontend/tests/audio-separate.test.ts b/studio/frontend/tests/audio-separate.test.ts
new file mode 100644
index 00000000000..83ce643e24a
--- /dev/null
+++ b/studio/frontend/tests/audio-separate.test.ts
@@ -0,0 +1,323 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const { groupSeparationClips, orderStems, stemLabel } = await import(
+ "../src/features/audio/separation-stems.ts"
+);
+const {
+ SEPARATE_MAX_SECONDS,
+ estimateSeparateSeconds,
+ overlapReloads,
+ overlapRequest,
+ roformerOverlapLogic,
+ separateBlocker,
+ separateCpuWarning,
+ separatePresentation,
+} = await import("../src/features/audio/separate-policy.ts");
+const { buildAudioRunBody } = await import(
+ "../src/features/audio/audio-run-request.ts"
+);
+const { audioRowMatchesWorkflow } = await import(
+ "../src/features/audio/picker-filter.ts"
+);
+const {
+ AUDIO_CPP_MODELS,
+ AUDIO_CPP_SEP_AUDIO_TYPE,
+ audioCppModelSpeaks,
+ audioCppWorkflowsFor,
+} = await import("../src/features/audio/audio-cpp-catalog.ts");
+const { isTtsAudioType } = await import(
+ "../src/features/audio/audio-page-policy.ts"
+);
+const { loadedAudioKind } = await import(
+ "../src/features/audio/audio-workspace-utils.ts"
+);
+const { audioCapabilityLine } = await import(
+ "../src/features/audio/catalog.ts"
+);
+
+const clip = (
+ id: string,
+ role: string,
+ extra: Record<string, unknown> = {},
+) => ({
+ id,
+ role,
+ url: `/api/inference/audio/gallery/${id}/file`,
+ prompt: "song.wav",
+ model: "audio-cpp/audio.cpp-gguf/HTDemucs-GGUF",
+ audio_type: "audiocpp_sep",
+ sample_rate: 44100,
+ duration_s: 180,
+ created_at: "2026-10-03T00:00:00Z",
+ workflow: "separate",
+ ...extra,
+});
+const ids = (groups: { groupId: string }[]) => groups.map((g) => g.groupId);
+
+test("stems show in a fixed order with readable names", () => {
+ assert.deepEqual(orderStems(["other", "bass", "drums", "vocals"]), [
+ "vocals",
+ "drums",
+ "bass",
+ "other",
+ ]);
+ assert.deepEqual(orderStems(["instrumental", "zzz", "vocals", "aaa"]), [
+ "vocals",
+ "instrumental",
+ "zzz",
+ "aaa",
+ ]);
+ assert.equal(stemLabel("lead_guitar"), "Lead guitar");
+ assert.equal(stemLabel(""), "Stem");
+});
+
+test("history shows one item per separation, stems in display order", () => {
+ const groups = groupSeparationClips([
+ clip("a2", "drums", {
+ group_id: "g1",
+ settings: { stems: ["drums", "vocals"] },
+ }),
+ clip("b1", "instrumental", { group_id: "g2", pinned: true }),
+ clip("a1", "vocals", { group_id: "g1" }),
+ clip("b2", "vocals", { group_id: "g2" }),
+ clip("solo", "vocals"),
+ ]);
+ assert.deepEqual(
+ groups.map((group) => [group.groupId, group.stems.map((stem) => stem.id)]),
+ [
+ ["g1", ["a1", "a2"]],
+ ["g2", ["b2", "b1"]],
+ ["clip:solo", ["solo"]],
+ ],
+ );
+ assert.equal(groups[0].complete, true);
+ assert.equal(groups[1].pinned, true);
+ assert.equal(groups[0].title, "song.wav");
+ const [partial] = groupSeparationClips([
+ clip("a", "vocals", {
+ group_id: "g",
+ settings: { stems: ["vocals", "drums", "bass", "other"] },
+ }),
+ ]);
+ assert.equal(partial.complete, false);
+ assert.equal(partial.expectedStems, 4);
+});
+
+test("a group cut by the page boundary waits for the next page, which the page loads itself", () => {
+ const clips = [
+ clip("a", "vocals", { group_id: "g1", settings: { stems: ["vocals"] } }),
+ clip("b", "vocals", {
+ group_id: "g2",
+ settings: { stems: ["vocals", "drums"] },
+ }),
+ ];
+ assert.deepEqual(ids(groupSeparationClips(clips, true)), ["g1"]);
+ assert.deepEqual(ids(groupSeparationClips(clips, false)), ["g1", "g2"]);
+ // A short list never scrolls, so a hidden oldest run must trigger loadMore itself.
+ const page = readSrc("features/audio/pages/separate-page.tsx");
+ assert.match(
+ page,
+ /hasMore && groupSeparationClips\(clips\)\.length > groups\.length/,
+ );
+ assert.match(
+ page,
+ /if \(tailHidden\) void loadMore\(\);\s*\}, \[tailHidden, clips, loadMore\]\)/,
+ );
+});
+
+test("Generate says what the track needs, in order", () => {
+ const source = {
+ kind: "input" as const,
+ id: "i",
+ name: "a.wav",
+ durationS: 30,
+ };
+ const base = {
+ source,
+ sourceBusy: false,
+ sourceExpired: false,
+ sourceError: null,
+ };
+ const long = { ...source, durationS: 601 };
+ // Each row also carries every lower-priority problem, so the order is pinned too.
+ const cases: [
+ Partial<Parameters<typeof separateBlocker>[0]>,
+ string | null,
+ RegExp | string | null,
+ ][] = [
+ [{}, null, null],
+ [{ source: { ...source, durationS: SEPARATE_MAX_SECONDS } }, null, null],
+ [{ sourceBusy: true, sourceError: "x", source: null }, "source-busy", null],
+ [{ sourceError: "Not audio.", source: null }, "source-error", "Not audio."],
+ [
+ { source: null, sourceExpired: true },
+ "source",
+ "Add a track to separate.",
+ ],
+ [
+ { sourceExpired: true, source: long },
+ "source-expired",
+ "This track expired. Add it again.",
+ ],
+ [{ source: long }, "too-long", /up to 10 minutes.*10:01/],
+ ];
+ for (const [patch, kind, reason] of cases) {
+ const blocker = separateBlocker({ ...base, ...patch });
+ assert.equal(blocker?.kind ?? null, kind, JSON.stringify(patch));
+ if (typeof reason === "string") {
+ assert.equal(blocker?.reason, reason);
+ }
+ if (reason instanceof RegExp) {
+ assert.match(blocker?.reason ?? "", reason);
+ }
+ }
+});
+
+test("the estimate follows the model, the length and the overlap", () => {
+ const cases: [string, number | null, boolean, number | null][] = [
+ ["htdemucs", 180, true, 9],
+ ["bs_roformer", 180, true, 13],
+ ["bs_roformer", 180, false, 6],
+ ["htdemucs", 1, true, 1],
+ ["unknown", 180, true, null],
+ ["htdemucs", null, true, null],
+ ];
+ for (const [family, seconds, overlap, want] of cases) {
+ assert.equal(
+ estimateSeparateSeconds(family, seconds, overlap),
+ want,
+ `${family} ${seconds} ${overlap}`,
+ );
+ }
+ assert.match(separateCpuWarning("bs_roformer", "cpu") ?? "", /very slow/);
+ assert.equal(separateCpuWarning("bs_roformer", "auto"), null);
+});
+
+test("Overlap off asks for one pass; on sends nothing; a change is a reload", () => {
+ assert.deepEqual(overlapRequest({ overlap: false }), {
+ options: { num_overlap: 1 },
+ });
+ assert.deepEqual(overlapRequest({ overlap: true }), {});
+ assert.deepEqual(overlapRequest(undefined), {});
+ const reloads: [boolean | undefined, boolean, boolean][] = [
+ [undefined, true, false],
+ [undefined, false, true],
+ [false, false, false],
+ [false, true, true],
+ ];
+ for (const [loaded, wanted, want] of reloads) {
+ assert.equal(overlapReloads(loaded, wanted), want);
+ }
+ assert.deepEqual(roformerOverlapLogic.initial([]), { overlap: true });
+});
+
+test("progress copy names the separation phases", () => {
+ const p = { status: "Generating audio…", actionLabel: "Stop", canStop: true };
+ assert.equal(
+ separatePresentation(p, "generating", false)?.status,
+ "Separating…",
+ );
+ assert.match(
+ separatePresentation(p, "generating", true)?.status ?? "",
+ /Reloading/,
+ );
+ assert.equal(
+ separatePresentation(p, "finishing", false)?.status,
+ "Saving stems…",
+ );
+ assert.equal(separatePresentation(null, "generating", false), null);
+});
+
+test("a separate run sends the source and options only", () => {
+ const body = {
+ workflow: "separate" as const,
+ inputs: { source: { input_id: "abc" } },
+ options: { num_overlap: 1 },
+ };
+ assert.deepEqual(buildAudioRunBody({ ...body, seed: undefined }), body);
+ assert.equal(buildAudioRunBody({ workflow: "speak", text: "hi" }).text, "hi");
+});
+
+test("separation rows list only on Separate; sep catalog models only separate", () => {
+ const pages = ["speak", "clone", "music", "separate", "transcribe"] as const;
+ const on = (row: Parameters<typeof audioRowMatchesWorkflow>[0]) =>
+ pages.filter((page) => audioRowMatchesWorkflow(row, page));
+ assert.deepEqual(on({ audioWorkflows: ["separate"] }), ["separate"]);
+ assert.deepEqual(on({ audioType: AUDIO_CPP_SEP_AUDIO_TYPE }), ["separate"]);
+ assert.deepEqual(on({ id: "audio-cpp/audio.cpp-gguf/HTDemucs-GGUF" }), [
+ "separate",
+ ]);
+ assert.equal(
+ on({ task: "text-to-speech", audioType: "audiocpp_tts" }).includes(
+ "separate",
+ ),
+ false,
+ );
+ assert.equal(on({}).includes("separate"), false);
+ const seps = AUDIO_CPP_MODELS.filter((model) => model.task === "sep");
+ assert.equal(seps.length, 4);
+ for (const model of seps) {
+ assert.deepEqual(audioCppWorkflowsFor(model), ["separate"]);
+ assert.equal(audioCppModelSpeaks(model.id), false);
+ }
+});
+
+test("a loaded separation model is a main-slot audio model", () => {
+ assert.equal(isTtsAudioType("audiocpp_sep"), true);
+ assert.equal(isTtsAudioType("audiocpp_sep", true), true);
+ assert.equal(loadedAudioKind("audiocpp_sep"), "separation");
+ assert.equal(
+ audioCapabilityLine("separate", "audiocpp_sep"),
+ "Source separation · GGUF",
+ );
+});
+
+test("host and page wiring for Separate", () => {
+ const host = readSrc("features/audio/audio-page.tsx");
+ // Leaving Audio stops a Separate recording and releases the mic, as on Clone.
+ assert.match(
+ host,
+ /ttsWorkflow === "separate" \? \(\s*<AudioActiveProvider value=\{active\}>\s*<SeparateRail/,
+ );
+ assert.match(host, /ttsWorkflow === "separate"\s*\? separate\.blocker/);
+ assert.match(host, /reason: "The loaded model separates audio\."/);
+ // Trained speech checkpoints are not separation models.
+ assert.match(
+ host,
+ /ttsWorkflow === "clone" \|\| ttsWorkflow === "separate"\s*\? \[\]/,
+ );
+ // A stem sent to Transcribe mid-run would be dropped when the run stops.
+ assert.match(
+ host,
+ /target\.workflow === "transcribe"\) \{[\s\S]*?busyRef\.current !== null[\s\S]*?return;[\s\S]*?if \(!transitionWorkflow\("transcribe"\)\) return;[\s\S]*?useAudioTranscribeStore\.setState\(\{\s*source: \{\s*kind: "clip",\s*id: clip\.id/,
+ );
+ // A separation that a refresh missed falls back on the Separate page.
+ const generation = readSrc("features/audio/hooks/use-separate-generation.ts");
+ assert.match(generation, /showRunResult\(\{[^}]*workflow: "separate"/);
+ const page = readSrc("features/audio/pages/separate-page.tsx");
+ assert.match(page, /allowSavedVoice=\{false\}/);
+ assert.match(page, /expiredMessage=\{SEPARATE_TRACK_EXPIRED_MESSAGE\}/);
+ assert.match(page, /maxRecordSeconds=\{SEPARATE_MAX_SECONDS - 1\}/);
+ // The mixer's own sources pin the group; a failed fetch retries instead of loading forever,
+ // and a stem that cannot load is marked so the mixer plays the others.
+ assert.match(page, /useStemSources\(inputs, attempt\)/);
+ assert.match(page, /sources\.failedIds\.length > 0/);
+ assert.match(page, /setAttempt\(\(n\) => n \+ 1\)/);
+ assert.match(page, /failed: sources\.failedIds\.includes\(clip\.id\)/);
+});
+
+test("a stem sent to Clone is adopted, so the old reference's transcript goes with it", () => {
+ const host = readSrc("features/audio/audio-page.tsx");
+ const send = host.slice(host.indexOf("const handleSendStem = useCallback("));
+ assert.match(
+ send,
+ /if \(!transitionWorkflow\("clone"\)\) return;[\s\S]{0,160}?useAudioCloneStore\.getState\(\)\.adoptReference\(\{/,
+ );
+});
diff --git a/studio/frontend/tests/audio-stem-mixer-source.test.ts b/studio/frontend/tests/audio-stem-mixer-source.test.ts
new file mode 100644
index 00000000000..2124387f2a1
--- /dev/null
+++ b/studio/frontend/tests/audio-stem-mixer-source.test.ts
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const mixer = readSrc("features/audio/components/stem-mixer.tsx");
+const sources = readSrc("features/audio/hooks/use-stem-sources.ts");
+
+test("stem audio uses its own pinned cache, not the gallery's LRU", () => {
+ assert.match(sources, /new BlobUrlCache\(/);
+ assert.doesNotMatch(sources, /galleryCache/);
+ assert.match(sources, /cache\.clear\(\)/);
+ // The CSP's connect-src refuses fetch() on blob: URLs, so peaks decode from the Blob itself.
+ assert.match(sources, /blob\.arrayBuffer\(\)/);
+ assert.doesNotMatch(sources, /fetch\(objectUrl\)/);
+});
+
+test("a stem that failed to load is left out of playback, so the rest still play", () => {
+ assert.match(
+ mixer,
+ /const playable = useMemo\(\s*\(\) => stems\.filter\(\(stem\) => !stem\.failed\)/,
+ );
+ assert.match(
+ mixer,
+ /playable\.map\(\(stem\) => \(\{ id: stem\.clipId, src: stem\.src \}\)\)/,
+ );
+ assert.match(
+ mixer,
+ /const loadingCount = playable\.filter\(\(stem\) => !stem\.src\)\.length/,
+ );
+ // The zip needs every stem, so Download all waits for all of them.
+ assert.match(mixer, /disabled=\{stems\.some\(\(stem\) => !stem\.src\)\}/);
+});
diff --git a/studio/frontend/tests/audio-stem-mixer-state.test.ts b/studio/frontend/tests/audio-stem-mixer-state.test.ts
new file mode 100644
index 00000000000..164edb570a1
--- /dev/null
+++ b/studio/frontend/tests/audio-stem-mixer-state.test.ts
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+const {
+ INITIAL_STEM_MIXER_STATE,
+ anySolo,
+ effectiveGain,
+ stemMix,
+ stemMixerReducer,
+} = await import("../src/features/audio/components/stem-mixer-state.ts");
+const {
+ sanitizeFileNamePart,
+ stemFileName,
+ stemZipName,
+ uniqueStemNames,
+ zipStems,
+} = await import("../src/features/audio/components/stem-zip.ts");
+const { unzipSync, strFromU8 } = await import("fflate");
+
+type State = typeof INITIAL_STEM_MIXER_STATE;
+type Action = Parameters<typeof stemMixerReducer>[1];
+const run = (...actions: Action[]): State =>
+ actions.reduce(stemMixerReducer, INITIAL_STEM_MIXER_STATE);
+const ROLES = ["vocals", "drums", "bass", "other"];
+const gains = (state: State) => ROLES.map((role) => effectiveGain(state, role));
+
+test("gain follows solo, mute and volume; solo beats mute", () => {
+ const solo = (role: string): Action => ({ type: "toggleSolo", role });
+ const mute = (role: string): Action => ({ type: "toggleMute", role });
+ const vol = (role: string, volume: number): Action => ({
+ type: "setVolume",
+ role,
+ volume,
+ });
+ const cases: [string, Action[], number[]][] = [
+ ["initial", [], [1, 1, 1, 1]],
+ ["one solo silences the rest", [solo("vocals")], [1, 0, 0, 0]],
+ [
+ "solo toggled off restores",
+ [solo("vocals"), solo("vocals")],
+ [1, 1, 1, 1],
+ ],
+ [
+ "several solos play together",
+ [solo("vocals"), solo("bass")],
+ [1, 0, 1, 0],
+ ],
+ ["mute silences only that stem", [mute("drums")], [1, 0, 1, 1]],
+ ["mute toggled off restores", [mute("drums"), mute("drums")], [1, 1, 1, 1]],
+ ["solo beats mute", [mute("vocals"), solo("vocals")], [1, 0, 0, 0]],
+ [
+ "mute returns when the solo ends",
+ [mute("vocals"), solo("vocals"), solo("vocals")],
+ [0, 1, 1, 1],
+ ],
+ ["volume scales the gain", [vol("bass", 0.5)], [1, 1, 0.5, 1]],
+ ["volume clamps high", [vol("bass", 3)], [1, 1, 1, 1]],
+ ["volume clamps low", [vol("bass", -2)], [1, 1, 0, 1]],
+ [
+ "NaN volume is ignored",
+ [vol("bass", 0.5), vol("bass", Number.NaN)],
+ [1, 1, 0.5, 1],
+ ],
+ ["solo keeps the volume", [vol("bass", 0.5), solo("bass")], [0, 0, 0.5, 0]],
+ ];
+ for (const [name, actions, want] of cases) {
+ assert.deepEqual(gains(run(...actions)), want, name);
+ }
+ assert.deepEqual(stemMix(INITIAL_STEM_MIXER_STATE, "vocals"), {
+ volume: 1,
+ muted: false,
+ solo: false,
+ });
+ assert.equal(anySolo(run(solo("vocals"))), true);
+ assert.equal(anySolo(run(solo("vocals"), solo("vocals"))), false);
+});
+
+test("reset returns to the initial levels; no-op actions keep the same object", () => {
+ const busy = run(
+ { type: "toggleSolo", role: "vocals" },
+ { type: "toggleMute", role: "drums" },
+ { type: "setVolume", role: "bass", volume: 0.2 },
+ );
+ const reset = stemMixerReducer(busy, { type: "reset" });
+ assert.deepEqual(reset, INITIAL_STEM_MIXER_STATE);
+ assert.equal(stemMixerReducer(reset, { type: "reset" }), reset);
+ assert.equal(
+ stemMixerReducer(busy, { type: "setVolume", role: "bass", volume: 0.2 }),
+ busy,
+ );
+});
+
+test("stem file names read '<title> - <Label>.wav' and are safe on disk", () => {
+ const cases: [string, string, string][] = [
+ ["My Song.mp3", "Vocals", "My Song - Vocals.wav"],
+ ['a/b:c*?"<>|', "Drums", "a_b_c______ - Drums.wav"],
+ [" ", "Bass", "Separated track - Bass.wav"],
+ ["trailing...", "Other", "trailing - Other.wav"],
+ ["tab\there", "Piano", "tab_here - Piano.wav"],
+ ];
+ for (const [title, label, want] of cases) {
+ assert.equal(stemFileName(title, label), want);
+ }
+ const long = stemFileName("x".repeat(500), "Instrumental");
+ assert.ok(long.endsWith(" - Instrumental.wav"));
+ assert.ok(long.length <= 120 + " - Instrumental.wav".length);
+ assert.equal(stemZipName("My Song.wav"), "My Song - stems.zip");
+ assert.equal(sanitizeFileNamePart("", "fallback"), "fallback");
+});
+
+test("repeated names get a counter instead of overwriting", () => {
+ assert.deepEqual(
+ uniqueStemNames(["a - X.wav", "a - X.wav", "A - x.wav", "b.wav"]),
+ ["a - X.wav", "a - X (2).wav", "A - x (3).wav", "b.wav"],
+ );
+});
+
+test("zipStems stores every file under its name", async () => {
+ const zip = await zipStems([
+ { name: "t - Vocals.wav", blob: new Blob(["vocals-bytes"]) },
+ { name: "t - Vocals.wav", blob: new Blob(["second"]) },
+ { name: "t - Drums.wav", blob: new Blob([new Uint8Array(0)]) },
+ ]);
+ assert.equal(zip.type, "application/zip");
+ const files = unzipSync(new Uint8Array(await zip.arrayBuffer()));
+ assert.deepEqual(Object.keys(files), [
+ "t - Vocals.wav",
+ "t - Vocals (2).wav",
+ "t - Drums.wav",
+ ]);
+ assert.equal(strFromU8(files["t - Vocals.wav"]), "vocals-bytes");
+ assert.equal(strFromU8(files["t - Vocals (2).wav"]), "second");
+ assert.equal(files["t - Drums.wav"].length, 0);
+});
+
+test("zipStems loads each stem only when the archive reaches it", async () => {
+ const order: string[] = [];
+ const lazy = (name: string, body: string) => async () => {
+ order.push(name);
+ return new Blob([body]);
+ };
+ const zip = await zipStems([
+ { name: "a.wav", blob: lazy("a", "first") },
+ { name: "b.wav", blob: lazy("b", "second") },
+ ]);
+ assert.deepEqual(order, ["a", "b"]);
+ const files = unzipSync(new Uint8Array(await zip.arrayBuffer()));
+ assert.equal(strFromU8(files["a.wav"]), "first");
+ assert.equal(strFromU8(files["b.wav"]), "second");
+});
diff --git a/studio/frontend/tests/audio-workflows.test.ts b/studio/frontend/tests/audio-workflows.test.ts
index 885d9996835..5d2b14b8bab 100644
--- a/studio/frontend/tests/audio-workflows.test.ts
+++ b/studio/frontend/tests/audio-workflows.test.ts
@@ -20,11 +20,12 @@ const {
workflowForLoadedModel,
} = await import("../src/features/audio/workflows.ts");
-test("the Audio page offers Speak, Clone, Music and Transcribe in that order", () => {
+test("the Audio page offers Speak, Clone, Music, Separate and Transcribe in that order", () => {
assert.deepEqual(
AUDIO_WORKFLOWS.map((tab) => tab.id),
- ["speak", "clone", "music", "transcribe"],
+ ["speak", "clone", "music", "separate", "transcribe"],
);
+ assert.equal(audioWorkflowTab("separate").heading, "Separate audio");
assert.equal(audioWorkflowTab("speak").heading, "Text to speech");
assert.equal(audioWorkflowTab("clone").heading, "Clone a voice");
assert.equal(audioWorkflowTab("music").heading, "Create music");
@@ -35,6 +36,7 @@ test("Speak, Clone and Music share the main slot; only Transcribe uses the sidec
assert.equal(slotForWorkflow("speak"), "speak");
assert.equal(slotForWorkflow("clone"), "speak");
assert.equal(slotForWorkflow("music"), "speak");
+ assert.equal(slotForWorkflow("separate"), "speak");
assert.equal(slotForWorkflow("transcribe"), "transcribe");
});
@@ -42,6 +44,7 @@ test("Create|Train is offered on Speak, Clone and Transcribe, not Music", () =>
assert.equal(audioWorkflowTab("speak").createTrain, true);
assert.equal(audioWorkflowTab("clone").createTrain, true);
assert.equal(audioWorkflowTab("music").createTrain, false);
+ assert.equal(audioWorkflowTab("separate").createTrain, false);
assert.equal(audioWorkflowTab("transcribe").createTrain, true);
});
@@ -52,7 +55,8 @@ test("pipeline tags map onto workflows and anything else is ignored", () => {
assert.equal(audioWorkflowForTask("text-generation"), null);
assert.equal(audioWorkflowForTask(undefined), null);
assert.equal(isAudioWorkflowId("music"), true);
- assert.equal(isAudioWorkflowId("separate"), false);
+ assert.equal(isAudioWorkflowId("separate"), true);
+ assert.equal(isAudioWorkflowId("convert"), false);
});
test("a clip without a workflow falls back to its audio type", () => {
@@ -62,6 +66,7 @@ test("a clip without a workflow falls back to its audio type", () => {
assert.equal(clipWorkflow({ audio_type: "snac" }), "speak");
assert.equal(clipWorkflow({ workflow: "music", audio_type: "snac" }), "music");
assert.equal(clipWorkflow({ workflow: "clone", audio_type: "audiocpp_tts" }), "clone");
+ assert.equal(clipWorkflow({ workflow: "separate", audio_type: "audiocpp_sep" }), "separate");
assert.equal(clipWorkflow({ workflow: "edit", audio_type: "audiocpp_music" }), "music");
assert.equal(audioWorkflowForAudioType(null), "speak");
});
@@ -87,6 +92,10 @@ test("a loaded model keeps the open page when it can run it, else opens its own"
workflowForLoadedModel({ current: "speak", audioWorkflows: ["clone"], music: false }),
"clone",
);
+ assert.equal(
+ workflowForLoadedModel({ current: "speak", audioWorkflows: ["separate"], music: false }),
+ "separate",
+ );
assert.equal(
workflowForLoadedModel({ current: "speak", audioWorkflows: ["transcribe"], music: true }),
"music",
diff --git a/studio/frontend/tests/audio-workspace-split.test.ts b/studio/frontend/tests/audio-workspace-split.test.ts
index 24822157e5b..a914d3eb696 100644
--- a/studio/frontend/tests/audio-workspace-split.test.ts
+++ b/studio/frontend/tests/audio-workspace-split.test.ts
@@ -94,7 +94,7 @@ test("Generate says why it is off and runs from Mod+Enter anywhere on the page",
assert.match(host, /!pageRootRef\.current\?\.contains\(event\.target\)/);
assert.match(
host,
- /const handlePageGenerate =\s*ttsWorkflow === "clone" \? clone\.handleGenerate : handleGenerate;/,
+ /const handlePageGenerate =\s*ttsWorkflow === "separate" \? separate\.handleGenerate :\s*ttsWorkflow === "clone" \? clone\.handleGenerate : handleGenerate;/,
);
assert.match(host, /if \(canGenerate\) void handlePageGenerate\(\);/);
});