Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: add the Edit speech page (#12606)

Etherl9a43a457b04a
diff --git a/studio/backend/core/inference/audio_cpp_backend.py b/studio/backend/core/inference/audio_cpp_backend.py
index e0667cbf548..4d679f14469 100644
--- a/studio/backend/core/inference/audio_cpp_backend.py
+++ b/studio/backend/core/inference/audio_cpp_backend.py
@@ -19,7 +19,9 @@ from __future__ import annotations
import base64
import io
import json
+import os
import re
+import tempfile
import threading
import wave
from dataclasses import replace
@@ -119,6 +121,7 @@ def model_info_fields(model: AudioCppModel) -> dict[str, Any]:
"audio_reference_text": model.clone.reference_text if model.clone else None,
"audio_required_inputs": list(model.required_inputs),
"audio_clone": clone_rules(model),
+ "audio_edit": edit_rules(model),
"audio_music": music_rules(model),
}
@@ -136,6 +139,74 @@ def clone_rules(model: AudioCppModel) -> Optional[dict[str, Any]]:
}
+def edit_rules(model: AudioCppModel) -> Optional[dict[str, Any]]:
+ edit = getattr(model, "edit", None)
+ if edit is None:
+ return None
+ return {
+ "style": edit.style,
+ "delivery": edit.delivery_template is not None,
+ "max_changes": edit.max_changes,
+ }
+
+
+def edit_options(model: AudioCppModel) -> tuple[dict, ...]:
+ claims = set(model.edit.claims) if model.edit is not None else set()
+ return tuple(option for option in model.options if option["name"] not in claims)
+
+
+def edit_request_bodies(
+ model: AudioCppModel,
+ text: str,
+ reference_text: Optional[str],
+ edit: Optional[dict],
+ options: dict,
+ seed: Optional[int],
+ model_id: str,
+) -> list[dict[str, Any]]:
+ """The ``/v1/tasks/run`` bodies an edit posts, in order, without the recording field: the
+ first call reads the source, each later one the previous call's output."""
+ spec = model.edit
+ if spec is None:
+ raise RuntimeError(f"{model.display_name} cannot edit speech.")
+ edit = edit or {}
+ advanced = {name: _option_string(value) for name, value in options.items()}
+ top: dict[str, Any] = {"model": model_id}
+ if seed is not None:
+ top["seed"] = str(int(seed))
+ if spec.style == "markup":
+ markup = str(edit.get("markup") or "")
+ if not markup.strip():
+ raise RuntimeError(f"{model.display_name} needs the marked-up changes.")
+ return [{**top, "text": markup, "options": {"template_name": spec.template, **advanced}}]
+ if spec.style == "sentence":
+ body = {**top, "target_text": text}
+ if spec.route:
+ body["route"] = spec.route
+ original = str(reference_text or "").strip()
+ if original:
+ body["reference_text"] = original
+ if advanced:
+ body["options"] = advanced
+ return [body]
+ if edit.get("mode") == "delivery":
+ from core.inference.audio_edit import delivery_instructions
+
+ template = spec.delivery_template
+ if template is None:
+ raise RuntimeError("Delivery changes need FireRedAudio.")
+ instructions = delivery_instructions(edit.get("speed"), edit.get("pitch_steps"))
+ else:
+ template = spec.template
+ instructions = [str(item) for item in edit.get("instructions") or ()]
+ if not instructions:
+ raise RuntimeError("Change at least one word.")
+ return [
+ {**top, "options": {"template_name": template, "instruction": line, **advanced}}
+ for line in instructions
+ ]
+
+
def language_name(language: Optional[str]) -> Optional[str]:
"""``en`` -> ``English``; None for Auto or an unknown code."""
text = str(language or "").strip()
@@ -334,6 +405,7 @@ class AudioCppBackend:
audio_inputs: Optional[dict] = None,
reference_text: Optional[str] = None,
speed: Optional[float] = None,
+ edit: Optional[dict] = None,
music: Optional[dict] = None,
output_dir: Optional[str] = None,
) -> Tuple[bytes, int]:
@@ -357,13 +429,36 @@ class AudioCppBackend:
output_dir,
cancel_event,
)
- cloning = workflow == "clone" or bool(audio_inputs)
+ editing = workflow == "edit"
+ # Speak in a saved voice sends a reference with workflow "speak": a clone too.
+ cloning = not editing and (workflow == "clone" or bool(audio_inputs))
+ if editing and model.edit is None:
+ raise RuntimeError(f"{model.display_name} cannot edit speech.")
if cloning and model.clone is None:
raise RuntimeError(f"{model.display_name} cannot clone a voice.")
- options = validate_options(model.clone_options if cloning else model.options, audio_options)
- server = self._running_server(model, cancel_event)
+ if editing:
+ options = validate_options(edit_options(model), audio_options)
+ served = _served_for_edit(model)
+ else:
+ options = validate_options(
+ model.clone_options if cloning else model.options, audio_options
+ )
+ served = model
+ server = self._running_server(served, cancel_event)
try:
- if cloning:
+ if editing:
+ wav = self._generate_edit(
+ server,
+ model,
+ text,
+ audio_inputs or {},
+ reference_text,
+ edit,
+ options,
+ seed,
+ cancel_event,
+ )
+ elif cloning:
wav = self._generate_clone(
server,
model,
@@ -616,6 +711,44 @@ class AudioCppBackend:
)
return data
+ @staticmethod
+ def _generate_edit(
+ server: AudioCppServer,
+ model: AudioCppModel,
+ text: str,
+ audio_inputs: dict,
+ reference_text: Optional[str],
+ edit: Optional[dict],
+ options: dict,
+ seed: Optional[int],
+ cancel_event,
+ ) -> bytes:
+ """Edit ``audio_inputs["source"]`` (a server-local WAV). A chain hands each output to the
+ next call through a temporary WAV. The runtime's answer ``text`` is not the edited
+ transcript (FireRedAudio), so it is never read."""
+ source = audio_inputs.get("source")
+ if not source:
+ raise RuntimeError(f"{model.display_name} needs a recording to edit.")
+ bodies = edit_request_bodies(
+ model, text, reference_text, edit, options, seed, server.model_id
+ )
+ with tempfile.TemporaryDirectory(prefix = "unsloth-audio-edit-") as scratch:
+ path = str(source)
+ for index, body in enumerate(bodies):
+ _raise_if_cancelled(cancel_event)
+ if index:
+ path = os.path.join(scratch, f"step{index}.wav")
+ with open(path, "wb") as handle:
+ handle.write(wav)
+ ctype, data = server.post_json(
+ "/v1/tasks/run",
+ {**body, model.edit.source_field: path},
+ timeout = _GENERATE_TIMEOUT_SECONDS,
+ cancel_event = cancel_event,
+ )
+ wav = _audio_from_task_response(ctype, data)
+ return wav
+
@staticmethod
def _generate_music(
server: AudioCppServer,
@@ -808,6 +941,13 @@ def _resolve_companion(
return found
+def _served_for_edit(model: AudioCppModel) -> AudioCppModel:
+ edit = model.edit
+ if edit is None or edit.server_task == model.server_task:
+ return model
+ return replace(model, server_task = edit.server_task)
+
+
def _with_companions(model: AudioCppModel) -> AudioCppModel:
"""``model`` with each companion's served path in its session options (MioTTS's codec)."""
if not model.companions:
diff --git a/studio/backend/core/inference/audio_cpp_models.py b/studio/backend/core/inference/audio_cpp_models.py
index 96c64b8faa6..38592160756 100644
--- a/studio/backend/core/inference/audio_cpp_models.py
+++ b/studio/backend/core/inference/audio_cpp_models.py
@@ -102,6 +102,23 @@ class CloneSpec:
instructions: bool = False
+@dataclass(frozen = True)
+class EditSpec:
+ """How a family edits a recording through ``/v1/tasks/run``."""
+
+ server_task: str
+ # markup (DotTTS <sub>/<del>/<ins>), sentence (Vevo2) or instructions (FireRedAudio, chained).
+ style: str
+ source_field: str = "source_audio"
+ route: Optional[str] = None
+ template: Optional[str] = None
+ # Speed/pitch template; None = no delivery control.
+ delivery_template: Optional[str] = None
+ max_changes: Optional[int] = None
+ # Options Studio fills itself, so Advanced never sends them.
+ claims: tuple[str, ...] = ("template_name", "source_text", "target_text", "instruction")
+
+
@dataclass(frozen = True)
class SeparationSpec:
"""How a family splits a track into stems on /v1/tasks/run (task ``sep``, 44.1 kHz input)."""
@@ -301,6 +318,11 @@ def _bindings_for(
clone = getattr(family, "clone", None)
if clone is not None:
bindings["clone"] = WorkflowBinding(server_task or "tts", "speech", None, _CLONE_INPUTS)
+ edit = getattr(family, "edit", None)
+ if edit is not None:
+ bindings["edit"] = WorkflowBinding(
+ edit.server_task, "tasks", edit.route, ("source", "text", "reference_text")
+ )
return bindings
if task == "music":
return {
@@ -337,6 +359,7 @@ class AudioCppFamily:
speaks: bool = True
clone: Optional[CloneSpec] = None
companions: tuple[CompanionModel, ...] = ()
+ edit: Optional[EditSpec] = None
separation: Optional[SeparationSpec] = None
music: Optional[MusicSpec] = field(default = None, hash = False, compare = False)
@@ -502,9 +525,30 @@ _CLONE_FAMILIES: tuple[AudioCppFamily, ...] = (
# Instruct under tts defaults to instruct_tts, which needs reference audio.
AudioCppFamily("fireredtts3", "tts", speaks = False, clone = CloneSpec("optional")),
# "FireRedAce": under tts it defaults to tts_clone, which needs reference audio.
- AudioCppFamily("firered_audio", "tts", speaks = False, clone = CloneSpec("optional")),
- # Vevo2 reads a sent transcript as text to speak, so it never gets one.
- AudioCppFamily("vevo2", "tts", speaks = False, clone = CloneSpec("unused")),
+ AudioCppFamily(
+ "firered_audio",
+ "tts",
+ speaks = False,
+ clone = CloneSpec("optional"),
+ edit = EditSpec(
+ "tts",
+ "instructions",
+ source_field = "audio",
+ template = "semantic_edit",
+ delivery_template = "acoustic_edit",
+ max_changes = 5,
+ claims = ("template_name", "instruction"),
+ ),
+ ),
+ # Vevo2 reads a sent transcript as text to speak, so it never gets one. It edits only as an s2s
+ # session, so an edit after a Clone restarts the server.
+ AudioCppFamily(
+ "vevo2",
+ "tts",
+ speaks = False,
+ clone = CloneSpec("unused"),
+ edit = EditSpec("s2s", "sentence", route = "editing"),
+ ),
# MioCodec's sibling-folder default never matches the umbrella layout: pass it by session option.
AudioCppFamily(
"miotts",
@@ -784,6 +828,8 @@ _FAMILY_LIST: tuple[AudioCppFamily, ...] = (
FAMILIES: dict[str, AudioCppFamily] = {f.family: f for f in _FAMILY_LIST}
+_DOTS_EDIT = EditSpec("tts", "markup", template = "edit")
+
# Sessions refuse any backend but CPU ("Niagara ASR CPU variants require --backend cpu").
CPU_ONLY_FAMILIES: frozenset[str] = frozenset({"niagara_asr"})
@@ -915,6 +961,9 @@ def family_policy(
return replace(policy, music = _stable_audio_music(names))
if family == "fireredtts3" and re.search(r"(^|[-_ /])base([-_ ./]|$)", " ".join(names).lower()):
return replace(policy, server_task = "clon")
+ if family == "dots_tts" and re.search(r"(^|[-_ /])edit([-_ ./]|$)", " ".join(names).lower()):
+ # DotTTS-MF and SOAR only speak.
+ return replace(policy, edit = _DOTS_EDIT)
return policy
@@ -1814,6 +1863,7 @@ class AudioCppModel:
clone: Optional[CloneSpec] = None
companions: tuple[CompanionModel, ...] = ()
required_inputs: tuple[str, ...] = ()
+ edit: Optional[EditSpec] = None
separation: Optional[SeparationSpec] = None
music: Optional[MusicSpec] = field(default = None, hash = False, compare = False)
# A strict spec (``schema_version``) makes the runtime refuse any undeclared option.
@@ -2187,6 +2237,7 @@ def _resolve_uncached(
clone = policy.clone,
companions = policy.companions,
required_inputs = _required_inputs(spec, embedded, policy),
+ edit = policy.edit,
separation = policy.separation,
music = music_with_spec_bounds(policy.music, _raw_request_options(spec, embedded)),
request_keys = _request_keys(spec, embedded),
diff --git a/studio/backend/core/inference/audio_cpp_server.py b/studio/backend/core/inference/audio_cpp_server.py
index 8df89599381..d8c7dd3d78d 100644
--- a/studio/backend/core/inference/audio_cpp_server.py
+++ b/studio/backend/core/inference/audio_cpp_server.py
@@ -673,9 +673,12 @@ class AudioCppServer:
process.wait(timeout = 10)
except subprocess.TimeoutExpired:
process.kill()
- # A killed server can sit in GPU context teardown well past 10 s on a busy card;
- # the music reload restarts it mid-session and must not fail on that.
- process.wait(timeout = 120)
+ try:
+ # A killed server can sit in GPU context teardown well past 10 s on a busy
+ # card; the music reload restarts it mid-session and must not fail on that.
+ process.wait(timeout = 120)
+ except subprocess.TimeoutExpired:
+ logger.warning("audiocpp_server %s did not exit after SIGKILL", process.pid)
forget_pid(process.pid)
shutil.rmtree(self._config_dir, ignore_errors = True)
diff --git a/studio/backend/core/inference/audio_edit.py b/studio/backend/core/inference/audio_edit.py
new file mode 100644
index 00000000000..6fcfadeaadb
--- /dev/null
+++ b/studio/backend/core/inference/audio_edit.py
@@ -0,0 +1,234 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Checks for an Edit speech request, and the source clip an edit keeps in history.
+
+The client renders the word diff (DotTTS markup or FireRedAudio instructions); each check holds it
+against both transcripts, so a malformed or injected value never reaches the runtime.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from pathlib import Path
+from typing import Any, Optional, Sequence
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+# Refused, never cut: a cut recording no longer matches its transcript.
+EDIT_SOURCE_MAX_SECONDS = 30.0
+
+TOO_LONG = "Edit works on recordings up to 30 s. Record or upload a shorter take."
+NO_CHANGE = "Change at least one word."
+MISMATCH = "The changes do not match the transcript. Check ① and ② again."
+UNKNOWN_INSTRUCTION = "That change is not one FireRedAudio understands."
+NEEDS_DELIVERY_MODEL = "Delivery changes need FireRedAudio."
+NO_DELIVERY = "Pick a speed or a pitch change."
+
+_MARKUP_TAG_RE = re.compile(
+ r'<sub targ="([^"<>]*)">([^<>]*)</sub>|<del>([^<>]*)</del>|<ins>([^<>]*)</ins>'
+)
+INSTRUCTION_RE = re.compile(
+ r"^(?:Replace '(.+)' with '(.+)'|Delete '(.+)'|Insert '(.+)' before '(.+)')\.$"
+)
+
+
+def _collapse(text: Optional[str]) -> str:
+ return " ".join(str(text or "").split())
+
+
+def markup_tags(markup: str) -> int:
+ return len(_MARKUP_TAG_RE.findall(markup or ""))
+
+
+def markup_sides(markup: str) -> Optional[tuple[str, str]]:
+ """``(source, target)`` text a DotTTS markup describes, whitespace-collapsed; None when it holds
+ anything but the three tags."""
+ source: list[str] = []
+ target: list[str] = []
+ position = 0
+ for match in _MARKUP_TAG_RE.finditer(markup or ""):
+ between = markup[position : match.start()]
+ if "<" in between or ">" in between:
+ return None
+ source.append(between)
+ target.append(between)
+ targ, sub, deleted, inserted = match.groups()
+ if sub is not None:
+ source.append(sub)
+ target.append(targ)
+ elif deleted is not None:
+ source.append(deleted)
+ else:
+ target.append(inserted)
+ position = match.end()
+ rest = (markup or "")[position:]
+ if "<" in rest or ">" in rest:
+ return None
+ source.append(rest)
+ target.append(rest)
+ return _collapse("".join(source)), _collapse("".join(target))
+
+
+def check_markup(markup: str, original: str, edited: str) -> Optional[str]:
+ """Why ``markup`` does not turn ``original`` into ``edited``; None when it does."""
+ sides = markup_sides(markup)
+ if sides is None:
+ return MISMATCH
+ if not markup_tags(markup):
+ return NO_CHANGE
+ if sides != (_collapse(original), _collapse(edited)):
+ return MISMATCH
+ return None
+
+
+def check_instructions(
+ instructions: Sequence[str],
+ original: str,
+ edited: str,
+ max_changes: Optional[int],
+ label: str = "FireRedAudio",
+) -> Optional[str]:
+ """Why FireRedAudio ``instructions`` are refused; None when each is a known form whose old words
+ (or anchor) are in ``original`` and new words in ``edited``."""
+ items = list(instructions or ())
+ if not items:
+ return NO_CHANGE
+ if max_changes is not None and len(items) > max_changes:
+ return (
+ f"{label} applies at most {max_changes} changes. "
+ "Make fewer changes, or use DotTTS Edit."
+ )
+ before, after = _collapse(original), _collapse(edited)
+ for instruction in items:
+ match = INSTRUCTION_RE.match(str(instruction))
+ if match is None:
+ return UNKNOWN_INSTRUCTION
+ old, new, deleted, inserted, anchor = match.groups()
+ if old is not None:
+ olds, news = [old], [new]
+ elif deleted is not None:
+ olds, news = [deleted], []
+ else:
+ olds, news = [anchor], [inserted]
+ if not all(_collapse(o) and _collapse(o) in before for o in olds):
+ return MISMATCH
+ if not all(_collapse(n) and _collapse(n) in after for n in news):
+ return MISMATCH
+ return None
+
+
+def delivery_instructions(speed: Optional[float], pitch_steps: Optional[int]) -> list[str]:
+ """FireRedAudio acoustic_edit instructions, one per call; pitch only rises."""
+ out: list[str] = []
+ if speed is not None and float(speed) != 1.0:
+ out.append(f"adjust the speed to {float(speed):g}x")
+ if pitch_steps:
+ out.append(f"shift the pitch by {int(pitch_steps)} steps")
+ return out
+
+
+def change_count(edit: dict[str, Any]) -> int:
+ if edit.get("mode") == "delivery":
+ return len(delivery_instructions(edit.get("speed"), edit.get("pitch_steps")))
+ if edit.get("markup"):
+ return markup_tags(edit["markup"])
+ if edit.get("instructions"):
+ return len(edit["instructions"])
+ return 1
+
+
+def request_problem(
+ rules: Optional[dict[str, Any]],
+ edit: Optional[dict[str, Any]],
+ text: str,
+ reference_text: Optional[str],
+ label: str,
+) -> Optional[str]:
+ """Why the loaded model's ``audio_edit`` rules refuse this edit; else None."""
+ edit = edit or {}
+ rules = rules or {}
+ style = rules.get("style")
+ original = reference_text or ""
+ if edit.get("mode") == "delivery":
+ if not rules.get("delivery"):
+ return NEEDS_DELIVERY_MODEL
+ if not delivery_instructions(edit.get("speed"), edit.get("pitch_steps")):
+ return NO_DELIVERY
+ return None
+ if style == "markup":
+ markup = edit.get("markup")
+ if not markup:
+ if _collapse(text) == _collapse(original):
+ return NO_CHANGE
+ return f"{label} needs the marked-up changes."
+ return check_markup(markup, original, text)
+ if style == "instructions":
+ return check_instructions(
+ edit.get("instructions") or (), original, text, rules.get("max_changes"), label
+ )
+ if style == "sentence":
+ if original.strip() and _collapse(text) == _collapse(original):
+ return NO_CHANGE
+ return None
+ return "Load a model that can edit speech."
+
+
+def wav_seconds(path: Path) -> Optional[float]:
+ """Length of a WAV from its header; None when unreadable."""
+ import wave
+ try:
+ with wave.open(str(path)) as w:
+ return w.getnframes() / w.getframerate()
+ except Exception: # noqa: BLE001 - the caller measures the prepared copy instead
+ return None
+
+
+def save_source_clip(
+ source, prepared_path: Path, reference_text: Optional[str]
+) -> Optional[dict[str, Any]]:
+ """Keep an uploaded recording in history beside its edit, so A/B outlives the upload. Reuses a
+ source clip of the same audio; None for a history clip or an upload with no record."""
+ import wave
+
+ from core.inference import audio_gallery, audio_inputs
+
+ if getattr(source, "kind", None) != "input":
+ return None
+ sidecar = audio_inputs._read_sidecar(audio_inputs._sidecar(source.path.parent, source.id))
+ sha = (sidecar or {}).get("sha256")
+ if not sha:
+ return None
+ directory = audio_gallery.gallery_dir()
+ try:
+ sidecars = list(directory.glob("*.json"))
+ except OSError:
+ sidecars = []
+ for path in sidecars:
+ meta = audio_gallery._read_meta(path)
+ if (
+ meta is not None
+ and meta.get("role") == "source"
+ and meta.get("workflow") == "edit"
+ and meta.get("source_sha256") == sha
+ and audio_gallery.owned_audio_path(path.stem) is not None
+ ):
+ return audio_gallery._record(path.stem, meta)
+ with wave.open(str(prepared_path)) as w:
+ sample_rate, duration_s = w.getframerate(), round(w.getnframes() / w.getframerate(), 3)
+ meta: dict[str, Any] = {
+ "prompt": " ".join(str(reference_text or "").split()) or source.name,
+ "model": "Recording",
+ "audio_type": "recording",
+ "workflow": "edit",
+ "role": "source",
+ "source_sha256": sha,
+ "reference_name": source.name,
+ "sample_rate": sample_rate,
+ "duration_s": duration_s,
+ "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+ }
+ return audio_gallery.save(Path(prepared_path).read_bytes(), meta)
diff --git a/studio/backend/core/inference/audio_gallery.py b/studio/backend/core/inference/audio_gallery.py
index 0156b7bca4b..fd84a8f230c 100644
--- a/studio/backend/core/inference/audio_gallery.py
+++ b/studio/backend/core/inference/audio_gallery.py
@@ -188,14 +188,17 @@ def _prune_to_cap() -> int:
# parse, which here would drop the clips the shelf exists to keep. It also covers filesystems where the
# cross-process lock degrades to a no-op.
flags = gallery_flags.read_trusted(directory)
+ victims = [
+ record["id"]
+ for record, _cursor in entries[keep:]
+ if record.get("group_id") not in kept_groups
+ and not gallery_flags.is_archived(flags, record["id"])
+ and gallery_flags.pin_rank(flags, record["id"]) == float("-inf")
+ ]
+ in_use = _sources_in_use(directory, set(victims))
pruned: list[str] = []
- for record, _cursor in entries[keep:]:
- if record.get("group_id") in kept_groups:
- continue
- audio_id = record["id"]
- if gallery_flags.is_archived(flags, audio_id) or gallery_flags.pin_rank(
- flags, audio_id
- ) > float("-inf"):
+ for audio_id in victims:
+ if audio_id in in_use:
continue
path = audio_path(audio_id)
if path is None or _read_meta(_sidecar_path(audio_id)) is None:
@@ -251,7 +254,7 @@ def _record(
def _workflow(meta: dict[str, Any]) -> str:
workflow = meta.get("workflow")
- if workflow in ("speak", "clone", "music", "separate"):
+ if workflow in ("speak", "clone", "edit", "music", "separate"):
return workflow
return workflow_for_audio_type(meta.get("audio_type"))
@@ -272,6 +275,25 @@ def _sidecar_path(audio_id: str) -> Path:
return gallery_dir() / f"{audio_id}.json"
+def _sources_in_use(directory: Path, leaving: set[str]) -> set[str]:
+ """Hidden Edit sources that a clip staying in the gallery still plays as its Original."""
+ try:
+ paths = list(directory.glob("*.wav"))
+ except OSError:
+ return set()
+ sources: set[str] = set()
+ used: set[str] = set()
+ for path in paths:
+ meta = _read_meta(_sidecar_path(path.stem))
+ if meta is None:
+ continue
+ if meta.get("role") == "source":
+ sources.add(path.stem)
+ if path.stem not in leaving and meta.get("source_clip_id"):
+ used.add(str(meta["source_clip_id"]))
+ return used & sources
+
+
# Key-presence ownership test: a hand-dropped wav with a partial sidecar is neither counted as ours nor destroyed.
_REQUIRED_META = (
"prompt",
@@ -474,8 +496,8 @@ def clear(include_archived: bool = False, workflow: Optional[str] = None) -> int
orphan WAVs are preserved, since list_audio already hides them.
Archived clips are spared unless ``include_archived``, and sparing them raises
- FlagsUnavailable when the flag store cannot be read. A ``workflow`` (speak, clone, music or
- separate) spares the other workflows' clips."""
+ FlagsUnavailable when the flag store cannot be read. A ``workflow`` (speak, clone, edit, music
+ or separate) spares the other workflows' clips."""
removed = 0
directory = gallery_dir()
with gallery_flags.exclusive(directory, require_file_lock = not include_archived):
@@ -484,7 +506,7 @@ def clear(include_archived: bool = False, workflow: Optional[str] = None) -> int
paths = list(directory.glob("*.wav"))
except OSError:
return 0
- cleared: list[str] = []
+ doomed = []
for path in paths:
meta = _read_meta(_sidecar_path(path.stem))
if meta is None:
@@ -493,6 +515,12 @@ def clear(include_archived: bool = False, workflow: Optional[str] = None) -> int
continue
if not include_archived and gallery_flags.is_archived(flags, path.stem):
continue
+ doomed.append(path)
+ in_use = _sources_in_use(directory, {path.stem for path in doomed})
+ cleared: list[str] = []
+ for path in doomed:
+ if path.stem in in_use:
+ continue
try:
path.unlink()
except OSError:
diff --git a/studio/backend/core/inference/audio_workflows.py b/studio/backend/core/inference/audio_workflows.py
index e46521d20fb..35b45372a6f 100644
--- a/studio/backend/core/inference/audio_workflows.py
+++ b/studio/backend/core/inference/audio_workflows.py
@@ -1,8 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Which Audio page workflows (Speak, Clone, Music, Separate, Transcribe) a model or a clip
-belongs to.
+"""Which Audio page workflows (Speak, Clone, Edit, Music, Separate, Transcribe) a model or a
+clip belongs to.
Mirrors ``studio/frontend/src/features/audio/workflows.ts``. This module reads the audio.cpp
constants; ``audio_cpp_models`` never imports it.
@@ -18,7 +18,7 @@ from core.inference.audio_cpp_models import (
HUB_TASKS,
)
-AUDIO_WORKFLOW_IDS = ("speak", "clone", "music", "separate", "transcribe")
+AUDIO_WORKFLOW_IDS = ("speak", "clone", "edit", "music", "separate", "transcribe")
# Generation audio types that make music; every other generation audio type speaks.
MUSIC_AUDIO_TYPES = frozenset(("minimax_music3", AUDIO_CPP_MUSIC_AUDIO_TYPE))
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 71f246a7f37..63683156da5 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -391,6 +391,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"audio_reference_text": model_info.get("audio_reference_text"),
"audio_required_inputs": model_info.get("audio_required_inputs"),
"audio_clone": model_info.get("audio_clone"),
+ "audio_edit": model_info.get("audio_edit"),
"audio_music": model_info.get("audio_music"),
"audio_cpp_backend": model_info.get("audio_cpp_backend"),
}
@@ -3373,6 +3374,7 @@ class InferenceOrchestrator:
audio_inputs: Optional[dict[str, str]] = None,
reference_text: Optional[str] = None,
speed: Optional[float] = None,
+ edit: Optional[dict] = None,
music: Optional[dict] = None,
output_dir: Optional[str] = None,
stats_holder: Optional[dict] = None,
@@ -3457,6 +3459,8 @@ class InferenceOrchestrator:
cmd["reference_text"] = reference_text
if speed is not None:
cmd["speed"] = float(speed)
+ if edit is not None:
+ cmd["edit"] = dict(edit)
if music is not None:
cmd["music"] = dict(music)
try:
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 40772009b88..249f531702a 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -795,6 +795,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"audio_reference_text",
"audio_required_inputs",
"audio_clone",
+ "audio_edit",
"audio_music",
"audio_cpp_backend",
)
@@ -1632,7 +1633,7 @@ def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) ->
# workflow its backend has no keyword for.
params = inspect.signature(backend.generate_audio_response).parameters
takes_any = any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
- for key in ("workflow", "audio_inputs", "reference_text", "speed"):
+ for key in ("workflow", "audio_inputs", "reference_text", "speed", "edit"):
if cmd.get(key) is None:
continue
if takes_any or key in params:
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index ceb17379ac9..62c5ab01395 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -5149,7 +5149,8 @@ class AudioGalleryItem(BaseModel):
pinned: bool = Field(False, description = "Pinned to the top of history")
archived: bool = Field(False, description = "Moved to the archived shelf, hidden from history")
workflow: Optional[str] = Field(
- None, description = "Audio page workflow that made the clip: speak, clone, music or separate"
+ None,
+ description = "Audio page workflow that made the clip: speak, clone, edit, music or separate",
)
order_at: Optional[float] = Field(
None,
@@ -5209,6 +5210,21 @@ class AudioRunInputs(BaseModel):
emotion: Optional[AudioSourceRef] = None
+class AudioRunEdit(BaseModel):
+ """Words: client-rendered DotTTS markup or FireRedAudio instructions, checked against both
+ transcripts. Delivery (FireRedAudio): numbers only."""
+
+ model_config = ConfigDict(extra = "forbid")
+
+ mode: Literal["words", "delivery"] = "words"
+ markup: Optional[str] = Field(None, max_length = 8000)
+ instructions: Optional[List[Annotated[str, Field(min_length = 1, max_length = 300)]]] = Field(
+ None, max_length = 8
+ )
+ speed: Optional[float] = Field(None, ge = 0.5, le = 2.0)
+ pitch_steps: Optional[int] = Field(None, ge = 1, le = 12)
+
+
class AudioMusicRange(BaseModel):
model_config = ConfigDict(extra = "forbid")
@@ -5236,7 +5252,7 @@ class AudioRunRequest(BaseModel):
model_config = ConfigDict(extra = "forbid")
- workflow: Literal["clone", "speak", "music", "separate"]
+ workflow: Literal["clone", "speak", "edit", "music", "separate"]
# Required except for a separation (the route answers a separation given text with a 400).
text: Optional[str] = None
language: Optional[str] = Field(None, max_length = 64)
@@ -5247,7 +5263,8 @@ class AudioRunRequest(BaseModel):
instrumental: bool = False
duration_s: Optional[float] = Field(None, ge = 0.5, le = 600)
variations: int = Field(1, ge = 1, le = 4)
- edit: Optional[AudioMusicEdit] = None
+ # Music edits carry an action; speech edits (workflow edit) never do.
+ edit: Optional[Union[AudioMusicEdit, AudioRunEdit]] = None
options: Optional[Dict[str, Any]] = Field(
None, description = "Per-model options, as listed in audio_options or by a tool panel"
)
@@ -5255,6 +5272,18 @@ class AudioRunRequest(BaseModel):
seed: Optional[int] = Field(None, ge = -(2**63), le = 2**64 - 1)
max_tokens: Optional[int] = Field(None, ge = 1)
+ @model_validator(mode = "after")
+ def _edit_fields(self):
+ if self.workflow == "edit" and not isinstance(self.edit, AudioRunEdit):
+ raise ValueError("Send edit with workflow edit.")
+ if isinstance(self.edit, AudioRunEdit) and self.workflow != "edit":
+ raise ValueError("A speech edit is for workflow edit.")
+ if self.workflow == "edit" and (
+ self.inputs.reference is not None or self.inputs.emotion is not None
+ ):
+ raise ValueError("An edit takes inputs.source, not a reference or emotion clip.")
+ return self
+
@field_validator("options")
@classmethod
def _no_file_options(cls, value):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 595399bbe44..9360f0b99ca 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -22027,6 +22027,21 @@ def _audio_request_problem(
workflows = model_info.get("audio_workflows")
label = _audio_model_label(model_name)
inputs = (run_inputs or {}).get("audio_inputs") or {}
+ if (run_inputs or {}).get("workflow") == "edit":
+ from core.inference import audio_edit
+
+ # The llama.cpp direct path reports no workflows.
+ if not workflows or "edit" not in workflows or not model_info.get("audio_edit"):
+ return "Load a model that can edit speech."
+ if not inputs.get("source"):
+ return "Add a recording to edit."
+ return audio_edit.request_problem(
+ model_info.get("audio_edit"),
+ run_inputs.get("edit"),
+ str(run_inputs.get("text") or ""),
+ run_inputs.get("reference_text"),
+ label,
+ )
if run_inputs and run_inputs.get("workflow") == "music":
# The model may have changed since the route checked the mode.
if not model_info.get("audio_music"):
@@ -22173,7 +22188,15 @@ async def _generate_tts_wav(
key: value
for key, value in (run_inputs or {}).items()
if key
- in ("workflow", "audio_inputs", "reference_text", "speed", "music", "output_dir")
+ in (
+ "workflow",
+ "audio_inputs",
+ "reference_text",
+ "speed",
+ "edit",
+ "music",
+ "output_dir",
+ )
and value is not None
},
stats_holder = stats_holder,
@@ -22435,6 +22458,138 @@ def _audio_run_settings(body: AudioRunRequest, reference_text_used: bool) -> dic
}
+def _audio_edit_settings(body: AudioRunRequest) -> dict[str, Any]:
+ """An edit's recipe for history; never a file or server path."""
+ from core.inference import audio_edit
+
+ edit = body.edit.model_dump(exclude_none = True) if body.edit is not None else {}
+ settings: dict[str, Any] = {
+ "mode": edit.get("mode", "words"),
+ "changes": audio_edit.change_count(edit),
+ "original_text": (body.inputs.reference_text or "").strip() or None,
+ "options": _audio_run_settings(body, False)["options"],
+ }
+ if edit.get("mode") == "delivery":
+ for name in ("speed", "pitch_steps"):
+ if edit.get(name) is not None:
+ settings[name] = edit[name]
+ return settings
+
+
+def _audio_run_clip(record: dict[str, Any], role: str = "output") -> dict[str, Any]:
+ return {
+ "role": role,
+ **{k: record[k] for k in ("id", "url", "sample_rate", "duration_s", "workflow")},
+ }
+
+
+def _audio_run_response(
+ record: Optional[dict[str, Any]],
+ wav_bytes: bytes,
+ sample_rate: int,
+ model_name: str,
+ sources: list[dict[str, Any]] = (),
+) -> AudioRunResponse:
+ """The clips a run made; the WAV inline when history could not keep it."""
+ import base64
+
+ if record is None:
+ audio = {"data": base64.b64encode(wav_bytes).decode("ascii"), "sample_rate": sample_rate}
+ return AudioRunResponse(clips = list(sources), model = model_name, audio = audio)
+ return AudioRunResponse(clips = [_audio_run_clip(record), *sources], model = model_name)
+
+
+async def _run_audio_edit(
+ body: AudioRunRequest, request: Request, current_subject: str
+) -> AudioRunResponse:
+ """Edit speech on a recording from the caller's account, prepared as 24 kHz mono. Over 30 s is
+ refused, never cut; an uploaded recording is kept in history as the run's source."""
+ from core.inference import audio_edit, audio_inputs
+
+ if any(
+ name in body.model_fields_set and getattr(body, name) not in (None, False, 1)
+ for name in _MUSIC_ONLY_FIELDS
+ if name != "edit"
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = "A music mode, lyrics and variations apply to Music only.",
+ )
+ source_ref = body.inputs.source
+ if source_ref is None:
+ raise HTTPException(status_code = 400, detail = "Add a recording to edit.")
+ if source_ref.voice_id:
+ raise HTTPException(status_code = 400, detail = "Pick a recording or a history clip to edit.")
+ ref = source_ref.model_dump(exclude_none = True)
+ limit = audio_edit.EDIT_SOURCE_MAX_SECONDS + 0.05
+
+ def prepare():
+ source = audio_inputs.resolve_source(ref)
+ seconds = audio_edit.wav_seconds(source.path)
+ if seconds is not None and seconds > limit:
+ raise audio_inputs.AudioInputError(400, audio_edit.TOO_LONG)
+ path = audio_inputs.prepared_path(source, audio_inputs.REFERENCE_RATE)
+ if seconds is None and (audio_edit.wav_seconds(path) or 0.0) > limit:
+ raise audio_inputs.AudioInputError(400, audio_edit.TOO_LONG)
+ return source, path
+
+ try:
+ source, source_path = await asyncio.to_thread(prepare)
+ except audio_inputs.AudioInputError as exc:
+ raise _audio_source_error(exc) from None
+ reference_text = (body.inputs.reference_text or "").strip() or None
+ edit = body.edit.model_dump(exclude_none = True) if body.edit is not None else {}
+ payload = ChatCompletionRequest(
+ messages = [{"role": "user", "content": body.text}],
+ max_tokens = body.max_tokens,
+ audio_options = body.options,
+ seed = body.seed,
+ )
+ wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(
+ body.text,
+ payload,
+ request,
+ current_subject,
+ run_inputs = {
+ "workflow": "edit",
+ "audio_inputs": {"source": str(source_path)},
+ "reference_text": reference_text,
+ "edit": edit,
+ # For the request check; the worker gets the text as the prompt.
+ "text": body.text,
+ },
+ )
+ source_record = None
+ try:
+ source_record = await asyncio.to_thread(
+ audio_edit.save_source_clip, source, source_path, reference_text
+ )
+ except Exception as exc: # noqa: BLE001 - the edit is served even when its source is not kept
+ logger.warning("audio_edit.source_save_failed: %s", exc)
+ if source_record is not None:
+ source_clip_id = source_record["id"]
+ else:
+ source_clip_id = source.id if source.kind == "clip" else None
+ extra_meta: dict[str, Any] = {
+ "role": "output",
+ "source_clip_id": source_clip_id,
+ "reference_name": source.name,
+ "settings": _audio_edit_settings(body),
+ }
+ record = await asyncio.to_thread(
+ _persist_tts_clip,
+ wav_bytes,
+ sample_rate,
+ body.text,
+ model_name,
+ audio_type,
+ extra_meta,
+ "edit",
+ )
+ sources = [_audio_run_clip(source_record, "source")] if source_record is not None else []
+ return _audio_run_response(record, wav_bytes, sample_rate, model_name, sources)
+
+
_REPAINT_BEYOND_END_S = 30.0
_MUSIC_ONLY_FIELDS = ("mode", "lyrics", "instrumental", "duration_s", "variations", "edit")
_MUSIC_MODE_NAMES = {"song": "songs", "sfx": "sound effects", "edit": "edits"}
@@ -22755,15 +22910,17 @@ async def run_audio_workflow(
request: Request,
current_subject: str = Depends(get_current_subject),
):
- """Run one Audio page workflow (Clone, Speak in a saved voice, Music or Separate); clips go to
- history. Audio is named by id and resolved here in the caller's account; the worker gets a
- prepared copy's path, never bytes."""
+ """Run one Audio page workflow (Clone, Speak in a saved voice, Edit, Music or Separate); clips
+ go to history. Audio is named by id and resolved here in the caller's account; the worker gets
+ a prepared copy's path, never bytes."""
if body.workflow == "separate":
return await _run_separation(body, request, current_subject)
import base64
from core.inference import audio_inputs
+ if body.workflow == "edit":
+ return await _run_audio_edit(body, request, current_subject)
if body.workflow == "music":
return await _run_music_workflow(body, request, current_subject)
music_fields = sorted(
@@ -22842,11 +22999,7 @@ async def run_audio_workflow(
extra_meta,
body.workflow,
)
- if record is None:
- audio = {"data": base64.b64encode(wav_bytes).decode("ascii"), "sample_rate": sample_rate}
- return AudioRunResponse(model = model_name, audio = audio)
- clip = {k: record[k] for k in ("id", "url", "sample_rate", "duration_s", "workflow")}
- return AudioRunResponse(clips = [clip], model = model_name)
+ return _audio_run_response(record, wav_bytes, sample_rate, model_name)
# Stems in the order the Separate page lists them; ids a runtime adds follow in its own order.
@@ -45420,7 +45573,7 @@ async def delete_gallery_audio(audio_id: str, current_subject: str = Depends(get
@studio_router.delete("/audio/gallery")
async def clear_gallery_audio(
- workflow: Optional[Literal["speak", "clone", "music", "separate"]] = None,
+ workflow: Optional[Literal["speak", "clone", "edit", "music", "separate"]] = None,
current_subject: str = Depends(get_current_subject),
):
from core.inference import audio_gallery
diff --git a/studio/backend/tests/test_audio_cpp_models.py b/studio/backend/tests/test_audio_cpp_models.py
index 29db7a6670b..da2bd272244 100644
--- a/studio/backend/tests/test_audio_cpp_models.py
+++ b/studio/backend/tests/test_audio_cpp_models.py
@@ -535,7 +535,8 @@ def test_every_task_family_binds_its_workflows():
"separate": "tasks",
}
for family in acm.FAMILIES.values():
- bindings = family.workflows
+ # Edit is checked by its own table below.
+ bindings = {k: v for k, v in family.workflows.items() if k != "edit"}
if not family.task:
assert bindings == {}, family.family
continue
@@ -576,6 +577,29 @@ def test_every_task_family_binds_its_workflows():
}
+def test_edit_families_bind_the_edit_workflow():
+ edit_names = ["DotTTS-Edit-GGUF", "dots-tts-edit-q8_0.gguf"]
+ dots_edit = acm.family_policy("dots_tts", names = edit_names)
+ binding = dots_edit.workflows["edit"]
+ assert list(dots_edit.workflows) == ["speak", "edit"]
+ assert (binding.server_task, binding.endpoint) == ("tts", "tasks")
+ for names in (["DotTTS-MF-GGUF"], ["DotTTS-SOAR-GGUF"], []):
+ assert list(acm.family_policy("dots_tts", names = names).workflows) == ["speak"], names
+ vevo2, firered = acm.FAMILIES["vevo2"], acm.FAMILIES["firered_audio"]
+ assert list(vevo2.workflows) == list(firered.workflows) == ["clone", "edit"]
+ # Vevo2 edits in an s2s session though it loads as tts.
+ assert vevo2.default_server_task == "tts"
+ assert (vevo2.workflows["edit"].server_task, vevo2.workflows["edit"].route) == (
+ "s2s",
+ "editing",
+ )
+ assert firered.workflows["edit"].server_task == "tts"
+ assert {n for n, f in acm.FAMILIES.items() if f.edit is not None} == {"vevo2", "firered_audio"}
+ assert acm.family_policy("auk").unsupported
+ for family in (dots_edit, vevo2, firered):
+ hash(family)
+
+
def test_a_resolved_model_carries_its_workflow_binding(hub):
snap = _snapshot(hub)
folder = "Qwen3-TTS-12Hz-1.7B-VoiceDesign-GGUF"
diff --git a/studio/backend/tests/test_audio_cpp_server_http.py b/studio/backend/tests/test_audio_cpp_server_http.py
index f84d1336343..9ed3562ed0d 100644
--- a/studio/backend/tests/test_audio_cpp_server_http.py
+++ b/studio/backend/tests/test_audio_cpp_server_http.py
@@ -281,3 +281,30 @@ def test_a_custom_build_launches_on_a_backend_it_was_compiled_with(tmp_path, mon
assert srv.select_backend(build("cuda", "cpu,cuda"), False) == "cuda"
# A build too old to report keeps the host guess.
assert srv.select_backend(build("silent", ""), False) == "cuda"
+
+
+def test_stop_survives_a_server_slow_to_exit_after_sigkill(tmp_path):
+ # A server switching task mid-request (Vevo2 Clone <-> Edit) failed the request when the old
+ # process outlived the SIGKILL wait in GPU teardown.
+ class Stuck:
+ pid = 999999999
+ waits: list = []
+
+ def poll(self):
+ return None
+
+ def terminate(self):
+ pass
+
+ def kill(self):
+ pass
+
+ def wait(self, timeout):
+ self.waits.append(timeout)
+ raise srv.subprocess.TimeoutExpired("audiocpp_server", timeout)
+
+ server = object.__new__(srv.AudioCppServer)
+ server.process = Stuck()
+ server._config_dir = tmp_path / "cfg"
+ server.stop()
+ assert Stuck.waits == [10, 120]
diff --git a/studio/backend/tests/test_audio_edit_requests.py b/studio/backend/tests/test_audio_edit_requests.py
new file mode 100644
index 00000000000..4365869dd99
--- /dev/null
+++ b/studio/backend/tests/test_audio_edit_requests.py
@@ -0,0 +1,314 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The exact requests audio.cpp receives for an Edit run, per family. The golden cases share a
+fixture with the frontend, which builds each case's run body; a fake server records the calls."""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import re
+import threading
+import wave
+from pathlib import Path
+
+import pytest
+
+from core.inference import audio_cpp_backend as acb
+from core.inference import audio_cpp_files
+from core.inference import audio_cpp_models as acm
+from core.inference import audio_edit
+from core.inference.audio_cpp_models import AUDIO_CPP_REPO, AudioCppModel, AudioCppVariant, RepoFile
+from core.inference.audio_errors import AudioGenerationCancelledError
+
+FIXTURE = Path(__file__).resolve().parents[2] / "frontend/tests/fixtures/audio-edit-requests.json"
+_CASES = json.loads(FIXTURE.read_text(encoding = "utf-8"))["cases"]
+SOURCE = "/srv/accounts/a/audio/inputs/0123.24000.mono.wav"
+REF = "/srv/accounts/a/audio/inputs/4567.24000.mono.m30.wav"
+ORIGINAL = "Okay, I'm Cemo and what you just heard wasn't a human voice."
+EDITED = "Okay, I'm Sam and what you just heard wasn't a robot voice."
+MARKUP = (
+ "Okay, I'm <sub targ=\"Sam\">Cemo</sub> and what you just heard wasn't a "
+ '<sub targ="robot">human</sub> voice.'
+)
+TWO = ["Replace 'Cemo' with 'Sam'.", "Replace 'human' with 'robot'."]
+MODEL_ID = "studio-test"
+_NAMES = {
+ "dots_tts": ("DotTTS-Edit-GGUF", "dots-tts-edit-q8_0.gguf"),
+ "vevo2": ("Vevo2-GGUF", "vevo2-q8_0.gguf"),
+ "firered_audio": ("FireRedAudio-GGUF", "firered-audio-q8_0.gguf"),
+ "voxcpm2": ("VoxCPM2-GGUF", "voxcpm2-q8_0.gguf"),
+}
+_RULES = {
+ "dots_tts": {"style": "markup", "delivery": False, "max_changes": None},
+ "vevo2": {"style": "sentence", "delivery": False, "max_changes": None},
+ "firered_audio": {"style": "instructions", "delivery": True, "max_changes": 5},
+}
+
+
+def _wav(frames):
+ with wave.open(buf := io.BytesIO(), "wb") as w:
+ w.setparams((1, 2, 24000, 0, "NONE", "not compressed"))
+ w.writeframes(b"\x01\x00" * frames)
+ return buf.getvalue()
+
+
+# fmt: off
+def _model(family, options = ()):
+ folder, filename = _NAMES[family]
+ policy = acm.family_policy(family, None, [f"{AUDIO_CPP_REPO}/{folder}", folder, filename])
+ variant = AudioCppVariant("Q8_0", (RepoFile(f"{folder}/{filename}", 1),), f"{folder}/{filename}")
+ return AudioCppModel(
+ id = f"{AUDIO_CPP_REPO}/{folder}", repo_id = AUDIO_CPP_REPO, folder = folder, display_name = folder,
+ family = family, server_task = policy.default_server_task, variant = variant, variants = (variant,),
+ default_variant = "Q8_0", options = tuple(options), task = policy.task, speaks = policy.speaks,
+ clone = policy.clone, edit = policy.edit,
+ )
+# fmt: on
+
+
+class _Recorder:
+ """A fake server: each call answers with a distinct WAV (FireRed's JSON shape, with a ``text``
+ that is not the edited transcript), and checks a chained call reads the previous answer."""
+
+ model_id, backend, cancel = MODEL_ID, "cpu", None
+ alive = staticmethod(lambda: True)
+ stop = staticmethod(lambda: None)
+
+ def __init__(self, model):
+ self.model = model
+ self.field = model.edit.source_field if model.edit else None
+ self.calls, self.answers, self.step_files = [], [], []
+
+ def post_json(self, path, payload, **_kwargs):
+ body = json.loads(json.dumps(payload))
+ self.calls.append((path, body))
+ read = body.get(self.field) if self.field else None
+ if read and re.search(r"step\d+\.wav$", str(read)):
+ assert Path(read).read_bytes() == self.answers[-1]
+ self.step_files.append(read)
+ answer = _wav(240 * (len(self.calls) + 1))
+ self.answers.append(answer)
+ if self.cancel:
+ self.cancel.set()
+ if path == "/v1/audio/speech":
+ return "audio/wav", answer
+ text = "Okay, I'm Cemo and what you just heard wasn't a voice."
+ out = {"text": text, "audio": base64.b64encode(answer).decode(), "sample_rate": 24000}
+ return "application/json", json.dumps(out).encode()
+
+
+def started(monkeypatch, tmp_path):
+ """Records the task each server start asks for, and keeps temp files under tmp_path."""
+ starts = []
+ monkeypatch.setattr(audio_cpp_files, "materialize", lambda m: str(tmp_path / "m.gguf"))
+ start = lambda served, path, **_k: starts.append(served.server_task) or _Recorder(served)
+ monkeypatch.setattr(acb.AudioCppServer, "start", start)
+ (tmp_path / "tmp").mkdir()
+ monkeypatch.setattr("tempfile.tempdir", str(tmp_path / "tmp"))
+ return starts
+
+
+def _backend(model, server = None):
+ backend = acb.AudioCppBackend()
+ backend._server, backend._model = server or _Recorder(model), model
+ backend.models = {model.id: {"is_audio": True}}
+ backend.active_model_name = model.id
+ return backend
+
+
+def _edit(backend, edit, **kwargs):
+ kwargs = {
+ "text": EDITED,
+ "reference_text": ORIGINAL,
+ "audio_inputs": {"source": SOURCE},
+ **kwargs,
+ }
+ return backend.generate_audio_response(workflow = "edit", edit = edit, **kwargs)
+
+
[email protected]("case", _CASES, ids = [c["name"] for c in _CASES])
+def test_the_runtime_bodies_match_the_shared_fixture(case, started, tmp_path):
+ backend = _backend(_model(case["family"]))
+ run = case["run_body"]
+ reference_text = run["inputs"].get("reference_text")
+ options = case["input"]["advanced"] or None
+ kwargs = {"text": run["text"], "reference_text": reference_text, "audio_options": options}
+ wav, rate = _edit(backend, run["edit"], **kwargs)
+ server = backend._server
+ assert rate == 24000 and wav == server.answers[-1]
+ names = {MODEL_ID: "<model>", SOURCE: "<recording>"}
+ names |= {p: f"<result of call {i}>" for i, p in enumerate(server.step_files, 1)}
+ swap = lambda body: {k: names.get(v, v) if isinstance(v, str) else v for k, v in body.items()}
+ assert [{"path": path, "body": swap(body)} for path, body in server.calls] == case["runtime"]
+ assert list((tmp_path / "tmp").iterdir()) == []
+
+
+def test_a_firered_chain_reads_each_previous_output_and_a_cancel_stops_it(started):
+ backend = _backend(_model("firered_audio"))
+ instructions = [*TWO, "Delete 'Okay,'."]
+ _edit(backend, {"mode": "words", "instructions": instructions}, text = EDITED[6:], seed = 7)
+ server = backend._server
+ assert server.calls[0][1]["audio"] == SOURCE
+ assert [Path(p).name for p in server.step_files] == ["step1.wav", "step2.wav"]
+ assert not Path(server.step_files[0]).parent.exists()
+ sent = [(body["seed"], body["options"]["instruction"]) for _path, body in server.calls]
+ assert sent == [("7", i) for i in instructions]
+ server = _Recorder(_model("firered_audio"))
+ server.cancel = cancel = threading.Event()
+ with pytest.raises(AudioGenerationCancelledError):
+ _edit(
+ _backend(server.model, server),
+ {"mode": "words", "instructions": TWO},
+ cancel_event = cancel,
+ )
+ assert len(server.calls) == 1
+
+
+def test_vevo2_edits_in_an_s2s_session_and_firered_in_its_clone_session(started):
+ backend = _backend(_model("vevo2"))
+ _edit(backend, {"mode": "words"})
+ _edit(backend, {"mode": "words"}, reference_text = None)
+ assert started == ["s2s"]
+ body = backend._server.calls[-1][1]
+ assert "reference_text" not in body and body["target_text"] == EDITED
+ backend.generate_audio_response(EDITED, workflow = "clone", audio_inputs = {"reference": REF})
+ assert started == ["s2s", "tts"]
+ ((path, body),) = backend._server.calls
+ assert path == "/v1/audio/speech" and body["voice_ref"] == REF
+ backend = _backend(_model("firered_audio"))
+ before = backend._server
+ _edit(backend, {"mode": "words", "instructions": TWO[1:]})
+ assert started == ["s2s", "tts"] and backend._server is before
+
+
+# fmt: off
+DOTS_OPTIONS = (
+ {"name": "template_name", "type": "enum", "values": ["tts", "edit"]},
+ {"name": "source_text", "type": "string"},
+ {"name": "num_inference_steps", "type": "int", "min": 1, "max": 64},
+ {"name": "use_xvector", "type": "bool"},
+)
+FIRERED_OPTIONS = (
+ {"name": "template_name", "type": "enum", "values": ["semantic_edit", "acoustic_edit"]},
+ {"name": "num_inference_steps", "type": "int", "min": 1},
+)
+CLAIMED = {name: "ignored" for name in ("source_text", "target_text", "instruction")}
+
+
[email protected]("family, options, edit, sent, expected", [
+ ("dots_tts", DOTS_OPTIONS, {"mode": "words", "markup": "x"},
+ {"template_name": "tts", **CLAIMED, "num_inference_steps": 10.0, "use_xvector": True, "not_in_the_spec": 1},
+ {"template_name": "edit", "num_inference_steps": "10", "use_xvector": "true"}),
+ ("firered_audio", FIRERED_OPTIONS, {"mode": "delivery", "speed": 0.5},
+ {"template_name": "semantic_edit", "num_inference_steps": 4},
+ {"template_name": "acoustic_edit", "instruction": "adjust the speed to 0.5x", "num_inference_steps": "4"}),
+])
+# fmt: on
+def test_advanced_options_leave_out_the_claimed_ones_and_unknown_ones(
+ started, family, options, edit, sent, expected
+):
+ backend = _backend(_model(family, options))
+ _edit(backend, edit, audio_options = sent)
+ assert backend._server.calls[0][1]["options"] == expected
+
+
+def test_a_saved_voice_clones_on_speak_and_an_edit_never_clones(started, monkeypatch):
+ backend = _backend(_model("voxcpm2"))
+ backend.generate_audio_response("Hi.", workflow = "speak", audio_inputs = {"reference": REF})
+ ((path, body),) = backend._server.calls
+ assert path == "/v1/audio/speech" and body["voice_ref"] == REF
+ # A model that cannot edit refuses before any call.
+ with pytest.raises(RuntimeError, match = "cannot edit speech"):
+ _edit(backend, {"mode": "words"})
+ assert len(backend._server.calls) == 1 and started == []
+ # An edit carries audio_inputs too; that must not make it a clone.
+ no_clone = staticmethod(lambda *_a, **_k: pytest.fail("an edit took the clone path"))
+ monkeypatch.setattr(acb.AudioCppBackend, "_generate_clone", no_clone)
+ for family, edit in (
+ ("firered_audio", {"mode": "words", "instructions": TWO[1:]}),
+ ("dots_tts", {"mode": "words", "markup": 'a <sub targ="robot">human</sub> voice.'}),
+ ):
+ backend = _backend(_model(family))
+ _edit(backend, edit)
+ ((path, body),) = backend._server.calls
+ assert path == "/v1/tasks/run" and "voice_ref" not in body
+
+
+def test_status_fields_carry_the_edit_rules():
+ for family in _NAMES:
+ fields = acb.model_info_fields(_model(family))
+ assert fields["audio_edit"] == _RULES.get(family), family
+ assert ("edit" in fields["audio_workflows"]) == (family in _RULES), family
+ assert acb.model_info_fields(_model("dots_tts"))["audio_workflows"] == ["speak", "edit"]
+ assert acb.model_info_fields(_model("vevo2"))["audio_workflows"] == ["clone", "edit"]
+
+
+# fmt: off
+def test_markup_sides_and_check_markup():
+ assert audio_edit.markup_sides(MARKUP) == (ORIGINAL, EDITED)
+ assert audio_edit.markup_sides("what you <del>just</del> heard <ins>really</ins> wasn't") == (
+ "what you just heard wasn't", "what you heard really wasn't")
+ # Anything but the three tags, or a tag left open, is refused.
+ for bad in ("a <b>human</b> voice", 'a <sub targ="robot">human voice', 'a <sub targ="x" onload="y">human</sub>', "a > b"):
+ assert audio_edit.markup_sides(bad) is None
+ assert audio_edit.check_markup(bad, ORIGINAL, EDITED) == audio_edit.MISMATCH
+ for markup, original, edited, problem in (
+ (MARKUP, ORIGINAL, EDITED, None),
+ (ORIGINAL, ORIGINAL, ORIGINAL, audio_edit.NO_CHANGE),
+ (MARKUP, ORIGINAL, ORIGINAL, audio_edit.MISMATCH),
+ (MARKUP, "Something else.", EDITED, audio_edit.MISMATCH),
+ ):
+ assert audio_edit.check_markup(markup, original, edited) == problem
+
+
+def test_check_instructions():
+ unknown, mismatch = audio_edit.UNKNOWN_INSTRUCTION, audio_edit.MISMATCH
+ for instructions, edited, problem in (
+ (TWO, EDITED, None),
+ (["Insert 'really' before 'wasn't'."], ORIGINAL.replace("heard", "heard really"), None),
+ (["Delete 'Okay,'."], ORIGINAL[6:], None),
+ ([], EDITED, audio_edit.NO_CHANGE),
+ (["Replace 'human' with 'robot'."] * 6, EDITED,
+ "FireRedAudio applies at most 5 changes. Make fewer changes, or use DotTTS Edit."),
+ (["Replace 'human' with 'robot'"], EDITED, unknown),
+ (["replace 'human' with 'robot'."], EDITED, unknown),
+ (["Speak faster."], EDITED, unknown),
+ (["Replace 'human' with 'robot'. Replace 'Cemo' with 'Sam'.x"], EDITED, unknown),
+ # Words that are not in the transcripts.
+ (["Replace 'alien' with 'robot'."], EDITED, mismatch),
+ (["Replace 'human' with 'cat'."], EDITED, mismatch),
+ ):
+ check = audio_edit.check_instructions(instructions, ORIGINAL, edited, 5, "FireRedAudio")
+ assert check == problem, instructions
+
+
+def test_delivery_instructions():
+ for speed, pitch, expected in (
+ (1.5, None, ["adjust the speed to 1.5x"]),
+ (2.0, None, ["adjust the speed to 2x"]),
+ (1.0, 3, ["shift the pitch by 3 steps"]),
+ (0.5, 2, ["adjust the speed to 0.5x", "shift the pitch by 2 steps"]),
+ (None, None, []),
+ ):
+ assert audio_edit.delivery_instructions(speed, pitch) == expected
+
+
+def test_request_problem_per_style():
+ dots, vevo, firered = _RULES.values()
+ words, no_change = {"mode": "words"}, audio_edit.NO_CHANGE
+ for rules, edit, text, reference, problem in (
+ (dots, words, EDITED, ORIGINAL, "DotTTS-Edit needs the marked-up changes."),
+ (dots, words, ORIGINAL, ORIGINAL, no_change),
+ (dots, {"mode": "delivery", "speed": 1.5}, EDITED, ORIGINAL, "Delivery changes need FireRedAudio."),
+ (vevo, words, ORIGINAL, ORIGINAL, no_change),
+ (vevo, words, EDITED, None, None),
+ (firered, {"mode": "delivery", "speed": 1.0}, EDITED, ORIGINAL, "Pick a speed or a pitch change."),
+ (firered, {"mode": "delivery", "pitch_steps": 3}, ORIGINAL, ORIGINAL, None),
+ (firered, words, EDITED, ORIGINAL, no_change),
+ ):
+ assert audio_edit.request_problem(rules, edit, text, reference, "DotTTS-Edit") == problem
+# fmt: on
diff --git a/studio/backend/tests/test_audio_edit_route.py b/studio/backend/tests/test_audio_edit_route.py
new file mode 100644
index 00000000000..cd35d4836d1
--- /dev/null
+++ b/studio/backend/tests/test_audio_edit_route.py
@@ -0,0 +1,261 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""POST /audio/run with workflow edit, on test_audio_run_route's stub orchestrator: never a client
+path in, never a server path in a response or sidecar."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+from core.inference import audio_gallery
+from routes import inference
+from utils.account_context import run_as
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+from test_audio_run_route import ( # noqa: E402,F401 - fixtures
+ ALICE,
+ BOB,
+ _client,
+ _input,
+ _inputs_root,
+ _sidecar,
+ _voice,
+ _wav,
+ isolated,
+ stub,
+)
+
+ORIGINAL = "Okay, I'm Cemo and what you just heard wasn't a human voice."
+EDITED = "Okay, I'm Sam and what you just heard wasn't a robot voice."
+MARKUP = (
+ "Okay, I'm <sub targ=\"Sam\">Cemo</sub> and what you just heard wasn't a "
+ '<sub targ="robot">human</sub> voice.'
+)
+MISMATCH = "The changes do not match the transcript. Check ① and ② again."
+CANNOT = "Load a model that can edit speech."
+_MODELS = {
+ "dots": ("DotTTS-Edit-GGUF", ["speak", "edit"], ("markup", False, None)),
+ "firered": ("FireRedAudio-GGUF", ["clone", "edit"], ("instructions", True, 5)),
+ "mf": ("DotTTS-MF-GGUF", ["speak"], None),
+}
+
+
+def _load(stub, name = "dots"):
+ folder, workflows, rules = _MODELS[name]
+ info = {
+ "is_audio": True,
+ "audio_type": "audiocpp_tts",
+ "audio_workflows": workflows,
+ "audio_reference_text": None,
+ "audio_required_inputs": [],
+ "audio_clone": None,
+ "audio_edit": rules and dict(zip(("style", "delivery", "max_changes"), rules)),
+ }
+ return stub["use"](f"audio-cpp/audio.cpp-gguf/{folder}", info)
+
+
+def _edit(
+ client,
+ source = None,
+ edit = None,
+ **body,
+):
+ if isinstance(source, str):
+ source = {"input_id": source}
+ payload = {
+ "workflow": "edit",
+ "text": EDITED,
+ "inputs": {"reference_text": ORIGINAL, **({"source": source} if source else {})},
+ "edit": edit or {"mode": "words", "markup": MARKUP},
+ **body,
+ }
+ return client.post("/api/inference/audio/run", json = payload)
+
+
+_SRC = {"source": {"input_id": "a" * 32}}
+
+
+# fmt: off
+ {"inputs": {"source": {"path": "/etc/passwd"}}},
+ {"inputs": {"source": {"input_id": "a" * 32, "source_audio": "/etc/passwd"}}},
+ {"inputs": _SRC, "edit": {"mode": "words", "path": "/x"}},
+ {"inputs": _SRC, "edit": {"source_audio": "/x.wav"}},
+ {"inputs": _SRC, "edit": {"mode": "rewrite"}},
+ {"inputs": _SRC, "edit": {"speed": 3.0}},
+ {"inputs": _SRC, "edit": {"instructions": ["x"] * 9}},
+ {"workflow": "clone", "inputs": {"reference": {"input_id": "a" * 32}}},
+ {"inputs": _SRC, "edit": None},
+ {"inputs": {**_SRC, "reference": {"input_id": "a" * 32}}},
+ {"options": {"source_audio": "/etc/passwd"}},
+])
+# fmt: on
+def test_client_paths_and_misplaced_edit_fields_are_422(stub, body):
+ _load(stub)
+ payload = {"workflow": "edit", "text": EDITED, "edit": {"mode": "words", "markup": MARKUP}}
+ payload.update(body)
+ if payload["edit"] is None:
+ payload.pop("edit")
+ with _client(ALICE) as client:
+ response = client.post("/api/inference/audio/run", json = payload)
+ assert response.status_code == 422, response.text
+ assert stub["backend"].calls == []
+
+
+W = {"mode": "words"}
+
+
+# fmt: off
[email protected]("model, source, edit, text, detail", [
+ ("dots", None, None, EDITED, "Add a recording to edit."),
+ ("dots", "voice", None, EDITED, "Pick a recording or a history clip to edit."),
+ ("dots", 31.0, None, EDITED, "Edit works on recordings up to 30 s. Record or upload a shorter take."),
+ ("mf", 1.0, None, EDITED, CANNOT),
+ # A GGUF speech model on llama.cpp has no workflows to read.
+ ("llama", 1.0, None, EDITED, CANNOT),
+ ("dots", 1.0, {**W, "markup": MARKUP.replace("Sam", "Sammy")}, EDITED, MISMATCH),
+ ("dots", 1.0, {**W, "markup": "Okay <b>x</b>"}, EDITED, MISMATCH),
+ ("dots", 1.0, W, EDITED, "DotTTS-Edit needs the marked-up changes."),
+ ("dots", 1.0, W, ORIGINAL, "Change at least one word."),
+ ("dots", 1.0, {"mode": "delivery", "speed": 1.5}, EDITED, "Delivery changes need FireRedAudio."),
+ ("firered", 1.0, {**W, "instructions": ["Replace 'human' with 'robot'."] * 6}, EDITED,
+ "FireRedAudio applies at most 5 changes. Make fewer changes, or use DotTTS Edit."),
+ ("firered", 1.0, {**W, "instructions": ["Make it sound happy."]}, EDITED,
+ "That change is not one FireRedAudio understands."),
+ ("firered", 1.0, W, EDITED, "Change at least one word."),
+ ("firered", 1.0, {"mode": "delivery", "speed": 1.0}, EDITED, "Pick a speed or a pitch change."),
+])
+# fmt: on
+def test_edits_the_route_refuses_are_400(stub, monkeypatch, model, source, edit, text, detail):
+ _load(stub, "mf" if model == "llama" else model)
+ if model == "llama":
+ from utils.audio_tokens import GGUF_TTS_AUDIO_TYPES
+ class _Llama:
+ is_loaded = _is_audio = True
+ _audio_type = sorted(GGUF_TTS_AUDIO_TYPES)[0]
+ model_identifier = "someone/tts-GGUF"
+ context_length = None
+
+ def generate_audio_response(self, **kwargs):
+ raise AssertionError("llama.cpp was asked to edit")
+
+ monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: _Llama())
+ if source == "voice":
+ source = {"voice_id": _voice(ALICE, _input(ALICE))["id"]}
+ elif source:
+ source = _input(ALICE, source)
+ with _client(ALICE) as client:
+ response = _edit(client, source, edit, text = text)
+ assert response.status_code == 400, response.text
+ assert response.json()["detail"] == detail
+ assert stub["backend"].calls == []
+
+
+def test_an_edit_run_hands_the_worker_paths_and_keeps_the_source(stub, tmp_path):
+ backend = _load(stub)
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ response = _edit(client, input_id, options = {"num_inference_steps": 10})
+ assert response.status_code == 200, response.text
+ body = response.json()
+ assert [client.get(clip["url"]).status_code for clip in body["clips"]] == [200, 200]
+ (call,) = backend.calls
+ source = Path(call["audio_inputs"]["source"])
+ assert source.is_absolute() and source.parent == _inputs_root(tmp_path)
+ assert source.name.startswith(f"{input_id}.24000.mono")
+ assert "speed" not in call and body["audio"] is None
+ output, original = body["clips"]
+ out_meta, src_meta = (json.loads(_sidecar(tmp_path, c["id"])) for c in (output, original))
+ # fmt: off
+ assert (call["workflow"], call["text"], call["reference_text"], call["edit"]) == (
+ "edit", EDITED, ORIGINAL, {"mode": "words", "markup": MARKUP})
+ assert [(c["role"], c["workflow"]) for c in body["clips"]] == [("output", "edit"), ("source", "edit")]
+ assert (out_meta["source_clip_id"], out_meta["prompt"], out_meta["reference_name"]) == (
+ original["id"], EDITED, "me.webm")
+ assert out_meta["settings"] == {
+ "mode": "words", "changes": 2, "original_text": ORIGINAL, "options": {"num_inference_steps": 10}}
+ assert (src_meta["role"], src_meta["model"], src_meta["prompt"]) == ("source", "Recording", ORIGINAL)
+ # fmt: on
+ for clip in body["clips"]:
+ text = _sidecar(tmp_path, clip["id"])
+ assert str(tmp_path) not in text and "inputs" not in text
+ assert str(tmp_path) not in response.text
+
+
+def test_one_source_clip_per_upload_kept_while_an_edit_plays_it(stub, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_AUDIO_GALLERY_MAX_CLIPS", "2")
+ _load(stub)
+ input_id = _input(ALICE)
+ with _client(ALICE) as client:
+ first = _edit(client, input_id).json()["clips"]
+ output, source = _edit(client, input_id).json()["clips"]
+ assert first[1]["id"] == source["id"] and first[0]["id"] != output["id"]
+ # The cap pruned the first output, not the source the second still plays.
+ assert client.get(source["url"]).status_code == 200
+ assert sum(c.get("role") == "source" for c in run_as(ALICE, audio_gallery.list_audio)) == 1
+ clip = f"/api/inference/audio/gallery/{output['id']}"
+ assert client.patch(clip, json = {"archived": True}).status_code == 200
+ assert client.delete("/api/inference/audio/gallery?workflow=edit").status_code == 200
+ assert client.get(source["url"]).status_code == 200
+ # Once nothing plays it, the source goes with the next clear.
+ client.patch(clip, json = {"archived": False})
+ assert client.delete("/api/inference/audio/gallery?workflow=edit").status_code == 200
+ assert client.get(source["url"]).status_code == 404
+
+
+def test_a_history_clip_source_makes_no_copy(stub, tmp_path):
+ _load(stub, "firered")
+ meta = {"prompt": ORIGINAL, "model": "m", "audio_type": "audiocpp_tts", "workflow": "clone"}
+ meta |= {"sample_rate": 24000, "duration_s": 0.1, "created_at": "2026-10-02T00:00:00Z"}
+ clip = run_as(ALICE, audio_gallery.save, _wav(), meta)
+ delivery = {"mode": "delivery", "speed": 1.5, "pitch_steps": 3}
+ with _client(ALICE) as client:
+ response = _edit(client, {"clip_id": clip["id"]}, delivery, text = ORIGINAL)
+ assert response.status_code == 200, response.text
+ (output,) = response.json()["clips"]
+ meta = json.loads(_sidecar(tmp_path, output["id"]))
+ assert meta["source_clip_id"] == clip["id"]
+ assert meta["settings"] == {**delivery, "changes": 2, "original_text": ORIGINAL, "options": {}}
+ assert len(run_as(ALICE, audio_gallery.list_audio)) == 2
+
+
+def test_another_accounts_recording_is_404(stub):
+ _load(stub)
+ input_id = _input(ALICE)
+ with _client(BOB) as client:
+ assert _edit(client, input_id).status_code == 404
+ assert stub["backend"].calls == []
+
+
+def test_the_worker_carries_the_edit():
+ import queue
+ import threading
+
+ from core.inference.worker import _handle_generate_audio
+
+ seen = []
+
+ class _Backend:
+ def generate_audio_response(self, **kwargs):
+ seen.append(kwargs)
+ return _wav(), 24000
+
+ edit = {"mode": "words", "instructions": ["Replace 'human' with 'robot'."]}
+ source = {"source": "/abs/source.wav"}
+ for cmd in (
+ {"text": EDITED, "workflow": "edit", "audio_inputs": source, "edit": edit},
+ {"text": "hi"},
+ ):
+ _handle_generate_audio(
+ _Backend(), {"request_id": "r", **cmd}, queue.Queue(), threading.Event()
+ )
+ assert seen[0]["edit"] == edit and seen[0]["audio_inputs"] == source
+ assert "edit" not in seen[1]
diff --git a/studio/backend/tests/test_audio_gallery.py b/studio/backend/tests/test_audio_gallery.py
index 22f6c3ff51d..0a049ad5f16 100644
--- a/studio/backend/tests/test_audio_gallery.py
+++ b/studio/backend/tests/test_audio_gallery.py
@@ -783,6 +783,38 @@ def test_a_clone_clip_keeps_its_workflow_and_run_fields_and_survives_a_speak_cle
assert [r["id"] for r in gallery.list_audio()] == [clone["id"]]
+def test_an_edit_keeps_its_workflow_and_a_scoped_clear_takes_its_source_clips():
+ from models.inference import AudioGalleryItem
+ from routes.inference import _persist_tts_clip, clear_gallery_audio
+
+ speech = gallery.save(_wav(), _meta())
+ source = gallery.save(
+ _wav(),
+ _meta(
+ model = "Recording",
+ audio_type = "recording",
+ workflow = "edit",
+ role = "source",
+ ),
+ )
+ edited = _persist_tts_clip(
+ _wav(),
+ 24000,
+ "edited",
+ "m",
+ "audiocpp_tts",
+ {"role": "output", "source_clip_id": source["id"]},
+ "edit",
+ )
+ listed = {r["id"]: r for r in gallery.list_audio()}
+ assert listed[source["id"]]["workflow"] == listed[edited["id"]]["workflow"] == "edit"
+ assert AudioGalleryItem(**listed[edited["id"]]).source_clip_id == source["id"]
+ assert gallery.set_flags(edited["id"], pinned = True)["workflow"] == "edit"
+ cleared = asyncio.run(clear_gallery_audio(workflow = "edit", current_subject = "tester"))
+ assert cleared == {"removed": 2}
+ assert [r["id"] for r in gallery.list_audio()] == [speech["id"]]
+
+
def test_the_inputs_and_voices_folders_never_list_as_clips():
gallery.save(_wav(), _meta())
for folder in ("inputs", "voices"):
@@ -795,6 +827,18 @@ def test_the_inputs_and_voices_folders_never_list_as_clips():
assert (gallery.gallery_dir() / "inputs" / "abc.wav").is_file()
+def test_only_a_hidden_edit_source_is_kept_for_the_clip_that_plays_it(monkeypatch):
+ reference = gallery.save(_wav(), _meta(workflow = "speak"))
+ gallery.save(_wav(), _meta(workflow = "clone", source_clip_id = reference["id"]))
+ # A visible clip another clip was made from still goes with its own page's Clear and the cap.
+ assert gallery.clear(workflow = "speak") == 1
+ monkeypatch.setenv("UNSLOTH_AUDIO_GALLERY_MAX_CLIPS", "1")
+ reference = _save_with_mtime("reference", 1000)
+ clone = gallery.save(_wav(), _meta(workflow = "clone", source_clip_id = reference["id"]))
+ gallery._prune_to_cap()
+ assert [r["id"] for r in gallery.list_audio()] == [clone["id"]]
+
+
def _stem_meta(**over):
return _meta(
audio_type = "audiocpp_sep",
diff --git a/studio/backend/tests/test_audio_run_route.py b/studio/backend/tests/test_audio_run_route.py
index 81b16c5bd79..a4507f10159 100644
--- a/studio/backend/tests/test_audio_run_route.py
+++ b/studio/backend/tests/test_audio_run_route.py
@@ -446,6 +446,17 @@ def test_the_orchestrator_command_carries_paths_not_bytes(monkeypatch):
assert {k: sent[0][k] for k in fields} == fields
assert all(not isinstance(v, (bytes, bytearray)) for v in sent[0].values())
assert not CLONE_FIELDS & set(sent[1])
+ sent.clear()
+ edit = {"mode": "words", "instructions": ["Replace 'human' with 'robot'."]}
+ orchestrator.generate_audio_response(
+ "edited", workflow = "edit", audio_inputs = {"source": reference}, edit = edit
+ )
+ (cmd,) = sent
+ assert (cmd["workflow"], cmd["audio_inputs"], cmd["edit"]) == (
+ "edit",
+ {"source": reference},
+ edit,
+ )
def test_the_worker_forwards_clone_fields_only_when_present():
diff --git a/studio/backend/tests/test_audio_workflows.py b/studio/backend/tests/test_audio_workflows.py
index 29786d98194..3c549b6b35d 100644
--- a/studio/backend/tests/test_audio_workflows.py
+++ b/studio/backend/tests/test_audio_workflows.py
@@ -105,7 +105,7 @@ def test_workflow_for_audio_type_is_speak_or_music():
assert aw.workflow_for_audio_type("snac") == "speak"
assert aw.workflow_for_audio_type("unknown") == "speak"
assert aw.workflow_for_audio_type(None) == "speak"
- assert aw.AUDIO_WORKFLOW_IDS == ("speak", "clone", "music", "separate", "transcribe")
+ assert aw.AUDIO_WORKFLOW_IDS == ("speak", "clone", "edit", "music", "separate", "transcribe")
def test_workflow_ids_match_the_frontend_order():
@@ -130,6 +130,10 @@ def test_status_carries_the_clone_fields():
("VoxCPM2-GGUF", "voxcpm2", ["speak", "clone"]),
("Qwen3-TTS-12Hz-0.6B-Base-GGUF", "qwen3_tts", ["clone"]),
("Kokoro-82M-GGUF", "kokoro_tts", ["speak"]),
+ ("DotTTS-Edit-GGUF", "dots_tts", ["speak", "edit"]),
+ ("DotTTS-MF-GGUF", "dots_tts", ["speak"]),
+ ("Vevo2-GGUF", "vevo2", ["clone", "edit"]),
+ ("FireRedAudio-GGUF", "firered_audio", ["clone", "edit"]),
],
)
def test_inventory_rows_of_clone_families_list_clone(tmp_path, folder, family, workflows):
diff --git a/studio/backend/tests/test_pr7699_worker_audio_mirror.py b/studio/backend/tests/test_pr7699_worker_audio_mirror.py
index 9a52cf0c2e0..56e294eb739 100644
--- a/studio/backend/tests/test_pr7699_worker_audio_mirror.py
+++ b/studio/backend/tests/test_pr7699_worker_audio_mirror.py
@@ -127,6 +127,7 @@ def test_audio_cpp_workflow_fields_cross_the_worker_and_the_parent(monkeypatch):
"audio_reference_text": "required",
"audio_required_inputs": ["instruct"],
"audio_clone": {"reference_text": "required", "emotion_audio": False},
+ "audio_edit": {"style": "instructions", "delivery": True, "max_changes": 5},
"audio_music": {"modes": [{"id": "song", "variations": None}]},
# The /audio/run route sizes CPU waits from it.
"audio_cpp_backend": "cpu",
diff --git a/studio/frontend/src/features/audio/api.ts b/studio/frontend/src/features/audio/api.ts
index 90cf8123418..84571a7bf93 100644
--- a/studio/frontend/src/features/audio/api.ts
+++ b/studio/frontend/src/features/audio/api.ts
@@ -110,8 +110,10 @@ export interface AudioGalleryClip {
reference_name?: string | null;
/** Clips one run made together share it (a separation's stems, music takes); null for one clip. */
group_id?: string | null;
- /** The stem a clip holds (vocals, drums, ...). */
+ /** A stem's name (vocals, drums, ...) or an edit's run part ("output" or "source"). */
role?: string | null;
+ /** The original an edit changed. */
+ source_clip_id?: string | null;
/** The run's settings, e.g. a separation's stem list. */
settings?: Record<string, unknown> | null;
}
@@ -218,7 +220,7 @@ export async function deleteAudioClip(id: string): Promise<void> {
}
export async function clearAudioGallery(
- workflow?: "speak" | "clone" | "music" | "separate",
+ workflow?: "speak" | "clone" | "edit" | "music" | "separate",
): Promise<number> {
const query = workflow ? `?workflow=${workflow}` : "";
const response = await authFetch(`/api/inference/audio/gallery${query}`, {
diff --git a/studio/frontend/src/features/audio/audio-cpp-catalog.ts b/studio/frontend/src/features/audio/audio-cpp-catalog.ts
index 18ae931ce93..bc6952c7634 100644
--- a/studio/frontend/src/features/audio/audio-cpp-catalog.ts
+++ b/studio/frontend/src/features/audio/audio-cpp-catalog.ts
@@ -22,6 +22,7 @@ export type AudioCppTask = "tts" | "music" | "asr" | "sep";
export type AudioCppWorkflow =
| "speak"
| "clone"
+ | "edit"
| "music"
| "separate"
| "transcribe";
@@ -67,6 +68,9 @@ export const AUDIO_CPP_MODELS: readonly AudioCppModel[] = [
{ id: folder("CosyVoice3-GGUF"), task: "tts", workflows: ["clone"] },
{ id: folder("Qwen3-TTS-12Hz-1.7B-CustomVoice-GGUF"), task: "tts" },
{ id: folder("Qwen3-TTS-12Hz-1.7B-VoiceDesign-GGUF"), task: "tts" },
+ { id: folder("DotTTS-Edit-GGUF"), task: "tts", workflows: ["speak", "edit"] },
+ { id: folder("Vevo2-GGUF"), task: "tts", workflows: ["clone", "edit"] },
+ { id: folder("FireRedAudio-GGUF"), task: "tts", workflows: ["clone", "edit"] },
{ id: "audio-cpp/MiniMax-Music3-GGUF", task: "music" },
{ id: "audio-cpp/Yue2-3B-GGUF", task: "music" },
{ id: folder("ACE-Step1.5-GGUF"), task: "music" },
diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsx
index 6602fbfacd4..fba5c945b52 100644
--- a/studio/frontend/src/features/audio/audio-page.tsx
+++ b/studio/frontend/src/features/audio/audio-page.tsx
@@ -55,6 +55,7 @@ import { galleryCache, useAudioGallery, useWorkflowHistory } from "./hooks/use-a
import { useAudioHandoff } from "./hooks/use-audio-handoff";
import { useAudioModelSlot } from "./hooks/use-audio-model-slot";
import { useCloneGeneration } from "./hooks/use-clone-generation";
+import { useEditGeneration } from "./hooks/use-edit-generation";
import { useSeparateGeneration } from "./hooks/use-separate-generation";
import { useSpeechGeneration } from "./hooks/use-speech-generation";
import { useSttSidecar } from "./hooks/use-stt-sidecar";
@@ -66,6 +67,12 @@ import {
adoptReference,
clonePageModels,
} from "./pages/clone-page";
+import {
+ EditFooter,
+ EditOutput,
+ EditRail,
+ editPageModels,
+} from "./pages/edit-page";
import { useMusicGeneration } from "./hooks/use-music-generation";
import { MusicOutput, MusicRail, musicPageModels } from "./pages/music-page";
import {
@@ -91,6 +98,7 @@ import {
} from "./transcribe-languages";
import { type AudioPickerRow, audioRowMatchesWorkflow } from "./picker-filter";
import { useAudioCloneStore } from "./stores/audio-clone-store";
+import { useAudioEditStore } from "./stores/audio-edit-store";
import { useAudioWorkspaceStore } from "./stores/audio-workspace-store";
import { AudioToolPanels } from "./tools/tool-panel-host";
import {
@@ -510,6 +518,27 @@ export function AudioPage({
audioOptionValues,
sttRepo: cloneSttRepo,
});
+ const edit = useEditGeneration({
+ status,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ generateAbort,
+ setMode,
+ setAdvancedOpen,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ refreshGallery,
+ selectClip: selectGeneratedClip,
+ setFallbackClip,
+ setSelectedId,
+ pendingTranscribeRelease,
+ replayQueuedTtsPick,
+ audioOptionSpecs,
+ audioOptionValues,
+ sttRepo: cloneSttRepo,
+ });
const separate = useSeparateGeneration({
status,
busyRef,
@@ -625,6 +654,7 @@ export function AudioPage({
const target = clipWorkflow(clip);
if (!transitionWorkflow(target)) return;
if (target === "clone") useAudioCloneStore.getState().setText(clip.prompt);
+ else if (target === "edit") useAudioEditStore.getState().setEdited(clip.prompt);
else setPrompt(clip.prompt);
},
[transitionWorkflow, setPrompt],
@@ -700,6 +730,12 @@ export function AudioPage({
label: `use ${model.name}`,
onClick: () => void pickRecommendedModel(model.id),
}));
+ const recommendedEditActions = editPageModels(MODELS_BY_MODE.speak, isMac)
+ .slice(0, 2)
+ .map((model) => ({
+ label: `use ${model.name}`,
+ onClick: () => void pickRecommendedModel(model.id),
+ }));
const recommendedSeparateActions = separatePageModels(
MODELS_BY_MODE.speak,
isMac,
@@ -723,18 +759,34 @@ export function AudioPage({
? "Load a music model to generate."
: ttsWorkflow === "clone"
? "Load a model that can clone a voice."
- : ttsWorkflow === "separate"
- ? "Load a model that can separate audio."
+ : ttsWorkflow === "edit"
+ ? "Load a model that can edit speech."
+ : ttsWorkflow === "separate"
+ ? "Load a model that can separate audio."
: "Load a speech model to generate.",
actions:
ttsWorkflow === "clone"
? [chooseModelAction, ...recommendedCloneActions]
- : ttsWorkflow === "separate"
- ? [chooseModelAction, ...recommendedSeparateActions]
+ : ttsWorkflow === "edit"
+ ? [chooseModelAction, ...recommendedEditActions]
+ : ttsWorkflow === "separate"
+ ? [chooseModelAction, ...recommendedSeparateActions]
: ttsWorkflow === "music"
? [chooseModelAction, ...recommendedMusicActions]
: [chooseModelAction],
}
+ : !pageModelLoaded && ttsWorkflow === "edit"
+ ? {
+ reason: "The loaded model cannot edit speech.",
+ actions: [
+ { label: "Choose a model that can edit", onClick: openSelector },
+ ...recommendedEditActions,
+ {
+ label: "open Speak",
+ onClick: () => transitionWorkflow("speak"),
+ },
+ ],
+ }
: !pageModelLoaded && ttsWorkflow === "separate"
? {
reason: "The loaded model cannot separate audio.",
@@ -804,6 +856,8 @@ export function AudioPage({
? separate.blocker
: ttsWorkflow === "clone"
? clone.blocker
+ : ttsWorkflow === "edit"
+ ? edit.blocker
: musicStudio
? music.blocker
: !prompt.trim() && !lyricsOptional
@@ -826,6 +880,8 @@ export function AudioPage({
? separate.generationError
: ttsWorkflow === "clone"
? clone.generationError
+ : ttsWorkflow === "edit"
+ ? edit.generationError
: musicStudio
? music.generationError
: generationError;
@@ -838,31 +894,34 @@ export function AudioPage({
}
: null;
const setCloneGenerationError = clone.setGenerationError;
+ const setEditGenerationError = edit.setGenerationError;
const setSeparateGenerationError = separate.setGenerationError;
const setMusicGenerationError = music.setGenerationError;
useEffect(() => {
setGenerationError(null);
setCloneGenerationError(null);
+ setEditGenerationError(null);
setSeparateGenerationError(null);
setMusicGenerationError(null);
}, [
status?.active_model,
setGenerationError,
setCloneGenerationError,
+ setEditGenerationError,
setSeparateGenerationError,
setMusicGenerationError,
]);
useEffect(() => {
if (busy === "generating") setGenerationError(null);
if (busy === "generating") setCloneGenerationError(null);
- if (busy === "generating") {
- setSeparateGenerationError(null);
- setMusicGenerationError(null);
- }
+ if (busy === "generating") setEditGenerationError(null);
+ if (busy === "generating") setSeparateGenerationError(null);
+ if (busy === "generating") setMusicGenerationError(null);
}, [
busy,
setGenerationError,
setCloneGenerationError,
+ setEditGenerationError,
setSeparateGenerationError,
setMusicGenerationError,
]);
@@ -1037,7 +1096,8 @@ export function AudioPage({
const generateShortcut = useRef<() => void>(() => {});
const handlePageGenerate =
ttsWorkflow === "separate" ? separate.handleGenerate :
- ttsWorkflow === "clone" ? clone.handleGenerate : handleGenerate;
+ ttsWorkflow === "clone" ? clone.handleGenerate :
+ ttsWorkflow === "edit" ? edit.handleGenerate : handleGenerate;
generateShortcut.current = () => {
if (canTranscribe) handleTranscribe();
else if (canGenerate) void handlePageGenerate();
@@ -1065,6 +1125,8 @@ export function AudioPage({
? musicPageModels(MODELS_BY_MODE.speak, isMac)
: ttsWorkflow === "clone"
? clonePageModels(MODELS_BY_MODE.speak, isMac)
+ : ttsWorkflow === "edit"
+ ? editPageModels(MODELS_BY_MODE.speak, isMac)
: ttsWorkflow === "separate"
? separatePageModels(MODELS_BY_MODE.speak, isMac)
: speakPageModels(MODELS_BY_MODE.speak, isMac)
@@ -1120,7 +1182,9 @@ export function AudioPage({
? "music"
: ttsWorkflow === "separate"
? "separate"
- : ttsWorkflow === "clone" ? "clone" : "tts",
+ : ttsWorkflow === "clone" || ttsWorkflow === "edit"
+ ? ttsWorkflow
+ : "tts",
status?.audio_type,
)
: ttsWorkflow === "separate"
@@ -1129,6 +1193,8 @@ export function AudioPage({
? "The loaded model separates audio. Open Separate to use it."
: ttsWorkflow === "clone"
? "The loaded model cannot clone a voice."
+ : ttsWorkflow === "edit"
+ ? "The loaded model cannot edit speech."
: musicGeneration
? "The loaded model makes music. Pick a speech model."
: ttsWorkflow === "speak"
@@ -1142,7 +1208,9 @@ export function AudioPage({
? "No separation model loaded."
: ttsWorkflow === "clone"
? "No voice cloning model loaded."
- : "No TTS model loaded."
+ : ttsWorkflow === "edit"
+ ? "No speech editing model loaded."
+ : "No TTS model loaded."
: sttSelected
? audioCapabilityLine("stt", sttReady ? "ready" : "loading")
: transcribeRepo
@@ -1182,7 +1250,8 @@ export function AudioPage({
additionalOnDeviceModels={
mode === "transcribe"
? sttOnDeviceModels
- : ttsWorkflow === "clone" || ttsWorkflow === "separate"
+ : // Trained speech checkpoints only speak (or make music); no other page loads them.
+ ttsWorkflow !== "speak" && ttsWorkflow !== "music"
? []
: trainedTtsModels.filter(
(model) =>
@@ -1381,6 +1450,10 @@ export function AudioPage({
historyClips={clips}
/>
</AudioActiveProvider>
+ ) : ttsWorkflow === "edit" ? (
+ <AudioActiveProvider value={active}>
+ <EditRail {...railProps} edit={edit} historyClips={clips} />
+ </AudioActiveProvider>
) : (
<SpeakRail {...railProps} />
);
@@ -1405,7 +1478,23 @@ export function AudioPage({
{mode === "speak" ? (
/* The scroll mask provides the fade; leave the footer unpainted to avoid dark-mode banding. */
<div className="relative z-10 flex shrink-0 justify-center px-10 pt-0.5 pb-4">
- {ttsWorkflow === "separate" ? (
+ {ttsWorkflow === "edit" ? (
+ <EditFooter
+ busy={busy}
+ generationPresentation={
+ generationPresentation && edit.phaseLabel && generationPresentation.canStop
+ ? { ...generationPresentation, status: `${edit.phaseLabel}…` }
+ : generationPresentation
+ }
+ handleStopGeneration={handleStopGeneration}
+ ttsLoaded={pageModelLoaded}
+ blocker={generateBlocker}
+ shortcutLabel={shortcutLabel}
+ error={generateFailure}
+ elapsedSeconds={elapsedSeconds}
+ edit={edit}
+ />
+ ) : ttsWorkflow === "separate" ? (
<SeparateFooter
busy={busy}
generationPresentation={generationPresentation}
@@ -1558,6 +1647,13 @@ export function AudioPage({
};
return ttsWorkflow === "music" ? (
<MusicOutput {...outputProps} modelReady={pageModelLoaded} />
+ ) : ttsWorkflow === "edit" ? (
+ <EditOutput
+ {...outputProps}
+ modelReady={pageModelLoaded}
+ recommendedModels={selectorModels}
+ onPickModel={pickRecommendedModel}
+ />
) : ttsWorkflow === "separate" ? (
<SeparateOutput
{...outputProps}
diff --git a/studio/frontend/src/features/audio/audio-run-request.ts b/studio/frontend/src/features/audio/audio-run-request.ts
index c93ef08bdb7..b01801cf95f 100644
--- a/studio/frontend/src/features/audio/audio-run-request.ts
+++ b/studio/frontend/src/features/audio/audio-run-request.ts
@@ -8,6 +8,11 @@ export const AUDIO_INPUT_MAX_BYTES = 200 * 1024 * 1024;
export const REFERENCE_MAX_SECONDS = 30;
+/** Longer is refused, not cut: a cut recording no longer matches its transcript. */
+export const EDIT_SOURCE_MAX_SECONDS = 30;
+// The server allows the same slack: a take stopped by the 30 s timer can run a frame over.
+export const EDIT_SOURCE_SLACK_SECONDS = 0.05;
+
export type AudioSourceRef =
| { input_id: string }
| { clip_id: string }
@@ -28,13 +33,18 @@ export function clipReference(clip: {
id: string;
prompt: string;
duration_s: number | null;
+ workflow?: string | null;
+ reference_name?: string | null;
}): AudioSourceSelection {
+ // Only speech is a transcript: not a Music description, nor the file name an untranscribed edit is titled by.
+ const spoken =
+ clip.workflow !== "music" && clip.prompt !== clip.reference_name;
return {
kind: "clip",
id: clip.id,
name: clip.prompt || "Generated clip",
durationS: clip.duration_s,
- transcript: clip.prompt || null,
+ transcript: (spoken && clip.prompt) || null,
language: null,
};
}
@@ -85,6 +95,14 @@ export function selectionExpired(
export type AudioOptionScalar = boolean | number | string;
+export interface AudioRunEditPart {
+ mode: "words" | "delivery";
+ markup?: string | null;
+ instructions?: string[] | null;
+ speed?: number | null;
+ pitch_steps?: number | null;
+}
+
export interface AudioMusicRunFields {
mode: "song" | "sfx" | "edit";
lyrics?: string | null;
@@ -101,7 +119,7 @@ export interface AudioMusicRunFields {
}
export interface AudioRunRequest {
- workflow: "clone" | "speak" | "music" | "separate";
+ workflow: "clone" | "speak" | "edit" | "music" | "separate";
music?: AudioMusicRunFields;
/** Required for clone, speak and music; separate takes none. */
text?: string;
@@ -113,7 +131,8 @@ export interface AudioRunRequest {
reference_text?: string | null;
emotion?: AudioSourceRef | null;
};
- options?: Record<string, boolean | number | string>;
+ edit?: AudioRunEditPart | null;
+ options?: Record<string, AudioOptionScalar>;
speed?: number | null;
seed?: number | null;
max_tokens?: number | null;
@@ -154,6 +173,8 @@ export function buildAudioRunBody(
const emotion = cleanRef(request.inputs?.emotion);
if (emotion) inputs.emotion = emotion;
if (Object.keys(inputs).length > 0) body.inputs = inputs;
+ const edit = cleanEdit(request.edit);
+ if (edit) body.edit = edit;
const options = Object.fromEntries(
Object.entries(request.options ?? {}).filter(
([name, value]) =>
@@ -176,6 +197,29 @@ export function buildAudioRunBody(
return body;
}
+function cleanEdit(
+ edit: AudioRunEditPart | null | undefined,
+): Record<string, unknown> | null {
+ if (!edit || (edit.mode !== "words" && edit.mode !== "delivery")) return null;
+ const out: Record<string, unknown> = { mode: edit.mode };
+ if (typeof edit.markup === "string" && edit.markup.trim())
+ out.markup = edit.markup;
+ if (Array.isArray(edit.instructions)) {
+ const instructions = edit.instructions.filter(
+ (line): line is string => typeof line === "string" && line.length > 0,
+ );
+ if (instructions.length > 0) out.instructions = instructions;
+ }
+ if (typeof edit.speed === "number" && Number.isFinite(edit.speed))
+ out.speed = edit.speed;
+ if (
+ typeof edit.pitch_steps === "number" &&
+ Number.isInteger(edit.pitch_steps)
+ )
+ out.pitch_steps = edit.pitch_steps;
+ return out;
+}
+
function finiteSeconds(value: number | null | undefined): number | null {
return typeof value === "number" && Number.isFinite(value) && value >= 0
? value
diff --git a/studio/frontend/src/features/audio/catalog.ts b/studio/frontend/src/features/audio/catalog.ts
index 1f63089cb42..d696967aea7 100644
--- a/studio/frontend/src/features/audio/catalog.ts
+++ b/studio/frontend/src/features/audio/catalog.ts
@@ -148,18 +148,19 @@ export function audioModelsForTask(task: AudioTask): ModelOption[] {
}
const CAPABILITY_LABEL: Record<
- AudioTask | "music" | "clone" | "separate",
+ AudioTask | "music" | "clone" | "edit" | "separate",
string
> = {
music: "Music generation",
separate: "Source separation",
clone: "Voice cloning",
+ edit: "Speech editing",
tts: "Text-to-speech",
stt: "Speech-to-text",
};
export function audioCapabilityLine(
- task: AudioTask | "music" | "clone" | "separate",
+ task: AudioTask | "music" | "clone" | "edit" | "separate",
detail?: string | null,
): string {
const base = CAPABILITY_LABEL[task];
diff --git a/studio/frontend/src/features/audio/components/ab-compare-state.ts b/studio/frontend/src/features/audio/components/ab-compare-state.ts
new file mode 100644
index 00000000000..9604a13a802
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/ab-compare-state.ts
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+export type ABSide = "original" | "edited";
+
+export interface ABState {
+ side: ABSide;
+ position: number;
+ playing: boolean;
+}
+
+export const INITIAL_AB_STATE: ABState = {
+ side: "edited",
+ position: 0,
+ playing: false,
+};
+
+const knownDuration = (duration: number | null | undefined): number | null =>
+ typeof duration === "number" && Number.isFinite(duration) && duration > 0
+ ? duration
+ : null;
+
+export function clampPosition(
+ seconds: number,
+ duration: number | null | undefined,
+): number {
+ const value = Number.isFinite(seconds) ? Math.max(0, seconds) : 0;
+ const limit = knownDuration(duration);
+ return limit === null ? value : Math.min(value, limit);
+}
+
+export function switchSide(
+ state: ABState,
+ side: ABSide,
+ newDuration: number | null | undefined,
+): ABState {
+ if (side === state.side) return state;
+ return {
+ side,
+ position: clampPosition(state.position, newDuration),
+ playing: state.playing,
+ };
+}
+
+/** A position at or past the end restarts from the top rather than ending at once. */
+export function resumeAfterLoad(
+ state: ABState,
+ mediaDuration: number | null | undefined,
+): { time: number; play: boolean } {
+ const limit = knownDuration(mediaDuration);
+ const time = clampPosition(state.position, limit);
+ return {
+ time: limit !== null && time >= limit - 0.05 ? 0 : time,
+ play: state.playing,
+ };
+}
diff --git a/studio/frontend/src/features/audio/components/ab-compare.tsx b/studio/frontend/src/features/audio/components/ab-compare.tsx
new file mode 100644
index 00000000000..b8e28bdda6c
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/ab-compare.tsx
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Button } from "@/components/ui/button";
+import { PillTabs } from "@/features/model-picker/components/model-selector/pill-tabs";
+import { cn } from "@/lib/utils";
+import { PauseIcon, PlayIcon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import { type Ref, useCallback, useEffect, useRef, useState } from "react";
+import { fetchAudioBlob } from "../api";
+import {
+ type ABSide,
+ type ABState,
+ INITIAL_AB_STATE,
+ clampPosition,
+ resumeAfterLoad,
+ switchSide,
+} from "./ab-compare-state";
+import { WAVEFORM_BARS, computePeaks, formatSeconds } from "./waveform-peaks";
+
+const SEEK_STEP_SECONDS = 5;
+// Decoding holds every sample in memory; past this the bars stay flat.
+const DECODE_MAX_BYTES = 60 * 1024 * 1024;
+
+export interface ABClip {
+ src: string | null;
+ /** Server URL to draw bars from: the page CSP blocks fetching an object URL. */
+ fileUrl?: string | null;
+ label: string;
+ durationS: number | null;
+}
+
+async function decodeClipPeaks(
+ src: string,
+ signal: AbortSignal,
+): Promise<number[] | null> {
+ if (src.startsWith("blob:")) return null;
+ const blob = src.startsWith("data:")
+ ? await (await fetch(src, { signal })).blob()
+ : await fetchAudioBlob(src, signal);
+ if (blob.size > DECODE_MAX_BYTES) return null;
+ const Offline =
+ window.OfflineAudioContext ||
+ (
+ window as unknown as {
+ webkitOfflineAudioContext?: typeof OfflineAudioContext;
+ }
+ ).webkitOfflineAudioContext;
+ if (!Offline) return null;
+ const context = new Offline(1, 1, 22050);
+ const buffer = await context.decodeAudioData(await blob.arrayBuffer());
+ return computePeaks(
+ Array.from({ length: buffer.numberOfChannels }, (_, index) =>
+ buffer.getChannelData(index),
+ ),
+ );
+}
+
+function useClipPeaks(srcs: readonly (string | null)[]) {
+ const [peaks, setPeaks] = useState<Record<string, number[] | null>>({});
+ const key = srcs.join("\n");
+ // biome-ignore lint/correctness/useExhaustiveDependencies: `key` is the srcs, joined.
+ useEffect(() => {
+ const controller = new AbortController();
+ for (const src of srcs) {
+ if (!src) continue;
+ decodeClipPeaks(src, controller.signal)
+ .then((bars) => {
+ if (!controller.signal.aborted)
+ setPeaks((prev) => ({ ...prev, [src]: bars }));
+ })
+ .catch(() => {
+ if (!controller.signal.aborted)
+ setPeaks((prev) => ({ ...prev, [src]: null }));
+ });
+ }
+ return () => controller.abort();
+ }, [key]);
+ return peaks;
+}
+
+/** Original | Edited on one player; switching sides keeps the moment and the playing state. */
+export function ABCompare({
+ original,
+ edited,
+ autoFocusRef,
+ className,
+}: {
+ original: ABClip;
+ edited: ABClip;
+ autoFocusRef?: Ref<HTMLButtonElement>;
+ className?: string;
+}) {
+ const audioRef = useRef<HTMLAudioElement | null>(null);
+ const [ab, setAb] = useState<ABState>(INITIAL_AB_STATE);
+ const [mediaDuration, setMediaDuration] = useState<
+ Partial<Record<ABSide, number>>
+ >({});
+ // Set while the element loads the other side, so its pause/time events are not the user's.
+ const pendingRef = useRef<ABState | null>(null);
+ const originalBarsSrc = original.fileUrl ?? original.src;
+ const editedBarsSrc = edited.fileUrl ?? edited.src;
+ const peaks = useClipPeaks([originalBarsSrc, editedBarsSrc]);
+
+ const clips: Record<ABSide, ABClip> = { original, edited };
+ const current = clips[ab.side];
+ const durationOf = (side: ABSide): number | null => {
+ const fromProps = clips[side].durationS;
+ return fromProps && fromProps > 0
+ ? fromProps
+ : (mediaDuration[side] ?? null);
+ };
+ const duration = durationOf(ab.side) ?? 0;
+ const fraction = duration > 0 ? Math.min(1, ab.position / duration) : 0;
+ const barsSrc = ab.side === "original" ? originalBarsSrc : editedBarsSrc;
+ const bars = barsSrc ? (peaks[barsSrc] ?? null) : null;
+ const count = bars?.length ?? WAVEFORM_BARS;
+ const src = current.src;
+
+ // A new pair resets the player; the Original's object URL arriving late does not.
+ const originalKey = original.fileUrl ?? original.src;
+ const editedKey = edited.fileUrl ?? edited.src;
+ // biome-ignore lint/correctness/useExhaustiveDependencies: the clip keys are the trigger, not inputs.
+ useEffect(() => {
+ pendingRef.current = null;
+ setAb(INITIAL_AB_STATE);
+ setMediaDuration({});
+ }, [originalKey, editedKey]);
+
+ const toggle = useCallback(() => {
+ const audio = audioRef.current;
+ if (!(audio && src)) return;
+ if (audio.paused) {
+ audio.play().catch(() => setAb((s) => ({ ...s, playing: false })));
+ } else {
+ audio.pause();
+ }
+ }, [src]);
+
+ const seekTo = useCallback(
+ (seconds: number) => {
+ const audio = audioRef.current;
+ if (!(audio && src) || duration <= 0) return;
+ const next = clampPosition(seconds, duration);
+ audio.currentTime = next;
+ setAb((s) => ({ ...s, position: next }));
+ },
+ [src, duration],
+ );
+
+ const choose = (side: ABSide) => {
+ if (side === ab.side) return;
+ const audio = audioRef.current;
+ const now: ABState = {
+ ...ab,
+ position: audio ? audio.currentTime : ab.position,
+ playing: audio ? !audio.paused : ab.playing,
+ };
+ const next = switchSide(now, side, durationOf(side));
+ pendingRef.current = clips[side].src ? next : null;
+ // A side still loading has nothing playing yet.
+ setAb(clips[side].src ? next : { ...next, playing: false });
+ };
+
+ return (
+ <div className={cn("grid gap-2", className)}>
+ <PillTabs
+ ariaLabel="Compare"
+ value={ab.side}
+ onValueChange={(side) => choose(side as ABSide)}
+ fit={true}
+ compact={true}
+ className="[&>button]:px-3"
+ tabs={[
+ { value: "original", label: "Original" },
+ { value: "edited", label: "Edited" },
+ ]}
+ />
+ <div className="flex items-center gap-2">
+ <Button
+ ref={autoFocusRef}
+ type="button"
+ variant="muted"
+ size="icon"
+ className="size-[calc(32px*var(--ui-space-scale,1))] shrink-0"
+ aria-label={
+ ab.playing ? `Pause ${current.label}` : `Play ${current.label}`
+ }
+ disabled={!src}
+ onClick={toggle}
+ >
+ <HugeiconsIcon
+ icon={ab.playing ? PauseIcon : PlayIcon}
+ className="size-3.5"
+ />
+ </Button>
+ <div
+ role="slider"
+ tabIndex={src ? 0 : -1}
+ aria-label={`${current.label} playback position`}
+ aria-valuemin={0}
+ aria-valuemax={Math.round(duration)}
+ aria-valuenow={Math.round(ab.position)}
+ aria-valuetext={`${formatSeconds(ab.position)} of ${formatSeconds(duration)}`}
+ aria-disabled={!src}
+ className="relative h-[calc(32px*var(--ui-space-scale,1))] min-w-0 flex-1 cursor-pointer rounded-md focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
+ onKeyDown={(event) => {
+ if (event.key === " ") {
+ event.preventDefault();
+ toggle();
+ } else if (event.key === "ArrowRight") {
+ event.preventDefault();
+ seekTo(ab.position + SEEK_STEP_SECONDS);
+ } else if (event.key === "ArrowLeft") {
+ event.preventDefault();
+ seekTo(ab.position - SEEK_STEP_SECONDS);
+ } else if (event.key === "Home") {
+ event.preventDefault();
+ seekTo(0);
+ }
+ }}
+ onClick={(event) => {
+ const box = event.currentTarget.getBoundingClientRect();
+ if (box.width <= 0) return;
+ seekTo(((event.clientX - box.left) / box.width) * duration);
+ }}
+ >
+ <svg
+ aria-hidden="true"
+ className="size-full"
+ viewBox={`0 0 ${count * 3} 32`}
+ preserveAspectRatio="none"
+ >
+ {Array.from({ length: count }, (_, index) => {
+ const peak = bars ? (bars[index] ?? 0) : 0;
+ // A floor keeps silence visible as a line rather than a gap.
+ const height = Math.max(2, peak * 30);
+ const played = (index + 0.5) / count <= fraction;
+ return (
+ <rect
+ // biome-ignore lint/suspicious/noArrayIndexKey: bars are positional and never reorder.
+ key={index}
+ x={index * 3 + 0.5}
+ y={(32 - height) / 2}
+ width={2}
+ height={height}
+ rx={1}
+ className={cn(
+ "fill-current",
+ played && ab.position > 0
+ ? "text-foreground"
+ : "text-muted-foreground",
+ bars ? "opacity-100" : "opacity-40",
+ )}
+ />
+ );
+ })}
+ </svg>
+ </div>
+ <span className="shrink-0 font-mono text-ui-11p5 tabular-nums text-muted-foreground">
+ {ab.position > 0 ? `${formatSeconds(ab.position)} / ` : ""}
+ {formatSeconds(duration)}
+ </span>
+ </div>
+ {src ? (
+ // biome-ignore lint/a11y/useMediaCaption: speech clips have no caption track; the clip's text is shown beside it.
+ <audio
+ ref={audioRef}
+ src={src}
+ preload="metadata"
+ className="hidden"
+ onPlay={() => {
+ if (!pendingRef.current) setAb((s) => ({ ...s, playing: true }));
+ }}
+ onPause={() => {
+ if (!pendingRef.current) setAb((s) => ({ ...s, playing: false }));
+ }}
+ onEnded={() => setAb((s) => ({ ...s, playing: false, position: 0 }))}
+ onTimeUpdate={(event) => {
+ if (pendingRef.current) return;
+ const time = event.currentTarget.currentTime;
+ setAb((s) => ({ ...s, position: time }));
+ }}
+ onLoadedMetadata={(event) => {
+ const audio = event.currentTarget;
+ const length = Number.isFinite(audio.duration)
+ ? audio.duration
+ : null;
+ const side = ab.side;
+ if (length !== null)
+ setMediaDuration((prev) => ({ ...prev, [side]: length }));
+ const pending = pendingRef.current;
+ if (!pending) return;
+ pendingRef.current = null;
+ const { time, play } = resumeAfterLoad(pending, length);
+ audio.currentTime = time;
+ setAb((s) => ({ ...s, position: time, playing: play }));
+ if (play)
+ audio
+ .play()
+ .catch(() => setAb((s) => ({ ...s, playing: false })));
+ }}
+ />
+ ) : null}
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/audio/components/clip-card.tsx b/studio/frontend/src/features/audio/components/clip-card.tsx
index afdbc53d547..f76cb214ed7 100644
--- a/studio/frontend/src/features/audio/components/clip-card.tsx
+++ b/studio/frontend/src/features/audio/components/clip-card.tsx
@@ -71,6 +71,7 @@ export function ClipCard({
status,
focusOnMount = false,
onFocused,
+ player,
}: {
title: string;
model: string;
@@ -83,6 +84,8 @@ export function ClipCard({
status?: string;
focusOnMount?: boolean;
onFocused?: () => void;
+ /** Replaces the waveform player, e.g. Edit's Original/Edited compare. */
+ player?: ReactNode;
}) {
const focusPlay = (element: HTMLDivElement | null) => {
if (!(element && focusOnMount && src)) return;
@@ -112,14 +115,16 @@ export function ClipCard({
</Button>
{menu}
</div>
- <div ref={focusPlay}>
- <Waveform
- peaks={peaks}
- durationS={durationS}
- src={src}
- label={title || "audio clip"}
- />
- </div>
+ {player ?? (
+ <div ref={focusPlay}>
+ <Waveform
+ peaks={peaks}
+ durationS={durationS}
+ src={src}
+ label={title || "audio clip"}
+ />
+ </div>
+ )}
</Card>
);
}
diff --git a/studio/frontend/src/features/audio/components/transcript-diff-editor.tsx b/studio/frontend/src/features/audio/components/transcript-diff-editor.tsx
new file mode 100644
index 00000000000..0f4dec673d2
--- /dev/null
+++ b/studio/frontend/src/features/audio/components/transcript-diff-editor.tsx
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Textarea } from "@/components/ui/textarea";
+import { type Ref, useMemo } from "react";
+import { EDIT_DIFF_MAX_WORDS, countChanges, diffSegments } from "../edit-diff";
+import { EDIT_COPY } from "../edit-policy";
+
+/** ② the transcript's editable copy, each change marked below it (with sr-only text too). */
+export function TranscriptDiffEditor({
+ value,
+ onChange,
+ transcript,
+ disabled = false,
+ onReset,
+ id,
+ textareaRef,
+ dataTour,
+}: {
+ value: string;
+ onChange: (value: string) => void;
+ transcript: string;
+ disabled?: boolean;
+ onReset: () => void;
+ id: string;
+ textareaRef?: Ref<HTMLTextAreaElement>;
+ dataTour?: string;
+}) {
+ const segments = useMemo(
+ () => diffSegments(transcript, value),
+ [transcript, value],
+ );
+ const changes = useMemo(
+ () => countChanges(transcript, value),
+ [transcript, value],
+ );
+ const hintId = `${id}-hint`;
+ const diffId = `${id}-diff`;
+
+ return (
+ <div className="grid gap-1.5" data-tour={dataTour}>
+ <div className="flex min-w-0 flex-wrap items-center justify-between gap-x-2 gap-y-1">
+ <label
+ htmlFor={id}
+ className="whitespace-nowrap text-ui-13 font-medium text-foreground"
+ >
+ {EDIT_COPY.changesLabel}
+ </label>
+ <div className="flex shrink-0 items-center gap-1.5">
+ {changes === null ? (
+ <Badge variant="outline">Too long</Badge>
+ ) : changes > 0 ? (
+ <Badge variant="outline">
+ {changes === 1 ? "1 change" : `${changes} changes`}
+ </Badge>
+ ) : null}
+ <Button
+ type="button"
+ variant="ghost"
+ size="xs"
+ disabled={disabled || value === transcript}
+ onClick={onReset}
+ >
+ {EDIT_COPY.resetLabel}
+ </Button>
+ </div>
+ </div>
+ <Textarea
+ id={id}
+ ref={textareaRef}
+ value={value}
+ disabled={disabled}
+ aria-describedby={`${hintId} ${diffId}`}
+ onChange={(event) => onChange(event.target.value)}
+ className="min-h-16"
+ />
+ {segments && changes ? (
+ <p
+ id={diffId}
+ aria-label="Changes from the transcript"
+ className="rounded-xl bg-muted/50 px-3 py-2 text-ui-13 leading-relaxed text-foreground [overflow-wrap:anywhere]"
+ >
+ {segments.map((segment, index) => {
+ return (
+ // biome-ignore lint/suspicious/noArrayIndexKey: segments are positional and rebuilt on every change.
+ <span key={index}>
+ {index > 0 ? " " : ""}
+ {segment.kind === "insert" ? (
+ <ins className="rounded-sm bg-secondary px-0.5 text-secondary-foreground underline decoration-2 underline-offset-2">
+ <span className="sr-only">{segment.srLabel}</span>
+ {segment.text}
+ </ins>
+ ) : segment.kind === "delete" ? (
+ <del className="text-muted-foreground line-through">
+ <span className="sr-only">{segment.srLabel}</span>
+ {segment.text}
+ </del>
+ ) : (
+ segment.text
+ )}
+ </span>
+ );
+ })}
+ </p>
+ ) : (
+ <span id={diffId} hidden={true} />
+ )}
+ <p
+ id={hintId}
+ className="min-w-0 text-ui-11p5 leading-snug text-muted-foreground"
+ >
+ {changes === null
+ ? `Diff shown for up to ${EDIT_DIFF_MAX_WORDS} words.`
+ : changes === 0
+ ? EDIT_COPY.changesHint
+ : "Underlined words are new, struck-through words are removed."}
+ </p>
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/audio/edit-adapters.ts b/studio/frontend/src/features/audio/edit-adapters.ts
new file mode 100644
index 00000000000..d480d34f7bc
--- /dev/null
+++ b/studio/frontend/src/features/audio/edit-adapters.ts
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// tests/fixtures/audio-edit-requests.json pins the run bodies built here against the backend.
+
+import type { AudioOptionValues } from "./audio-options";
+import type {
+ AudioRunEditPart,
+ AudioRunRequest,
+ AudioSourceRef,
+} from "./audio-run-request";
+import {
+ EDIT_DIFF_MAX_WORDS,
+ type EditChange,
+ changesBetween,
+ diffWords,
+} from "./edit-diff";
+import type { AudioModelContext } from "./tools/types";
+
+export const FIRERED_MAX_CHANGES = 5;
+/** The route's limit on one FireRedAudio instruction. */
+const FIRERED_INSTRUCTION_MAX_CHARS = 300;
+
+export const EDIT_TOO_LONG = `Keep the transcript under ${EDIT_DIFF_MAX_WORDS} words.`;
+export const FIRERED_TOO_MANY_CHANGES = `FireRedAudio applies at most ${FIRERED_MAX_CHANGES} changes. Make fewer changes, or use DotTTS Edit.`;
+export const FIRERED_INSERT_AT_END =
+ "FireRedAudio can only insert before another word. Move the new words, or use DotTTS Edit.";
+export const FIRERED_CHANGE_TOO_LONG =
+ "Make each change shorter for FireRedAudio, or use DotTTS Edit.";
+export const DOTS_BAD_CHARACTERS =
+ "Remove quotes and angle brackets from the changed words for DotTTS Edit.";
+export const DOTS_ANGLE_BRACKETS =
+ "Remove angle brackets from the transcript for DotTTS Edit.";
+export const DELIVERY_NEEDS_FIRERED = "Delivery changes need FireRedAudio.";
+
+export type EditMode = "words" | "delivery";
+
+export interface EditDelivery {
+ speed: number;
+ pitchSteps: number;
+}
+
+export interface EditRunInput {
+ source?: AudioSourceRef | null;
+ transcript: string;
+ edited: string;
+ mode: EditMode;
+ delivery?: EditDelivery | null;
+ advanced?: AudioOptionValues | null;
+ /** The text of a Delivery run with no transcript. */
+ sourceName?: string | null;
+}
+
+export interface EditAdapter {
+ family: string;
+ label: string;
+ style: "markup" | "sentence" | "instructions";
+ /** Spec options the adapter sets itself, which Advanced leaves out. */
+ claims: readonly string[];
+ maxChanges: number | null;
+ /** Speed [min, max, step] and raise-only pitch steps [min, max]; null without delivery control. */
+ delivery: {
+ speed: readonly [number, number, number];
+ pitchSteps: readonly [number, number];
+ } | null;
+ howItEdits: (changes: number) => string;
+ /** Zero changes is not an error here: editBlocker reports it with its own action. */
+ validateWords: (original: string, edited: string) => string | null;
+ buildEdit: (
+ input: Pick<EditRunInput, "transcript" | "edited" | "mode" | "delivery">,
+ ) => AudioRunEditPart;
+}
+
+/** DotTTS Edit markup: transcript words joined by single spaces, each change wrapped. */
+export function markupFor(original: string, edited: string): string | null {
+ const ops = diffWords(original, edited);
+ if (!ops) return null;
+ const pieces: string[] = [];
+ let k = 0;
+ while (k < ops.length) {
+ if (ops[k].kind === "equal") {
+ pieces.push(ops[k].word);
+ k += 1;
+ continue;
+ }
+ const old: string[] = [];
+ const added: string[] = [];
+ while (k < ops.length && ops[k].kind !== "equal") {
+ (ops[k].kind === "delete" ? old : added).push(ops[k].word);
+ k += 1;
+ }
+ if (old.length && added.length) {
+ pieces.push(`<sub targ="${added.join(" ")}">${old.join(" ")}</sub>`);
+ } else if (old.length) {
+ pieces.push(`<del>${old.join(" ")}</del>`);
+ } else {
+ pieces.push(`<ins>${added.join(" ")}</ins>`);
+ }
+ }
+ return pieces.join(" ");
+}
+
+/** Null for an insert with no word after it: FireRedAudio can only insert before an anchor. */
+function fireredInstruction(change: EditChange): string | null {
+ const old = change.old.join(" ");
+ const added = change.new.join(" ");
+ switch (change.kind) {
+ case "replace":
+ return `Replace '${old}' with '${added}'.`;
+ case "delete":
+ return `Delete '${old}'.`;
+ default:
+ return change.before === null
+ ? null
+ : `Insert '${added}' before '${change.before}'.`;
+ }
+}
+
+const isUnchangedSpeed = (speed: number | null | undefined): boolean =>
+ typeof speed !== "number" ||
+ !Number.isFinite(speed) ||
+ Math.abs(speed - 1) < 1e-6;
+
+/** Mirrors the backend's `delivery_instructions` (Python `{x:g}` formatting). */
+export function deliveryInstructions(
+ speed: number | null | undefined,
+ pitchSteps: number | null | undefined,
+): string[] {
+ const lines: string[] = [];
+ if (!isUnchangedSpeed(speed))
+ lines.push(
+ `adjust the speed to ${Number((speed as number).toPrecision(6))}x`,
+ );
+ if (typeof pitchSteps === "number" && pitchSteps > 0)
+ lines.push(`shift the pitch by ${Math.round(pitchSteps)} steps`);
+ return lines;
+}
+
+const STRUCTURAL_CLAIMS = [
+ "template_name",
+ "source_text",
+ "target_text",
+ "instruction",
+] as const;
+
+const plural = (count: number, word: string) =>
+ `${count} ${word}${count === 1 ? "" : "s"}`;
+
+export const EDIT_ADAPTERS: Readonly<Record<string, EditAdapter>> = {
+ dots_tts: {
+ family: "dots_tts",
+ label: "DotTTS Edit",
+ style: "markup",
+ claims: STRUCTURAL_CLAIMS,
+ maxChanges: null,
+ delivery: null,
+ howItEdits: (changes) =>
+ changes > 1
+ ? `Redoes the ${plural(changes, "changed word group")} in one pass and keeps the rest of the recording.`
+ : "Redoes only the changed words in one pass and keeps the rest of the recording.",
+ validateWords: (original, edited) => {
+ const changes = changesBetween(original, edited);
+ if (changes === null) return EDIT_TOO_LONG;
+ if (
+ changes.some((change) =>
+ [...change.old, ...change.new].some((word) => /["<>]/.test(word)),
+ )
+ )
+ return DOTS_BAD_CHARACTERS;
+ if (/[<>]/.test(original) || /[<>]/.test(edited))
+ return DOTS_ANGLE_BRACKETS;
+ return null;
+ },
+ buildEdit: ({ transcript, edited }) => {
+ const changed = (changesBetween(transcript, edited)?.length ?? 0) > 0;
+ const markup = changed ? markupFor(transcript, edited) : null;
+ return markup ? { mode: "words", markup } : { mode: "words" };
+ },
+ },
+ vevo2: {
+ family: "vevo2",
+ label: "Vevo2",
+ style: "sentence",
+ claims: STRUCTURAL_CLAIMS,
+ maxChanges: null,
+ delivery: null,
+ howItEdits: () =>
+ "Speaks the whole edited sentence again in the recording's voice, in one pass.",
+ validateWords: (original, edited) =>
+ changesBetween(original, edited) === null ? EDIT_TOO_LONG : null,
+ // Vevo2 reads the edited `text` itself.
+ buildEdit: () => ({ mode: "words" }),
+ },
+ firered_audio: {
+ family: "firered_audio",
+ label: "FireRedAudio",
+ style: "instructions",
+ claims: ["template_name", "instruction"],
+ maxChanges: FIRERED_MAX_CHANGES,
+ delivery: { speed: [0.5, 2, 0.1], pitchSteps: [1, 6] },
+ howItEdits: (changes) =>
+ changes > 1
+ ? `Applies ${plural(changes, "change")}, one pass each, every pass editing the last one's result.`
+ : "Applies each change in its own pass, every pass editing the last one's result.",
+ validateWords: (original, edited) => {
+ const changes = changesBetween(original, edited);
+ if (changes === null) return EDIT_TOO_LONG;
+ if (changes.length > FIRERED_MAX_CHANGES) return FIRERED_TOO_MANY_CHANGES;
+ const lines = changes.map(fireredInstruction);
+ if (lines.some((line) => line === null)) return FIRERED_INSERT_AT_END;
+ if (
+ lines.some(
+ (line) => (line?.length ?? 0) > FIRERED_INSTRUCTION_MAX_CHARS,
+ )
+ )
+ return FIRERED_CHANGE_TOO_LONG;
+ return null;
+ },
+ buildEdit: ({ transcript, edited, mode, delivery }) => {
+ if (mode === "delivery") {
+ const part: AudioRunEditPart = { mode: "delivery" };
+ if (!isUnchangedSpeed(delivery?.speed))
+ part.speed = Math.round((delivery?.speed as number) * 100) / 100;
+ if (delivery && delivery.pitchSteps > 0)
+ part.pitch_steps = Math.round(delivery.pitchSteps);
+ return part;
+ }
+ const instructions = (changesBetween(transcript, edited) ?? [])
+ .map(fireredInstruction)
+ .filter((line): line is string => line !== null);
+ return instructions.length
+ ? { mode: "words", instructions }
+ : { mode: "words" };
+ },
+ },
+};
+
+/** Needs the edit workflow too: DotTTS-MF is dots_tts but cannot edit. */
+export function editAdapterFor(
+ ctx: Pick<AudioModelContext, "audioFamily" | "audioWorkflows">,
+): EditAdapter | null {
+ if (!ctx.audioWorkflows?.includes("edit")) return null;
+ const family = ctx.audioFamily ?? "";
+ return Object.hasOwn(EDIT_ADAPTERS, family) ? EDIT_ADAPTERS[family] : null;
+}
+
+/** Does not validate: check editBlocker first. */
+export function buildEditRun(
+ adapter: EditAdapter,
+ input: EditRunInput,
+): AudioRunRequest {
+ const mode: EditMode =
+ input.mode === "delivery" && adapter.delivery ? "delivery" : "words";
+ const transcript = input.transcript.trim();
+ const text =
+ mode === "delivery"
+ ? transcript || input.sourceName?.trim() || "Recording"
+ : input.edited.trim();
+ const inputs: NonNullable<AudioRunRequest["inputs"]> = {};
+ if (input.source) inputs.source = { ...input.source };
+ if (transcript) inputs.reference_text = transcript;
+ const run: AudioRunRequest = {
+ workflow: "edit",
+ text,
+ inputs,
+ edit: adapter.buildEdit({
+ transcript,
+ edited: input.edited,
+ mode,
+ delivery: input.delivery,
+ }),
+ };
+ const options = Object.fromEntries(
+ Object.entries(input.advanced ?? {}).filter(
+ ([name]) => name && !adapter.claims.includes(name),
+ ),
+ );
+ if (Object.keys(options).length > 0) run.options = options;
+ return run;
+}
+
+/** The progress line for a chained run, or null when it is one pass. */
+export function editPhaseLabel(
+ adapter: EditAdapter,
+ run: AudioRunRequest,
+): string | null {
+ if (adapter.style !== "instructions") return null;
+ const delivery = run.edit?.mode === "delivery";
+ const passes = delivery
+ ? deliveryInstructions(run.edit?.speed, run.edit?.pitch_steps).length
+ : (run.edit?.instructions?.length ?? 0);
+ if (passes <= 1) return null;
+ return delivery
+ ? `Applying ${passes} delivery changes, one pass each`
+ : `Applying ${passes} changes, one pass each`;
+}
diff --git a/studio/frontend/src/features/audio/edit-diff.ts b/studio/frontend/src/features/audio/edit-diff.ts
new file mode 100644
index 00000000000..f6807a0055a
--- /dev/null
+++ b/studio/frontend/src/features/audio/edit-diff.ts
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// One diff drives both the highlighting in ② and the changes sent to the model. Words are
+// whitespace-separated tokens: punctuation stays on its word.
+
+/** The diff is quadratic: past this many words on either side it is not computed. */
+export const EDIT_DIFF_MAX_WORDS = 400;
+
+type DiffOpKind = "equal" | "delete" | "insert";
+
+interface DiffOp {
+ kind: DiffOpKind;
+ word: string;
+}
+
+export interface EditChange {
+ kind: "replace" | "delete" | "insert";
+ old: string[];
+ new: string[];
+ /** The transcript word right after the change; null at the end. */
+ before: string | null;
+ /** Index of the change's first word in the transcript. */
+ index: number;
+}
+
+interface DiffSegment {
+ kind: DiffOpKind;
+ text: string;
+ /** Screen-reader prefix, so colour and decoration never carry the meaning alone. */
+ srLabel: "inserted " | "deleted " | null;
+}
+
+export function tokenizeWords(text: string): string[] {
+ const trimmed = text.trim();
+ return trimmed ? trimmed.split(/\s+/) : [];
+}
+
+/** LCS word diff; null past the word cap. In a changed run removed words precede added ones. */
+export function diffWords(a: string, b: string): DiffOp[] | null {
+ const left = tokenizeWords(a);
+ const right = tokenizeWords(b);
+ if (left.length > EDIT_DIFF_MAX_WORDS || right.length > EDIT_DIFF_MAX_WORDS)
+ return null;
+ const n = left.length;
+ const m = right.length;
+ const width = m + 1;
+ const lcs = new Uint16Array((n + 1) * width);
+ for (let i = n - 1; i >= 0; i -= 1) {
+ for (let j = m - 1; j >= 0; j -= 1) {
+ lcs[i * width + j] =
+ left[i] === right[j]
+ ? lcs[(i + 1) * width + j + 1] + 1
+ : Math.max(lcs[(i + 1) * width + j], lcs[i * width + j + 1]);
+ }
+ }
+ const ops: DiffOp[] = [];
+ let i = 0;
+ let j = 0;
+ while (i < n && j < m) {
+ if (left[i] === right[j]) {
+ ops.push({ kind: "equal", word: left[i] });
+ i += 1;
+ j += 1;
+ } else if (lcs[(i + 1) * width + j] >= lcs[i * width + j + 1]) {
+ ops.push({ kind: "delete", word: left[i] });
+ i += 1;
+ } else {
+ ops.push({ kind: "insert", word: right[j] });
+ j += 1;
+ }
+ }
+ for (; i < n; i += 1) ops.push({ kind: "delete", word: left[i] });
+ for (; j < m; j += 1) ops.push({ kind: "insert", word: right[j] });
+ return ops;
+}
+
+export function groupChanges(ops: readonly DiffOp[]): EditChange[] {
+ const changes: EditChange[] = [];
+ let originalIndex = 0;
+ let k = 0;
+ while (k < ops.length) {
+ if (ops[k].kind === "equal") {
+ originalIndex += 1;
+ k += 1;
+ continue;
+ }
+ const start = originalIndex;
+ const old: string[] = [];
+ const added: string[] = [];
+ while (k < ops.length && ops[k].kind !== "equal") {
+ if (ops[k].kind === "delete") {
+ old.push(ops[k].word);
+ originalIndex += 1;
+ } else {
+ added.push(ops[k].word);
+ }
+ k += 1;
+ }
+ changes.push({
+ kind:
+ old.length && added.length
+ ? "replace"
+ : old.length
+ ? "delete"
+ : "insert",
+ old,
+ new: added,
+ before: k < ops.length ? ops[k].word : null,
+ index: start,
+ });
+ }
+ return changes;
+}
+
+export function changesBetween(a: string, b: string): EditChange[] | null {
+ const ops = diffWords(a, b);
+ return ops ? groupChanges(ops) : null;
+}
+
+export function countChanges(a: string, b: string): number | null {
+ return changesBetween(a, b)?.length ?? null;
+}
+
+export function diffSegments(a: string, b: string): DiffSegment[] | null {
+ const ops = diffWords(a, b);
+ if (!ops) return null;
+ const segments: DiffSegment[] = [];
+ const push = (kind: DiffOpKind, words: string[]) => {
+ if (words.length === 0) return;
+ segments.push({
+ kind,
+ text: words.join(" "),
+ srLabel:
+ kind === "insert" ? "inserted " : kind === "delete" ? "deleted " : null,
+ });
+ };
+ let k = 0;
+ while (k < ops.length) {
+ const equal: string[] = [];
+ while (k < ops.length && ops[k].kind === "equal") {
+ equal.push(ops[k].word);
+ k += 1;
+ }
+ push("equal", equal);
+ const deleted: string[] = [];
+ const inserted: string[] = [];
+ while (k < ops.length && ops[k].kind !== "equal") {
+ (ops[k].kind === "delete" ? deleted : inserted).push(ops[k].word);
+ k += 1;
+ }
+ push("delete", deleted);
+ push("insert", inserted);
+ }
+ return segments;
+}
diff --git a/studio/frontend/src/features/audio/edit-policy.ts b/studio/frontend/src/features/audio/edit-policy.ts
new file mode 100644
index 00000000000..cc02f3ef72a
--- /dev/null
+++ b/studio/frontend/src/features/audio/edit-policy.ts
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import {
+ type AudioSourceSelection,
+ EDIT_SOURCE_MAX_SECONDS,
+ EDIT_SOURCE_SLACK_SECONDS,
+} from "./audio-run-request";
+import {
+ EDIT_TOO_LONG,
+ type EditDelivery,
+ type EditMode,
+} from "./edit-adapters";
+import { countChanges } from "./edit-diff";
+
+export const EDIT_COPY = {
+ recordingLabel: "Recording",
+ recordingHint: "Up to 30 s of one voice.",
+ transcriptLabel: "① Check the transcript",
+ transcriptHint:
+ "Fix any word the recognizer got wrong, so it matches the audio.",
+ changesLabel: "② Make your changes",
+ changesHint: "Change, add or remove words. Each change is marked below.",
+ resetLabel: "Reset to transcript",
+ wordsTab: "Words",
+ deliveryTab: "Delivery",
+ speedLabel: "Speed",
+ pitchLabel: "Raise pitch",
+ deliveryLabel: "② Change the delivery",
+ pitchHint:
+ "Below 1× is slower, above is faster. FireRedAudio can raise pitch, not lower it, and makes one pass per change.",
+ howItEditsTitle: "How edits apply",
+ vevo2SwitchNote:
+ "Vevo2 reloads for editing when it was last used on Clone, about 1–9 s.",
+ emptyText:
+ "Edited speech lands here. Add a recording, fix its transcript, change the words, and press Generate.",
+} as const;
+
+export const EDIT_NO_SOURCE = "Add a recording to edit.";
+export const EDIT_SOURCE_BUSY = "Waiting for the recording…";
+export const EDIT_SOURCE_EXPIRED = "This recording expired. Add it again.";
+export const EDIT_SOURCE_TOO_LONG = `Edit works on recordings up to ${EDIT_SOURCE_MAX_SECONDS} s. Record or upload a shorter take.`;
+export const EDIT_TRANSCRIBING = "Checking the transcript…";
+export const EDIT_TRANSCRIPT_EMPTY = "Type what's said in the recording.";
+export const EDIT_NO_CHANGES = "Change at least one word in ②.";
+export const EDIT_CHANGES_EMPTY = "Keep at least one word in ②.";
+export const EDIT_DELIVERY_EMPTY = "Pick a speed or a pitch change.";
+
+export type EditBlockerActionId =
+ | "add-recording"
+ | "choose-recording"
+ | "transcribe"
+ | "type-transcript"
+ | "focus-changes";
+
+interface EditBlockerAction {
+ id: EditBlockerActionId;
+ label: string;
+}
+
+interface EditBlocker {
+ kind: string;
+ reason: string;
+ actions: readonly EditBlockerAction[];
+}
+
+interface EditBlockerInput {
+ source: AudioSourceSelection | null;
+ sourceBusy: boolean;
+ sourceExpired: boolean;
+ sourceError?: string | null;
+ sourceDurationS: number | null;
+ transcribing: boolean;
+ transcript: string;
+ edited: string;
+ mode: EditMode;
+ delivery: EditDelivery;
+ panelError: string | null;
+}
+
+const blocker = (
+ kind: string,
+ reason: string,
+ actions: EditBlockerAction[] = [],
+): EditBlocker => ({ kind, reason, actions });
+
+/** The first missing page input, in rail order. Model blockers come from the host first;
+ * Delivery reads no words, so the transcript steps apply to Words only. */
+export function editBlocker(input: EditBlockerInput): EditBlocker | null {
+ if (!input.source && !input.sourceBusy) {
+ return blocker("source", input.sourceError || EDIT_NO_SOURCE, [
+ { id: "add-recording", label: "Add recording" },
+ ]);
+ }
+ if (input.sourceBusy) return blocker("source-busy", EDIT_SOURCE_BUSY);
+ if (input.sourceExpired) {
+ return blocker("source-expired", EDIT_SOURCE_EXPIRED, [
+ { id: "add-recording", label: "Add it again" },
+ ]);
+ }
+ if (
+ typeof input.sourceDurationS === "number" &&
+ input.sourceDurationS > EDIT_SOURCE_MAX_SECONDS + EDIT_SOURCE_SLACK_SECONDS
+ ) {
+ return blocker("source-too-long", EDIT_SOURCE_TOO_LONG, [
+ { id: "choose-recording", label: "Choose another" },
+ ]);
+ }
+ if (input.mode === "words") {
+ if (input.transcribing) return blocker("transcribing", EDIT_TRANSCRIBING);
+ if (!input.transcript.trim()) {
+ return blocker("transcript", EDIT_TRANSCRIPT_EMPTY, [
+ { id: "transcribe", label: "Transcribe it" },
+ { id: "type-transcript", label: "Type it" },
+ ]);
+ }
+ if (!input.edited.trim()) {
+ return blocker("changes-empty", EDIT_CHANGES_EMPTY, [
+ { id: "focus-changes", label: "Go to ②" },
+ ]);
+ }
+ const changes = countChanges(input.transcript, input.edited);
+ if (changes === null) return blocker("too-long", EDIT_TOO_LONG);
+ if (changes === 0) {
+ return blocker("changes", EDIT_NO_CHANGES, [
+ { id: "focus-changes", label: "Go to ②" },
+ ]);
+ }
+ }
+ if (input.panelError) return blocker("panel", input.panelError);
+ if (
+ input.mode === "delivery" &&
+ Math.abs(input.delivery.speed - 1) < 1e-6 &&
+ !(input.delivery.pitchSteps > 0)
+ ) {
+ return blocker("delivery", EDIT_DELIVERY_EMPTY);
+ }
+ return null;
+}
diff --git a/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx b/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
index 52ba8945157..489e7610998 100644
--- a/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
+++ b/studio/frontend/src/features/audio/hooks/use-audio-gallery.tsx
@@ -64,7 +64,7 @@ export function useAudioGallery({
prompt: string;
model: string;
saved: boolean;
- workflow: "speak" | "music" | "clone" | "separate";
+ workflow: "speak" | "music" | "clone" | "edit" | "separate";
} | null>(null);
const fallbackClipRef = useRef(fallbackClip);
fallbackClipRef.current = fallbackClip;
@@ -425,7 +425,7 @@ export function useAudioGallery({
[refreshGallery],
);
- const handleClearGallery = useCallback(async (workflow?: "speak" | "clone" | "music" | "separate") => {
+ const handleClearGallery = useCallback(async (workflow?: "speak" | "clone" | "edit" | "music" | "separate") => {
try {
await clearAudioGallery(workflow);
if (workflow) {
@@ -555,7 +555,7 @@ export function useWorkflowHistory({
loadingMoreRef,
selectClip,
}: {
- workflow: "speak" | "clone" | "music" | "separate";
+ workflow: "speak" | "clone" | "edit" | "music" | "separate";
enabled: boolean;
} & Pick<
AudioGallery,
@@ -569,7 +569,9 @@ export function useWorkflowHistory({
| "selectClip"
>) {
const visibleClips = useMemo(
- () => clips.filter((clip) => clipWorkflow(clip) === workflow),
+ () =>
+ // An edit's original is kept for its A/B compare, not listed.
+ clips.filter((clip) => clipWorkflow(clip) === workflow && clip.role !== "source"),
[clips, workflow],
);
const selectedClip =
@@ -603,9 +605,11 @@ export function useWorkflowHistory({
const loadMoreVisible = useCallback(async () => {
if (visibleLoad.current) return;
visibleLoad.current = true;
+ // Counted like visibleClips: hidden edit originals add no row.
const countVisible = () =>
- galleryCache.clips.filter((clip) => clipWorkflow(clip) === workflow)
- .length;
+ galleryCache.clips.filter(
+ (clip) => clipWorkflow(clip) === workflow && clip.role !== "source",
+ ).length;
const before = countVisible();
try {
await loadGalleryUntil({
diff --git a/studio/frontend/src/features/audio/hooks/use-clone-generation.ts b/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
index 9992733e5a2..8d3d141dfe4 100644
--- a/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
+++ b/studio/frontend/src/features/audio/hooks/use-clone-generation.ts
@@ -55,13 +55,13 @@ export async function showRunResult({
}: {
response: AudioRunResponse;
text: string;
- workflow: "speak" | "clone" | "music" | "separate";
+ workflow: "speak" | "clone" | "edit" | "music" | "separate";
} & Pick<
AudioGallery,
"refreshGallery" | "selectClip" | "setFallbackClip" | "setSelectedId"
>): Promise<void> {
- const clip =
- response.clips.find((item) => item.role === "output") ?? response.clips[0];
+ // An edit whose output was not saved returns only its source clip: play the inline audio.
+ const clip = response.clips.find((item) => item.role === "output");
const refreshed = await refreshGallery();
if (clip) {
const listed = persistedClipForGeneration(clip.id, refreshed);
diff --git a/studio/frontend/src/features/audio/hooks/use-edit-generation.ts b/studio/frontend/src/features/audio/hooks/use-edit-generation.ts
new file mode 100644
index 00000000000..cbc44bd67c5
--- /dev/null
+++ b/studio/frontend/src/features/audio/hooks/use-edit-generation.ts
@@ -0,0 +1,421 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { toast } from "@/lib/toast";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { AudioApiError, runAudio } from "../api";
+import {
+ type AudioOptionSpec,
+ type AudioOptionValues,
+ audioOptionLabel,
+ audioOptionsForRequest,
+ missingRequiredAudioOptions,
+} from "../audio-options";
+import {
+ type AudioSourceSelection,
+ selectionExpired,
+ sourceRefOf,
+} from "../audio-run-request";
+import type { AudioSourceInputHandle } from "../components/audio-source-input";
+import { buildEditRun, editAdapterFor, editPhaseLabel } from "../edit-adapters";
+import { type EditBlockerActionId, editBlocker } from "../edit-policy";
+import type { GenerateBlocker } from "../pages/tts-workspace";
+import { useAudioCloneStore } from "../stores/audio-clone-store";
+import { useAudioEditStore } from "../stores/audio-edit-store";
+import { audioToolPanelsFor } from "../tools/registry";
+import {
+ audioModelContextFor,
+ claimedOptionNames,
+ collectToolRequest,
+ toolValueKey,
+} from "../tools/select";
+import type { EditCoreInputs } from "../tools/types";
+import type { AudioHostState } from "./audio-host-state";
+import type { AudioSourceStatus } from "./audio-source-state";
+import { REFERENCE_EXPIRED_MESSAGE } from "./audio-source-state";
+import type { AudioGallery } from "./use-audio-gallery";
+import type { AudioModelSlot } from "./use-audio-model-slot";
+import { showRunResult } from "./use-clone-generation";
+import { useReferenceTranscribe } from "./use-reference-transcribe";
+
+export const EDIT_TRANSCRIPT_FIELD_ID = "edit-transcript";
+export const EDIT_CHANGES_FIELD_ID = "edit-changes";
+
+/** A new recording restarts ① from its own text (a history clip's), or clears it for STT. */
+export function adoptEditSource(next: AudioSourceSelection | null) {
+ const store = useAudioEditStore.getState();
+ const same =
+ next?.id === store.source?.id && next?.kind === store.source?.kind;
+ store.setSource(next);
+ if (same || !next) return;
+ useAudioEditStore.setState({ editedTouched: false });
+ store.setTranscript(
+ next.transcript?.trim() ?? "",
+ next.transcript ? next.id : null,
+ );
+}
+
+/** Mirrors useCloneGeneration so Stop, phases and errors behave the same. */
+export function useEditGeneration({
+ status,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ generateAbort,
+ setMode,
+ setAdvancedOpen,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ pendingTranscribeRelease,
+ replayQueuedTtsPick,
+ audioOptionSpecs,
+ audioOptionValues,
+ sttRepo,
+}: Pick<
+ AudioHostState,
+ | "status"
+ | "busyRef"
+ | "setBusy"
+ | "updateGenerationPhase"
+ | "generateAbort"
+ | "setMode"
+ | "setAdvancedOpen"
+ | "refreshStatus"
+ | "activeRef"
+ | "modeRef"
+> &
+ Pick<
+ AudioGallery,
+ "refreshGallery" | "selectClip" | "setFallbackClip" | "setSelectedId"
+ > &
+ Pick<AudioModelSlot, "pendingTranscribeRelease" | "replayQueuedTtsPick"> & {
+ audioOptionSpecs: AudioOptionSpec[];
+ audioOptionValues: AudioOptionValues;
+ sttRepo: string | null;
+ }) {
+ const source = useAudioEditStore((state) => state.source);
+ const transcript = useAudioEditStore((state) => state.transcript);
+ const transcriptFor = useAudioEditStore((state) => state.transcriptFor);
+ const edited = useAudioEditStore((state) => state.edited);
+ const editMode = useAudioEditStore((state) => state.mode);
+ const delivery = useAudioEditStore((state) => state.delivery);
+ const storedToolValues = useAudioCloneStore((state) => state.toolValues);
+ const model = status?.active_model ?? null;
+ const [generationError, setGenerationError] = useState<string | null>(null);
+ const [phaseLabel, setPhaseLabel] = useState<string | null>(null);
+ const [sourceStatus, setSourceStatus] = useState<AudioSourceStatus>({
+ phase: "idle",
+ });
+ const sourceHandle = useRef<AudioSourceInputHandle | null>(null);
+ const changesRef = useRef<HTMLTextAreaElement | null>(null);
+ const [expiredSourceId, setExpiredSourceId] = useState<string | null>(null);
+
+ const transcriber = useReferenceTranscribe({
+ sttRepo,
+ onText: (next, from) => {
+ const store = useAudioEditStore.getState();
+ // The recording may have changed while this one was transcribed.
+ if (store.source?.id === from.id) store.setTranscript(next, from.id);
+ },
+ });
+
+ const toolContext = useMemo(
+ () =>
+ audioModelContextFor(status, {
+ musicGeneration: false,
+ cudaMusicGeneration: false,
+ musicNeedsDescription: false,
+ }),
+ [status],
+ );
+ const adapter = useMemo(() => editAdapterFor(toolContext), [toolContext]);
+ const mode =
+ editMode === "delivery" && adapter?.delivery ? "delivery" : "words";
+ const toolPanels = useMemo(
+ () => audioToolPanelsFor("edit", toolContext),
+ [toolContext],
+ );
+ const toolValues = useMemo(() => {
+ const values: Record<string, unknown> = {};
+ for (const panel of toolPanels) {
+ values[panel.id] =
+ storedToolValues[toolValueKey(model, "edit", panel.id)];
+ }
+ return values;
+ }, [toolPanels, storedToolValues, model]);
+ const handleToolValueChange = useCallback(
+ (panelId: string, value: unknown) =>
+ useAudioCloneStore
+ .getState()
+ .setToolValue(toolValueKey(model, "edit", panelId), value),
+ [model],
+ );
+ const claimedOptions = useMemo(
+ () => claimedOptionNames(toolPanels),
+ [toolPanels],
+ );
+ const advancedOptionSpecs = useMemo(
+ () => audioOptionSpecs.filter((spec) => !claimedOptions.has(spec.name)),
+ [audioOptionSpecs, claimedOptions],
+ );
+ const advancedRequest = useMemo(
+ () => audioOptionsForRequest(advancedOptionSpecs, audioOptionValues),
+ [advancedOptionSpecs, audioOptionValues],
+ );
+ const core: EditCoreInputs = useMemo(
+ () => ({
+ transcript,
+ edited,
+ mode,
+ speed: delivery.speed,
+ pitchSteps: delivery.pitchSteps,
+ }),
+ [transcript, edited, mode, delivery],
+ );
+ const toolRequest = useMemo(
+ () =>
+ collectToolRequest(
+ toolPanels,
+ toolValues,
+ { text: edited, edit: core },
+ toolContext,
+ audioOptionSpecs,
+ ),
+ [toolPanels, toolValues, edited, core, toolContext, audioOptionSpecs],
+ );
+
+ // An upload can pass its keep-until time while the page sits open.
+ const [now, setNow] = useState(() => Date.now());
+ useEffect(() => {
+ if (source?.kind !== "input" || !source.expiresAt) return;
+ const timer = window.setInterval(() => setNow(Date.now()), 60_000);
+ return () => window.clearInterval(timer);
+ }, [source]);
+ const sourceExpired =
+ sourceStatus.phase === "expired" ||
+ selectionExpired(source, now) ||
+ (source !== null && source.id === expiredSourceId);
+ const sourceBusy =
+ sourceStatus.phase === "uploading" || sourceStatus.phase === "recording";
+
+ // A new recording drops the old one's transcription and its error.
+ const cancelTranscribe = transcriber.cancel;
+ const sourceId = source?.id ?? null;
+ // biome-ignore lint/correctness/useExhaustiveDependencies: the recording id is the trigger.
+ useEffect(() => cancelTranscribe, [sourceId, cancelTranscribe]);
+
+ // ① transcribes itself once per recording.
+ const autoTranscribed = useRef<string | null>(null);
+ const transcribe = transcriber.transcribe;
+ useEffect(() => {
+ if (!source || sourceBusy || sourceExpired) return;
+ if (transcriptFor === source.id || transcript.trim()) return;
+ if (autoTranscribed.current === source.id) return;
+ autoTranscribed.current = source.id;
+ void transcribe(source);
+ }, [
+ source,
+ sourceBusy,
+ sourceExpired,
+ transcriptFor,
+ transcript,
+ transcribe,
+ ]);
+
+ const inputBlocker = editBlocker({
+ source,
+ sourceBusy,
+ sourceExpired,
+ sourceError: sourceStatus.phase === "error" ? sourceStatus.message : null,
+ sourceDurationS: source?.durationS ?? null,
+ transcribing: transcriber.transcribing,
+ transcript,
+ edited,
+ mode,
+ delivery,
+ panelError: toolRequest.error,
+ });
+ const runAction = useCallback(
+ (id: EditBlockerActionId) => {
+ switch (id) {
+ case "add-recording":
+ if (sourceExpired) useAudioEditStore.getState().setSource(null);
+ sourceHandle.current?.browse();
+ return;
+ case "choose-recording":
+ useAudioEditStore.getState().setSource(null);
+ sourceHandle.current?.browse();
+ return;
+ case "transcribe":
+ void transcribe(useAudioEditStore.getState().source);
+ return;
+ case "type-transcript":
+ document.getElementById(EDIT_TRANSCRIPT_FIELD_ID)?.focus();
+ return;
+ case "focus-changes":
+ changesRef.current?.focus();
+ return;
+ }
+ },
+ [sourceExpired, transcribe],
+ );
+ const blocker: GenerateBlocker | null = inputBlocker
+ ? {
+ reason: inputBlocker.reason,
+ actions: inputBlocker.actions.length
+ ? inputBlocker.actions.map((action) => ({
+ label: action.label,
+ onClick: () => runAction(action.id),
+ }))
+ : undefined,
+ }
+ : null;
+
+ const handleGenerate = useCallback(async () => {
+ const state = useAudioEditStore.getState();
+ if (!(state.source && adapter) || inputBlocker) return;
+ if (busyRef.current) return;
+ busyRef.current = "generating";
+ setBusy("generating");
+ updateGenerationPhase("preparing");
+ const finishEarly = () => {
+ updateGenerationPhase(null);
+ busyRef.current = null;
+ setBusy(null);
+ };
+ const releaseInFlight = pendingTranscribeRelease.current;
+ if (releaseInFlight && !(await releaseInFlight)) {
+ finishEarly();
+ setMode("transcribe");
+ return;
+ }
+ const missingOptions = missingRequiredAudioOptions(
+ advancedOptionSpecs,
+ audioOptionValues,
+ );
+ if (missingOptions.length > 0) {
+ finishEarly();
+ setAdvancedOpen(true);
+ toast.error(
+ `Set ${missingOptions.map((spec) => audioOptionLabel(spec.name)).join(", ")} in Advanced before generating.`,
+ );
+ return;
+ }
+ const run = buildEditRun(adapter, {
+ source: sourceRefOf(state.source),
+ transcript: state.transcript,
+ edited: state.edited,
+ mode,
+ delivery: state.delivery,
+ advanced: { ...advancedRequest, ...toolRequest.patch.options },
+ sourceName: state.source.name,
+ });
+ const controller = new AbortController();
+ generateAbort.current = controller;
+ setPhaseLabel(editPhaseLabel(adapter, run));
+ updateGenerationPhase("generating");
+ try {
+ const response = await runAudio(run, controller.signal);
+ updateGenerationPhase("finishing");
+ await showRunResult({
+ response,
+ text: run.text ?? "",
+ workflow: "edit",
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ });
+ } catch (error) {
+ if (!controller.signal.aborted) {
+ updateGenerationPhase("finishing");
+ const expired =
+ error instanceof AudioApiError &&
+ error.status === 404 &&
+ state.source.kind === "input";
+ if (expired) {
+ setExpiredSourceId(state.source.id);
+ sourceHandle.current?.markExpired();
+ }
+ const message = expired
+ ? REFERENCE_EXPIRED_MESSAGE
+ : error instanceof Error
+ ? error.message
+ : "Editing the recording failed.";
+ setGenerationError(message);
+ if (!expired) toast.error(message);
+ await refreshStatus();
+ }
+ } finally {
+ generateAbort.current = null;
+ setPhaseLabel(null);
+ updateGenerationPhase(null);
+ busyRef.current = null;
+ setBusy(null);
+ if (activeRef.current && modeRef.current === "speak")
+ replayQueuedTtsPick();
+ }
+ }, [
+ adapter,
+ inputBlocker,
+ busyRef,
+ setBusy,
+ updateGenerationPhase,
+ pendingTranscribeRelease,
+ setMode,
+ advancedOptionSpecs,
+ audioOptionValues,
+ setAdvancedOpen,
+ mode,
+ advancedRequest,
+ toolRequest,
+ generateAbort,
+ refreshGallery,
+ selectClip,
+ setFallbackClip,
+ setSelectedId,
+ refreshStatus,
+ activeRef,
+ modeRef,
+ replayQueuedTtsPick,
+ ]);
+
+ useEffect(() => {
+ if (source) setGenerationError(null);
+ }, [source]);
+
+ return {
+ source,
+ transcript,
+ edited,
+ mode,
+ delivery,
+ adapter,
+ sourceHandle,
+ changesRef,
+ sourceStatus,
+ setSourceStatus,
+ sourceExpired,
+ transcriber,
+ toolContext,
+ toolPanels,
+ toolValues,
+ handleToolValueChange,
+ claimedOptions,
+ advancedOptionSpecs,
+ core,
+ blocker,
+ inputsReady: inputBlocker === null,
+ handleGenerate,
+ generationError,
+ setGenerationError,
+ phaseLabel,
+ };
+}
+
+export type EditGeneration = ReturnType<typeof useEditGeneration>;
diff --git a/studio/frontend/src/features/audio/hooks/use-reference-transcribe.ts b/studio/frontend/src/features/audio/hooks/use-reference-transcribe.ts
index 8c212d0ab3a..36fc1d44c31 100644
--- a/studio/frontend/src/features/audio/hooks/use-reference-transcribe.ts
+++ b/studio/frontend/src/features/audio/hooks/use-reference-transcribe.ts
@@ -25,7 +25,9 @@ export function useReferenceTranscribe({
const transcribe = useCallback(
async (reference: AudioSourceSelection | null) => {
- if (!reference || abort.current) return;
+ if (!reference) return;
+ // A newer clip wins, so its own transcription is never skipped.
+ abort.current?.abort();
const controller = new AbortController();
abort.current = controller;
setTranscribing(true);
@@ -59,14 +61,23 @@ export function useReferenceTranscribe({
: "Could not transcribe the clip.",
);
} finally {
- if (abort.current === controller) abort.current = null;
- setTranscribing(false);
+ if (abort.current === controller) {
+ abort.current = null;
+ setTranscribing(false);
+ }
}
},
[sttRepo],
);
+ const cancel = useCallback(() => {
+ abort.current?.abort();
+ abort.current = null;
+ setTranscribing(false);
+ setError(null);
+ }, []);
+
useEffect(() => () => abort.current?.abort(), []);
- return { transcribe, transcribing, error };
+ return { transcribe, cancel, transcribing, error };
}
diff --git a/studio/frontend/src/features/audio/pages/edit-page.tsx b/studio/frontend/src/features/audio/pages/edit-page.tsx
new file mode 100644
index 00000000000..024331275d8
--- /dev/null
+++ b/studio/frontend/src/features/audio/pages/edit-page.tsx
@@ -0,0 +1,432 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Button } from "@/components/ui/button";
+import { Textarea } from "@/components/ui/textarea";
+import { ParamSlider } from "@/features/chat";
+import { PillTabs } from "@/features/model-picker/components/model-selector/pill-tabs";
+import type { ModelOption } from "@/features/model-picker/components/model-selector/types";
+import { SpeechToTextIcon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import { type ComponentProps, useEffect, useState } from "react";
+import { type AudioGalleryClip, fetchAudioBlob } from "../api";
+import { audioCppModelFor, audioCppWorkflowsFor } from "../audio-cpp-catalog";
+import { EDIT_SOURCE_MAX_SECONDS } from "../audio-run-request";
+import { macTtsCatalogChoiceIsRunnable } from "../catalog";
+import { ABCompare } from "../components/ab-compare";
+import {
+ AudioHistoryProvider,
+ AudioSourceInput,
+} from "../components/audio-source-input";
+import { TranscriptDiffEditor } from "../components/transcript-diff-editor";
+import { EDIT_COPY } from "../edit-policy";
+import {
+ EDIT_CHANGES_FIELD_ID,
+ EDIT_TRANSCRIPT_FIELD_ID,
+ type EditGeneration,
+ adoptEditSource,
+} from "../hooks/use-edit-generation";
+import { useAudioEditStore } from "../stores/audio-edit-store";
+import { AudioToolPanels } from "../tools/tool-panel-host";
+import { TtsFooter, TtsOutput, TtsRailFields } from "./tts-workspace";
+
+const EDIT_MODEL_ORDER = [
+ "DotTTS-Edit-GGUF",
+ "Vevo2-GGUF",
+ "FireRedAudio-GGUF",
+];
+
+function editRank(id: string): number {
+ const index = EDIT_MODEL_ORDER.findIndex((name) => id.endsWith(`/${name}`));
+ return index === -1 ? EDIT_MODEL_ORDER.length : index;
+}
+
+export function editPageModels(
+ models: ModelOption[],
+ isMac: boolean,
+): ModelOption[] {
+ return models
+ .filter((model) => {
+ const entry = audioCppModelFor(model.id);
+ return (
+ entry !== null &&
+ audioCppWorkflowsFor(entry).includes("edit") &&
+ (!isMac || macTtsCatalogChoiceIsRunnable(model.id))
+ );
+ })
+ .map((model, index) => ({ model, index }))
+ .sort(
+ (a, b) =>
+ editRank(a.model.id) - editRank(b.model.id) || a.index - b.index,
+ )
+ .map(({ model }) => model);
+}
+
+type RailProps = Omit<
+ ComponentProps<typeof TtsRailFields>,
+ | "musicGeneration"
+ | "prompt"
+ | "setPrompt"
+ | "toolPanels"
+ | "inputs"
+ | "claimedOptions"
+>;
+
+/** Selects ②'s last word so typing replaces it. */
+function selectLastWord(field: HTMLTextAreaElement | null) {
+ if (!field) return;
+ const match = /([\p{L}\p{N}'’-]+)[^\p{L}\p{N}]*$/u.exec(field.value);
+ field.focus();
+ if (match)
+ field.setSelectionRange(match.index, match.index + match[1].length);
+}
+
+function EditInputs({
+ edit,
+ historyClips,
+ disabled,
+}: {
+ edit: EditGeneration;
+ historyClips: readonly AudioGalleryClip[];
+ disabled: boolean;
+}) {
+ const setTranscript = useAudioEditStore((state) => state.setTranscript);
+ const setEdited = useAudioEditStore((state) => state.setEdited);
+ const resetEdited = useAudioEditStore((state) => state.resetEdited);
+ const setMode = useAudioEditStore((state) => state.setMode);
+ const setDelivery = useAudioEditStore((state) => state.setDelivery);
+ const { transcriber, source, adapter, mode, delivery } = edit;
+ const sourceUsable =
+ source !== null &&
+ !edit.sourceExpired &&
+ edit.sourceStatus.phase !== "uploading";
+ const deliveryRange = adapter?.delivery ?? null;
+ return (
+ <AudioHistoryProvider value={historyClips}>
+ <div data-tour="audio-edit-recording">
+ <AudioSourceInput
+ id="edit-recording"
+ label={EDIT_COPY.recordingLabel}
+ hint={EDIT_COPY.recordingHint}
+ value={source}
+ onChange={adoptEditSource}
+ disabled={disabled}
+ allowSavedVoice={false}
+ usesFirstSeconds={null}
+ maxRecordSeconds={EDIT_SOURCE_MAX_SECONDS}
+ handleRef={edit.sourceHandle}
+ onStatusChange={edit.setSourceStatus}
+ />
+ </div>
+
+ <div className="grid gap-1.5" data-tour="audio-edit-transcript">
+ <div className="flex items-center justify-between gap-2">
+ <label
+ htmlFor={EDIT_TRANSCRIPT_FIELD_ID}
+ className="text-ui-13 font-medium text-foreground"
+ >
+ {EDIT_COPY.transcriptLabel}
+ {mode === "delivery" ? (
+ <span className="ml-1.5 font-normal text-muted-foreground">
+ Optional
+ </span>
+ ) : null}
+ </label>
+ <Button
+ type="button"
+ variant="ghost"
+ size="sm"
+ className="h-auto px-2 py-1 text-ui-11p5"
+ disabled={!sourceUsable || transcriber.transcribing || disabled}
+ onClick={() => void transcriber.transcribe(source)}
+ >
+ <HugeiconsIcon icon={SpeechToTextIcon} className="size-3.5" />
+ {transcriber.transcribing ? "Transcribing…" : "Transcribe"}
+ </Button>
+ </div>
+ <Textarea
+ id={EDIT_TRANSCRIPT_FIELD_ID}
+ value={edit.transcript}
+ disabled={transcriber.transcribing || disabled}
+ aria-busy={transcriber.transcribing}
+ onChange={(event) =>
+ setTranscript(event.target.value, source?.id ?? null)
+ }
+ placeholder={
+ transcriber.transcribing
+ ? "Listening to the recording…"
+ : "What the recording says. It fills in once you add one."
+ }
+ className="min-h-16"
+ />
+ {transcriber.error ? (
+ <p
+ role="alert"
+ className="text-ui-11p5 leading-snug text-destructive"
+ >
+ {transcriber.error}{" "}
+ <Button
+ type="button"
+ variant="link"
+ className="h-auto p-0 text-ui-11p5 font-medium text-foreground"
+ onClick={() => void transcriber.transcribe(source)}
+ >
+ Try again
+ </Button>
+ </p>
+ ) : (
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {EDIT_COPY.transcriptHint}
+ </p>
+ )}
+ </div>
+
+ {deliveryRange ? (
+ <PillTabs
+ ariaLabel="What to change"
+ value={mode}
+ onValueChange={(next) => setMode(next as "words" | "delivery")}
+ disabled={disabled}
+ fit={true}
+ compact={true}
+ className="[&>button]:px-3"
+ tabs={[
+ { value: "words", label: EDIT_COPY.wordsTab },
+ { value: "delivery", label: EDIT_COPY.deliveryTab },
+ ]}
+ />
+ ) : null}
+
+ {mode === "words" ? (
+ <div className="grid min-w-0 gap-1.5">
+ <TranscriptDiffEditor
+ id={EDIT_CHANGES_FIELD_ID}
+ value={edit.edited}
+ onChange={setEdited}
+ transcript={edit.transcript}
+ onReset={resetEdited}
+ disabled={disabled || !edit.transcript.trim()}
+ textareaRef={edit.changesRef}
+ dataTour="audio-edit-changes"
+ />
+ {edit.transcript.trim() && edit.edited === edit.transcript ? (
+ <div className="flex flex-wrap gap-1.5" aria-label="Examples">
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ className="h-auto px-3 py-1 text-ui-11p5 font-normal"
+ disabled={disabled}
+ onClick={() => selectLastWord(edit.changesRef.current)}
+ >
+ Change one word
+ </Button>
+ </div>
+ ) : null}
+ </div>
+ ) : deliveryRange ? (
+ <div className="grid gap-3" data-tour="audio-edit-changes">
+ <p className="text-ui-13 font-medium text-foreground">
+ {EDIT_COPY.deliveryLabel}
+ </p>
+ <ParamSlider
+ label={EDIT_COPY.speedLabel}
+ value={delivery.speed}
+ min={deliveryRange.speed[0]}
+ max={deliveryRange.speed[1]}
+ step={deliveryRange.speed[2]}
+ displayValue={`${Number(delivery.speed.toFixed(2))}×`}
+ disabled={disabled}
+ onChange={(speed) => setDelivery({ speed })}
+ />
+ <ParamSlider
+ label={EDIT_COPY.pitchLabel}
+ value={delivery.pitchSteps}
+ min={0}
+ max={deliveryRange.pitchSteps[1]}
+ step={1}
+ displayValue={
+ delivery.pitchSteps > 0 ? `+${delivery.pitchSteps} steps` : "Off"
+ }
+ disabled={disabled}
+ onChange={(pitchSteps) => setDelivery({ pitchSteps })}
+ />
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {EDIT_COPY.pitchHint}
+ </p>
+ </div>
+ ) : null}
+ {adapter?.family === "vevo2" ? (
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {EDIT_COPY.vevo2SwitchNote}
+ </p>
+ ) : null}
+ </AudioHistoryProvider>
+ );
+}
+
+export function EditRail({
+ edit,
+ historyClips,
+ ...props
+}: RailProps & {
+ edit: EditGeneration;
+ historyClips: readonly AudioGalleryClip[];
+}) {
+ const disabled = props.busy === "generating";
+ return (
+ <TtsRailFields
+ {...props}
+ musicGeneration={false}
+ prompt={edit.edited}
+ setPrompt={useAudioEditStore.getState().setEdited}
+ claimedOptions={edit.claimedOptions}
+ inputs={
+ <EditInputs
+ edit={edit}
+ historyClips={historyClips}
+ disabled={disabled}
+ />
+ }
+ toolPanels={
+ <AudioToolPanels
+ workflow="edit"
+ ctx={edit.toolContext}
+ values={edit.toolValues}
+ onChange={edit.handleToolValueChange}
+ specs={props.audioOptionSpecs}
+ disabled={disabled}
+ core={{ text: edit.edited, edit: edit.core }}
+ />
+ }
+ />
+ );
+}
+
+export function EditFooter({
+ edit,
+ ...props
+}: Omit<
+ ComponentProps<typeof TtsFooter>,
+ | "prompt"
+ | "lyricsOptional"
+ | "musicNeedsDescription"
+ | "audioInstructions"
+ | "handleGenerate"
+> & { edit: EditGeneration }) {
+ return (
+ <TtsFooter
+ {...props}
+ prompt={edit.inputsReady ? "ready" : ""}
+ lyricsOptional={false}
+ musicNeedsDescription={false}
+ audioInstructions=""
+ handleGenerate={edit.handleGenerate}
+ />
+ );
+}
+
+function galleryFileUrl(id: string): string {
+ return `/api/inference/audio/gallery/${encodeURIComponent(id)}/file`;
+}
+
+function EditComparePlayer({
+ clip,
+ src,
+ focusRef,
+}: {
+ clip: AudioGalleryClip;
+ src: string;
+ focusRef: ((element: HTMLElement | null) => void) | undefined;
+}) {
+ const sourceId = clip.source_clip_id ?? null;
+ const [originalSrc, setOriginalSrc] = useState<string | null>(null);
+ useEffect(() => {
+ if (!sourceId) return;
+ let url: string | null = null;
+ let cancelled = false;
+ setOriginalSrc(null);
+ fetchAudioBlob(galleryFileUrl(sourceId))
+ .then((blob) => {
+ if (cancelled) return;
+ url = URL.createObjectURL(blob);
+ setOriginalSrc(url);
+ })
+ .catch(() => {
+ // Original deleted: the compare still plays the edit.
+ });
+ return () => {
+ cancelled = true;
+ if (url) URL.revokeObjectURL(url);
+ };
+ }, [sourceId]);
+ return (
+ <ABCompare
+ original={{
+ src: originalSrc,
+ fileUrl: sourceId ? galleryFileUrl(sourceId) : null,
+ label: "Original",
+ durationS: null,
+ }}
+ edited={{
+ src,
+ fileUrl: galleryFileUrl(clip.id),
+ label: "Edited",
+ durationS: clip.duration_s ?? null,
+ }}
+ autoFocusRef={focusRef}
+ />
+ );
+}
+
+export function EditOutput({
+ modelReady,
+ recommendedModels = [],
+ onPickModel,
+ ...props
+}: Omit<
+ ComponentProps<typeof TtsOutput>,
+ "emptyText" | "clipBadge" | "emptyActions" | "selectedPlayer"
+> & {
+ modelReady: boolean;
+ recommendedModels?: readonly ModelOption[];
+ onPickModel?: (id: string) => void;
+}) {
+ const picks = modelReady || !onPickModel ? [] : recommendedModels.slice(0, 3);
+ return (
+ <TtsOutput
+ {...props}
+ clipBadge={() => null}
+ selectedPlayer={(clip, src, focusRef) =>
+ clip.source_clip_id ? (
+ <EditComparePlayer clip={clip} src={src} focusRef={focusRef} />
+ ) : null
+ }
+ emptyActions={
+ picks.length > 0 ? (
+ <div
+ className="flex flex-wrap justify-center gap-2"
+ aria-label="Recommended models"
+ >
+ {picks.map((model) => (
+ <Button
+ key={model.id}
+ type="button"
+ variant="outline"
+ size="sm"
+ onClick={() => onPickModel?.(model.id)}
+ >
+ Use {model.name}
+ </Button>
+ ))}
+ </div>
+ ) : null
+ }
+ emptyText={
+ modelReady
+ ? EDIT_COPY.emptyText
+ : "Edited speech lands here. Load a model that can edit speech, add a recording, and press Generate."
+ }
+ />
+ );
+}
diff --git a/studio/frontend/src/features/audio/pages/tts-workspace.tsx b/studio/frontend/src/features/audio/pages/tts-workspace.tsx
index 5b939992c5d..868a4e77ccf 100644
--- a/studio/frontend/src/features/audio/pages/tts-workspace.tsx
+++ b/studio/frontend/src/features/audio/pages/tts-workspace.tsx
@@ -474,6 +474,7 @@ export function TtsOutput({
onFreshClipFocused,
announcement,
clipBadge,
+ selectedPlayer,
}: Pick<
AudioGallery,
| "srcById"
@@ -513,7 +514,26 @@ export function TtsOutput({
onFreshClipFocused: () => void;
announcement: string;
clipBadge?: (clip: AudioGalleryClip) => string | null;
+ /** Replaces the plain player; `focusRef` goes on its first control. */
+ selectedPlayer?: (
+ clip: AudioGalleryClip,
+ src: string,
+ focusRef: ((element: HTMLElement | null) => void) | undefined,
+ ) => ReactNode;
}) {
+ const focusFreshClip = (element: HTMLElement | null) => {
+ if (!element) return;
+ element.focus();
+ onFreshClipFocused();
+ };
+ const customPlayer =
+ selectedClip && selectedClipSrc && selectedPlayer
+ ? selectedPlayer(
+ selectedClip,
+ selectedClipSrc,
+ selectedClip.id === freshClipId ? focusFreshClip : undefined,
+ )
+ : null;
const [openGroups, setOpenGroups] = useState<ReadonlySet<string>>(
() => new Set(),
);
@@ -590,6 +610,7 @@ export function TtsOutput({
: null
}
menu={clipMenu(selectedClip, "row")}
+ player={customPlayer ?? undefined}
status={clipBadge?.(selectedClip) ?? undefined}
focusOnMount={selectedClip.id === freshClipId}
onFocused={onFreshClipFocused}
diff --git a/studio/frontend/src/features/audio/stores/audio-edit-store.ts b/studio/frontend/src/features/audio/stores/audio-edit-store.ts
new file mode 100644
index 00000000000..1d7d984b6a1
--- /dev/null
+++ b/studio/frontend/src/features/audio/stores/audio-edit-store.ts
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { create } from "zustand";
+import { persist } from "zustand/middleware";
+import type { AudioSourceSelection } from "../audio-run-request";
+import type { EditDelivery, EditMode } from "../edit-adapters";
+
+export const AUDIO_EDIT_STORAGE_KEY = "unsloth_audio_edit_v1";
+
+/** Tool values live in the Clone store with every page's. */
+interface AudioEditState {
+ source: AudioSourceSelection | null;
+ transcript: string;
+ /** The source id the transcript came from, so a new recording gets transcribed again. */
+ transcriptFor: string | null;
+ edited: string;
+ /** Until the user types in ②, ② follows ①. */
+ editedTouched: boolean;
+ mode: EditMode;
+ delivery: EditDelivery;
+ setSource: (source: AudioSourceSelection | null) => void;
+ setTranscript: (transcript: string, transcriptFor?: string | null) => void;
+ setEdited: (edited: string) => void;
+ resetEdited: () => void;
+ setMode: (mode: EditMode) => void;
+ setDelivery: (delivery: Partial<EditDelivery>) => void;
+}
+
+export const useAudioEditStore = create<AudioEditState>()(
+ persist(
+ (set) => ({
+ source: null,
+ transcript: "",
+ transcriptFor: null,
+ edited: "",
+ editedTouched: false,
+ mode: "words",
+ delivery: { speed: 1, pitchSteps: 0 },
+ setSource: (source) => set({ source }),
+ setTranscript: (transcript, transcriptFor) =>
+ set((state) => ({
+ transcript,
+ ...(transcriptFor !== undefined ? { transcriptFor } : {}),
+ ...(state.editedTouched ? {} : { edited: transcript }),
+ })),
+ setEdited: (edited) => set({ edited, editedTouched: true }),
+ resetEdited: () =>
+ set((state) => ({ edited: state.transcript, editedTouched: false })),
+ setMode: (mode) => set({ mode }),
+ setDelivery: (delivery) =>
+ set((state) => ({ delivery: { ...state.delivery, ...delivery } })),
+ }),
+ {
+ name: AUDIO_EDIT_STORAGE_KEY,
+ version: 1,
+ partialize: (state) => ({
+ source: state.source,
+ transcript: state.transcript,
+ transcriptFor: state.transcriptFor,
+ edited: state.edited,
+ editedTouched: state.editedTouched,
+ mode: state.mode,
+ delivery: state.delivery,
+ }),
+ },
+ ),
+);
diff --git a/studio/frontend/src/features/audio/tools/edit-panel-logic.ts b/studio/frontend/src/features/audio/tools/edit-panel-logic.ts
new file mode 100644
index 00000000000..cd3bf3480ff
--- /dev/null
+++ b/studio/frontend/src/features/audio/tools/edit-panel-logic.ts
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// JSX-free so the node test runner can load it; edit-panels.tsx adds the Components.
+
+import {
+ DELIVERY_NEEDS_FIRERED,
+ EDIT_ADAPTERS,
+ type EditAdapter,
+} from "../edit-adapters";
+import type { AudioToolPanel } from "./types";
+
+export type EditPanelLogic = Omit<AudioToolPanel<null>, "Component"> & {
+ adapter: EditAdapter;
+};
+
+function editPanelLogic(adapter: EditAdapter): EditPanelLogic {
+ const family = adapter.family;
+ return {
+ id: `edit-${family}`,
+ families: [family],
+ workflows: ["edit"],
+ title: "How edits apply",
+ claims: adapter.claims,
+ adapter,
+ // DotTTS-MF is dots_tts too but cannot edit.
+ appliesTo: (ctx) =>
+ ctx.audioFamily === family &&
+ ctx.audioWorkflows?.includes("edit") === true,
+ initial: () => null,
+ // buildEditRun builds the edit part from the page's inputs.
+ toRequest: () => ({}),
+ validate: (_value, core) => {
+ const edit = core.edit;
+ if (!edit) return null;
+ if (edit.mode === "delivery")
+ return adapter.delivery ? null : DELIVERY_NEEDS_FIRERED;
+ return adapter.validateWords(edit.transcript, edit.edited);
+ },
+ };
+}
+
+export const EDIT_PANEL_LOGIC: readonly EditPanelLogic[] =
+ Object.values(EDIT_ADAPTERS).map(editPanelLogic);
diff --git a/studio/frontend/src/features/audio/tools/edit-panels.tsx b/studio/frontend/src/features/audio/tools/edit-panels.tsx
new file mode 100644
index 00000000000..6c0dc255933
--- /dev/null
+++ b/studio/frontend/src/features/audio/tools/edit-panels.tsx
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { deliveryInstructions } from "../edit-adapters";
+import { countChanges } from "../edit-diff";
+import { EDIT_COPY } from "../edit-policy";
+import { EDIT_PANEL_LOGIC } from "./edit-panel-logic";
+import type { AudioToolPanel } from "./types";
+
+export const EDIT_TOOL_PANELS: readonly AudioToolPanel<null>[] =
+ EDIT_PANEL_LOGIC.map((logic) => ({
+ ...logic,
+ Component: ({ core }) => {
+ const edit = core?.edit;
+ const changes = !edit
+ ? 0
+ : edit.mode === "delivery" && logic.adapter.delivery
+ ? deliveryInstructions(edit.speed, edit.pitchSteps).length
+ : (countChanges(edit.transcript, edit.edited) ?? 0);
+ return (
+ <section
+ className="grid gap-0.5"
+ aria-label={EDIT_COPY.howItEditsTitle}
+ >
+ <h3 className="text-ui-13 font-medium text-foreground">
+ {EDIT_COPY.howItEditsTitle}
+ </h3>
+ <p className="text-ui-11p5 leading-snug text-muted-foreground">
+ {logic.adapter.howItEdits(changes)}
+ </p>
+ </section>
+ );
+ },
+ }));
diff --git a/studio/frontend/src/features/audio/tools/registry.tsx b/studio/frontend/src/features/audio/tools/registry.tsx
index 8cef957063a..682be6c0dce 100644
--- a/studio/frontend/src/features/audio/tools/registry.tsx
+++ b/studio/frontend/src/features/audio/tools/registry.tsx
@@ -8,6 +8,7 @@ import {
} from "../components/instructions-fields";
import type { AudioWorkflowId } from "../workflows";
import { CLONE_TOOL_PANELS } from "./clone-panels";
+import { EDIT_TOOL_PANELS } from "./edit-panels";
import { MUSIC_TOOL_PANELS } from "./music-panels";
import { SEPARATE_TOOL_PANELS } from "./separate-panels";
import {
@@ -115,6 +116,7 @@ export const AUDIO_TOOL_PANELS: readonly AnyAudioToolPanel[] = [
...INSTRUCTION_PANELS,
...SPEAK_TOOL_PANELS.slice(1),
...CLONE_TOOL_PANELS,
+ ...EDIT_TOOL_PANELS,
...MUSIC_TOOL_PANELS,
...SEPARATE_TOOL_PANELS,
];
diff --git a/studio/frontend/src/features/audio/tools/types.ts b/studio/frontend/src/features/audio/tools/types.ts
index 4cd43ab9ed7..55cb54f0c99 100644
--- a/studio/frontend/src/features/audio/tools/types.ts
+++ b/studio/frontend/src/features/audio/tools/types.ts
@@ -38,6 +38,15 @@ export interface CoreInputs {
text: string;
referenceText?: string;
hasReference?: boolean;
+ edit?: EditCoreInputs;
+}
+
+export interface EditCoreInputs {
+ transcript: string;
+ edited: string;
+ mode: "words" | "delivery";
+ speed: number;
+ pitchSteps: number;
}
export interface AudioToolPanelProps<V> {
diff --git a/studio/frontend/src/features/audio/tour/steps.tsx b/studio/frontend/src/features/audio/tour/steps.tsx
index af12d21bfaf..3c15a96d8cd 100644
--- a/studio/frontend/src/features/audio/tour/steps.tsx
+++ b/studio/frontend/src/features/audio/tour/steps.tsx
@@ -24,6 +24,7 @@ const MODEL_STEP_BODY: Record<AudioWorkflowId, string> = {
"Text-to-speech models, including voices you fine-tuned under On Device.",
clone:
"Models that can speak in the voice of a short recording you give them.",
+ edit: "Models that can change words in a recording and keep the voice.",
music: "Music models. Loading one replaces the model in the main slot.",
separate:
"Source separation models. Loading one replaces the model in the main slot.",
@@ -141,6 +142,47 @@ export function buildAudioTourSteps({
];
}
+ if (workflow === "edit") {
+ return [
+ modeStep,
+ modelStep(workflow),
+ {
+ id: "recording",
+ target: "audio-edit-recording",
+ title: "Recording",
+ body: (
+ <>
+ Upload, record or pick up to 30 seconds of one voice from your
+ history.
+ </>
+ ),
+ },
+ {
+ id: "transcript",
+ target: "audio-edit-transcript",
+ title: "Check the transcript",
+ body: (
+ <>
+ It fills in by itself. Fix any word the recognizer got wrong so it
+ matches what's said.
+ </>
+ ),
+ },
+ {
+ id: "changes",
+ target: "audio-edit-changes",
+ title: "Make your changes",
+ body: (
+ <>
+ Change, add or remove words. The voice stays the same, and the
+ result plays against the original.
+ </>
+ ),
+ },
+ outputStep,
+ ];
+ }
+
return [
modeStep,
modelStep(workflow),
diff --git a/studio/frontend/src/features/audio/workflows.ts b/studio/frontend/src/features/audio/workflows.ts
index 3c5fa9549ff..20d35e5e1d7 100644
--- a/studio/frontend/src/features/audio/workflows.ts
+++ b/studio/frontend/src/features/audio/workflows.ts
@@ -6,6 +6,7 @@
import {
AiVoiceIcon,
MusicNote03Icon,
+ QuillWrite01Icon,
SpeechToTextIcon,
SplitIcon,
VoiceIdIcon,
@@ -16,6 +17,7 @@ import { AUDIO_CPP_MUSIC_AUDIO_TYPE } from "./audio-cpp-catalog";
export type AudioWorkflowId =
| "speak"
| "clone"
+ | "edit"
| "music"
| "separate"
| "transcribe";
@@ -49,6 +51,15 @@ export const AUDIO_WORKFLOWS: ReadonlyArray<{
slot: "speak",
createTrain: true,
},
+ {
+ id: "edit",
+ label: "Edit",
+ heading: "Edit speech",
+ icon: QuillWrite01Icon,
+ hint: "Change words in a recording, same voice",
+ slot: "speak",
+ createTrain: false,
+ },
{
id: "music",
label: "Music",
@@ -120,10 +131,11 @@ export function audioWorkflowForAudioType(
export function clipWorkflow(clip: {
workflow?: string | null;
audio_type?: string | null;
-}): "speak" | "clone" | "music" | "separate" {
+}): "speak" | "clone" | "edit" | "music" | "separate" {
if (
clip.workflow === "speak" ||
clip.workflow === "clone" ||
+ clip.workflow === "edit" ||
clip.workflow === "music" ||
clip.workflow === "separate"
) {
diff --git a/studio/frontend/tests/audio-edit-adapters.test.ts b/studio/frontend/tests/audio-edit-adapters.test.ts
new file mode 100644
index 00000000000..825f49ee341
--- /dev/null
+++ b/studio/frontend/tests/audio-edit-adapters.test.ts
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// fixtures/audio-edit-requests.json is shared with the backend's test_audio_edit_requests.py.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readText, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const A = await import("../src/features/audio/edit-adapters.ts");
+const D = await import("../src/features/audio/edit-diff.ts");
+const { EDIT_ADAPTERS } = A;
+const R = await import("../src/features/audio/audio-run-request.ts");
+
+type RunInput = Parameters<typeof A.buildEditRun>[1];
+type Input = RunInput & { original: string };
+const { cases } = JSON.parse(readText("fixtures/audio-edit-requests.json")) as {
+ cases: { name: string; family: string; input: Input; run_body: object }[];
+};
+const S2 = "Okay, I'm Cemo and what you just heard wasn't a human voice.";
+const sub = (word: string) => S2.replace("human", word);
+const LONG = Array.from(
+ { length: D.EDIT_DIFF_MAX_WORDS + 1 },
+ (_, i) => `w${i}`,
+).join(" ");
+const TWELVE = "one two three four five six seven eight nine ten eleven twelve";
+const FIVE = "ONE two THREE four FIVE six SEVEN eight NINE ten eleven twelve";
+const SIX = FIVE.replace("eleven", "ELEVEN");
+
+const words = (text: string) => (text ? text.split(" ") : []);
+const change = (
+ kind: string,
+ old: string,
+ add: string,
+ before: string | null,
+ index: number,
+) => ({ kind, old: words(old), new: words(add), before, index });
+
+const build = (family: string, input: Partial<RunInput>) =>
+ A.buildEditRun(EDIT_ADAPTERS[family], {
+ transcript: S2,
+ edited: S2,
+ mode: "words",
+ ...input,
+ });
+const runFor = (family: string, { original, ...input }: Input) =>
+ build(family, {
+ ...input,
+ source: { input_id: "in_0123456789abcdef" },
+ transcript: original,
+ delivery: input.delivery ?? null,
+ });
+
+for (const { name, family, input, run_body } of cases) {
+ test(`golden: ${name}`, () => {
+ const adapter = EDIT_ADAPTERS[family];
+ assert.equal(adapter.validateWords(input.original, input.edited), null);
+ const run = runFor(family, input);
+ assert.deepEqual(run, run_body);
+ assert.deepEqual(R.buildAudioRunBody(run), run_body);
+ });
+}
+
+test("each family's word checks", () => {
+ const rows: [string, string, string, string | null][] = [
+ ["firered_audio", S2, `${S2} Really.`, A.FIRERED_INSERT_AT_END],
+ ["dots_tts", S2, `${S2} Really.`, null],
+ ["firered_audio", TWELVE, FIVE, null],
+ ["firered_audio", TWELVE, SIX, A.FIRERED_TOO_MANY_CHANGES],
+ ["dots_tts", TWELVE, SIX, null],
+ ["dots_tts", S2, sub('"robot"'), A.DOTS_BAD_CHARACTERS],
+ ["dots_tts", S2, sub("<robot>"), A.DOTS_BAD_CHARACTERS],
+ ["dots_tts", S2, sub("ro>bot"), A.DOTS_BAD_CHARACTERS],
+ ["dots_tts", `<b> ${S2}`, `<b> ${sub("robot")}`, A.DOTS_ANGLE_BRACKETS],
+ ["firered_audio", S2, sub('"robot"'), null],
+ ];
+ for (const [family, original, edited, reason] of rows) {
+ const got = EDIT_ADAPTERS[family].validateWords(original, edited);
+ assert.equal(got, reason, `${family}: ${edited}`);
+ }
+ for (const adapter of Object.values(EDIT_ADAPTERS)) {
+ assert.equal(adapter.validateWords(LONG, `${LONG} x`), A.EDIT_TOO_LONG);
+ }
+});
+
+test("run bodies: Advanced minus claims, Vevo2, and Delivery", () => {
+ const advanced = { num_inference_steps: 4, guidance_scale: 1.2, flag: true };
+ const claimed = { template_name: "x", instruction: "Speak faster." };
+ const fire = build("firered_audio", {
+ edited: sub("robot"),
+ advanced: { ...advanced, ...claimed },
+ });
+ assert.deepEqual(fire.options, advanced);
+ const vevo = build("vevo2", { source: { clip_id: "c1" }, transcript: " " });
+ assert.deepEqual(vevo.edit, { mode: "words" });
+ assert.equal(vevo.inputs?.reference_text, undefined);
+ const pitchOnly = build("firered_audio", {
+ transcript: "",
+ edited: "",
+ mode: "delivery",
+ delivery: { speed: 1, pitchSteps: 2 },
+ sourceName: "take-3.wav",
+ });
+ assert.equal(pitchOnly.text, "take-3.wav");
+ assert.deepEqual(pitchOnly.edit, { mode: "delivery", pitch_steps: 2 });
+ assert.equal(pitchOnly.inputs?.source, undefined);
+ // Delivery on a model without it falls back to Words.
+ const dots = build("dots_tts", {
+ mode: "delivery",
+ delivery: { speed: 1.5, pitchSteps: 3 },
+ });
+ assert.deepEqual(dots.edit, { mode: "words" });
+ assert.deepEqual(A.deliveryInstructions(1, 0), []);
+ assert.deepEqual(A.deliveryInstructions(2, 3), [
+ "adjust the speed to 2x",
+ "shift the pitch by 3 steps",
+ ]);
+ assert.deepEqual(A.deliveryInstructions(0.7000000000000001, 0), [
+ "adjust the speed to 0.7x",
+ ]);
+});
+
+test("the adapter follows the loaded model's family and edit workflow", () => {
+ const rows: [string, string[] | undefined, boolean][] = [
+ ["dots_tts", ["edit"], true],
+ ["vevo2", ["edit"], true],
+ ["firered_audio", ["edit"], true],
+ // DotTTS-MF is dots_tts but cannot edit.
+ ["dots_tts", ["speak"], false],
+ ["qwen3_tts", ["clone", "edit"], false],
+ ["toString", ["edit"], false],
+ ["vevo2", undefined, false],
+ ];
+ for (const [audioFamily, audioWorkflows, found] of rows) {
+ const adapter = A.editAdapterFor({ audioFamily, audioWorkflows });
+ assert.equal(adapter, found ? EDIT_ADAPTERS[audioFamily] : null);
+ }
+});
+
+test("markup and the chained-run progress line", () => {
+ assert.equal(A.markupFor("a b c", "a b c"), "a b c");
+ assert.equal(A.markupFor("a b", "x a b"), "<ins>x</ins> a b");
+ const fire = runFor("firered_audio", cases[4].input);
+ assert.equal(
+ A.editPhaseLabel(EDIT_ADAPTERS.firered_audio, fire),
+ "Applying 2 changes, one pass each",
+ );
+ const dots = runFor("dots_tts", cases[0].input);
+ assert.equal(A.editPhaseLabel(EDIT_ADAPTERS.dots_tts, dots), null);
+});
+
+test("words are whitespace tokens; punctuation stays on its word", () => {
+ assert.deepEqual(D.tokenizeWords(" Okay, I'm\nCemo. "), [
+ "Okay,",
+ "I'm",
+ "Cemo.",
+ ]);
+ assert.deepEqual(D.tokenizeWords(" "), []);
+ assert.equal(D.countChanges(S2, ` ${S2.replaceAll(" ", " ")} `), 0);
+ assert.equal(D.countChanges("", ""), 0);
+});
+
+test("changes group per run of words, anchored on the next word", () => {
+ const s2 = (from: string, to: string) => S2.replace(from, to);
+ const rows: [string, ReturnType<typeof change>[], string?][] = [
+ [s2("voice.", "voice!"), [change("replace", "voice.", "voice!", null, 11)]],
+ [s2("human", "robot"), [change("replace", "human", "robot", "voice.", 10)]],
+ [
+ s2("just heard", "heard really"),
+ [
+ change("delete", "just", "", "heard", 6),
+ change("insert", "", "really", "wasn't", 8),
+ ],
+ ],
+ ["a d", [change("delete", "b c", "", "d", 1)], "a b c d"],
+ ["a b x y", [change("insert", "", "x y", null, 2)], "a b"],
+ ];
+ for (const [b, changes, a = S2] of rows) {
+ assert.deepEqual(D.changesBetween(a, b), changes, b);
+ }
+ // An adjacent delete and insert merge into one replace.
+ const ops = D.diffWords("a human voice.", "a robot sound.");
+ assert.ok(ops);
+ assert.deepEqual(D.groupChanges(ops), [
+ change("replace", "human voice.", "robot sound.", null, 1),
+ ]);
+});
+
+test("past the word cap the diff is not computed", () => {
+ assert.equal(D.diffWords("short", LONG), null);
+ assert.equal(D.countChanges(LONG, LONG), null);
+ assert.equal(D.diffSegments(LONG, "x"), null);
+ const atCap = LONG.split(" ").slice(1).join(" ");
+ assert.equal(D.countChanges(atCap, atCap), 0);
+});
+
+test("segments carry the inserted and deleted labels for screen readers", () => {
+ assert.deepEqual(D.diffSegments("a human voice", "a robot voice really"), [
+ { kind: "equal", text: "a", srLabel: null },
+ { kind: "delete", text: "human", srLabel: "deleted " },
+ { kind: "insert", text: "robot", srLabel: "inserted " },
+ { kind: "equal", text: "voice", srLabel: null },
+ { kind: "insert", text: "really", srLabel: "inserted " },
+ ]);
+});
diff --git a/studio/frontend/tests/audio-edit-host.test.ts b/studio/frontend/tests/audio-edit-host.test.ts
new file mode 100644
index 00000000000..e11771034a0
--- /dev/null
+++ b/studio/frontend/tests/audio-edit-host.test.ts
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const { INITIAL_AB_STATE, clampPosition, resumeAfterLoad, switchSide } =
+ await import("../src/features/audio/components/ab-compare-state.ts");
+const compare = readSrc("features/audio/components/ab-compare.tsx");
+const page = readSrc("features/audio/pages/edit-page.tsx");
+
+test("the recording takes uploads, recordings and history, not saved voices", () => {
+ assert.match(page, /id="edit-recording"[\s\S]*?allowSavedVoice=\{false\}/);
+ // A long recording is refused, so the card does not promise its first 30 s.
+ assert.match(page, /id="edit-recording"[\s\S]*?usesFirstSeconds=\{null\}/);
+ assert.match(page, /maxRecordSeconds=\{EDIT_SOURCE_MAX_SECONDS\}/);
+ // History hides an edit's original.
+ const gallery = readSrc("features/audio/hooks/use-audio-gallery.tsx");
+ assert.match(gallery, /clip\.role !== "source"/);
+});
+
+test("a history pick fills the transcript only with speech, not a Music description or a file name", async () => {
+ const { clipReference } = await import(
+ "../src/features/audio/audio-run-request.ts"
+ );
+ const clip = { id: "c1", duration_s: 4, reference_name: "take.wav" };
+ const rows: [string, string, string | null][] = [
+ ["Hello there.", "edit", "Hello there."],
+ ["calm piano", "music", null],
+ ["take.wav", "edit", null],
+ ];
+ for (const [prompt, workflow, transcript] of rows) {
+ const ref = clipReference({ ...clip, prompt, workflow });
+ assert.equal(ref.transcript, transcript, prompt);
+ }
+ assert.equal(clipReference({ ...clip, prompt: "take.wav" }).name, "take.wav");
+});
+
+test("a new recording cancels the previous one's transcription", () => {
+ const hook = readSrc("features/audio/hooks/use-edit-generation.ts");
+ assert.match(
+ hook,
+ /useEffect\(\(\) => cancelTranscribe, \[sourceId, cancelTranscribe\]\);/,
+ );
+});
+
+test("a result opens on Edited at 0; switching keeps the moment and the playing flag", () => {
+ assert.deepEqual(INITIAL_AB_STATE, {
+ side: "edited",
+ position: 0,
+ playing: false,
+ });
+ const playing = { side: "edited" as const, position: 2.5, playing: true };
+ const toOriginal = switchSide(playing, "original", 4.7);
+ assert.deepEqual(toOriginal, { ...playing, side: "original" });
+ const paused = { ...playing, playing: false };
+ assert.equal(switchSide(paused, "original", 4.7).playing, false);
+ assert.equal(switchSide(playing, "edited", 1), playing);
+ // Clamped to the other clip's length.
+ const late = { side: "original" as const, position: 4.6, playing: true };
+ assert.equal(switchSide(late, "edited", 3.2).position, 3.2);
+ // Unknown length keeps the position; the element clamps it once loaded.
+ assert.equal(switchSide(late, "edited", null).position, 4.6);
+ assert.equal(clampPosition(-1, 3), 0);
+ assert.equal(clampPosition(Number.NaN, 3), 0);
+});
+
+test("after loading, the clip resumes at the kept moment, or the top when past its end", () => {
+ const state = { side: "edited" as const, position: 2, playing: true };
+ const rows: [typeof state, number | null, object][] = [
+ [state, 3.2, { time: 2, play: true }],
+ [{ ...state, position: 3.2 }, 3.2, { time: 0, play: true }],
+ [{ ...state, playing: false }, null, { time: 2, play: false }],
+ ];
+ for (const [at, duration, resume] of rows) {
+ assert.deepEqual(resumeAfterLoad(at, duration), resume);
+ }
+});
+
+test("the A/B player is one keyboard-operable slider over a single audio element", () => {
+ assert.match(compare, /role="slider"/);
+ for (const key of ['" "', '"ArrowRight"', '"ArrowLeft"', '"Home"']) {
+ assert.ok(compare.includes(`event.key === ${key}`), key);
+ }
+ assert.equal(compare.match(/<audio\b/g)?.length, 1);
+ // A new pair resets the player; the Original's file arriving does not.
+ assert.match(compare, /\}, \[originalKey, editedKey\]\);/);
+});
+
+test("the bars come from the server's file, since the CSP blocks fetching object URLs", () => {
+ assert.match(compare, /if \(src\.startsWith\("blob:"\)\) return null;/);
+ assert.match(page, /fileUrl: galleryFileUrl\(clip\.id\)/);
+});
diff --git a/studio/frontend/tests/audio-edit-policy.test.ts b/studio/frontend/tests/audio-edit-policy.test.ts
new file mode 100644
index 00000000000..82e7a9a174b
--- /dev/null
+++ b/studio/frontend/tests/audio-edit-policy.test.ts
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const L = await import("../src/features/audio/tools/edit-panel-logic.ts");
+const S = await import("../src/features/audio/tools/select.ts");
+const P = await import("../src/features/audio/edit-policy.ts");
+const A = await import("../src/features/audio/edit-adapters.ts");
+
+type Input = Parameters<typeof P.editBlocker>[0];
+type Action = { id: string; label: string };
+
+const S2 = "Okay, I'm Cemo and what you just heard wasn't a human voice.";
+const EDITED = S2.replace("human", "robot");
+const LONG = Array.from({ length: 401 }, (_, i) => `w${i}`).join(" ");
+const ready: Input = {
+ source: { kind: "input", id: "in_1", name: "take.wav", durationS: 4.7 },
+ sourceBusy: false,
+ sourceExpired: false,
+ sourceError: null,
+ sourceDurationS: 4.7,
+ transcribing: false,
+ transcript: S2,
+ edited: EDITED,
+ mode: "words",
+ delivery: { speed: 1, pitchSteps: 0 },
+ panelError: null,
+};
+
+const NONE = { source: null, sourceDurationS: null };
+// Mid-transcription, before any transcript.
+const BLANK = { transcript: "", edited: "", transcribing: true };
+
+// Actions as ids, or as full objects where the label matters.
+const check = (rows: [Input, string | null, (string | Action)[]?][]) => {
+ for (const [input, reason, actions] of rows) {
+ const blocker = P.editBlocker(input);
+ assert.equal(blocker?.reason ?? null, reason);
+ if (actions) {
+ const got = blocker?.actions ?? [];
+ const ids = typeof actions[0] === "string";
+ assert.deepEqual(ids ? got.map((a) => a.id) : got, actions);
+ }
+ }
+};
+
+test("Edit's blockers come in rail order, each with its action", () => {
+ // Every step is wrong at once; fixing one reveals the next.
+ const empty: Input = { ...ready, ...NONE, ...BLANK, panelError: "no" };
+ const picked = { ...empty, source: ready.source, sourceDurationS: 31 };
+ // A take the 30 s timer stopped can run a frame over; the server allows that too.
+ const fits = { ...picked, sourceDurationS: 30.02 };
+ const transcribed = { ...fits, transcribing: false };
+ const typed = {
+ ...transcribed,
+ transcript: S2,
+ edited: `${S2} `,
+ };
+ const changed = { ...typed, edited: EDITED };
+ assert.equal(P.editBlocker(changed)?.kind, "panel");
+ check([
+ [empty, P.EDIT_NO_SOURCE, ["add-recording"]],
+ [{ ...empty, sourceError: "Not audio." }, "Not audio."],
+ // Uploading or recording: wait, rather than ask for a recording.
+ [{ ...empty, sourceBusy: true }, P.EDIT_SOURCE_BUSY],
+ [
+ { ...picked, sourceExpired: true },
+ P.EDIT_SOURCE_EXPIRED,
+ [{ id: "add-recording", label: "Add it again" }],
+ ],
+ [picked, P.EDIT_SOURCE_TOO_LONG, ["choose-recording"]],
+ [fits, P.EDIT_TRANSCRIBING],
+ [transcribed, P.EDIT_TRANSCRIPT_EMPTY, ["transcribe", "type-transcript"]],
+ [typed, P.EDIT_NO_CHANGES, [{ id: "focus-changes", label: "Go to ②" }]],
+ // Deleting every word would send an empty text the run route refuses with a 422.
+ [{ ...typed, edited: " " }, P.EDIT_CHANGES_EMPTY],
+ [changed, "no"],
+ [
+ { ...changed, panelError: A.FIRERED_TOO_MANY_CHANGES },
+ A.FIRERED_TOO_MANY_CHANGES,
+ ],
+ [{ ...changed, panelError: null }, null],
+ [ready, null],
+ // An unknown length does not block; the server checks it again.
+ [{ ...ready, sourceDurationS: null }, null],
+ [{ ...ready, transcript: LONG, edited: `${LONG} x` }, A.EDIT_TOO_LONG],
+ ]);
+});
+
+test("Delivery needs a speed or a pitch change, but no transcript", () => {
+ const delivery: Input = { ...ready, ...BLANK, mode: "delivery" };
+ const needsFireRed = "Delivery changes need FireRedAudio.";
+ check([
+ [delivery, P.EDIT_DELIVERY_EMPTY],
+ [{ ...delivery, delivery: { speed: 1.5, pitchSteps: 0 } }, null],
+ [{ ...delivery, delivery: { speed: 1, pitchSteps: 3 } }, null],
+ [{ ...delivery, panelError: needsFireRed }, needsFireRed],
+ [{ ...delivery, ...NONE }, P.EDIT_NO_SOURCE],
+ ]);
+});
+
+const page = {
+ musicGeneration: false,
+ cudaMusicGeneration: false,
+ musicNeedsDescription: false,
+};
+const ctx = (audio_family: string, ...audio_workflows: string[]) =>
+ S.audioModelContextFor(
+ { audio_type: "audiocpp_tts", audio_family, audio_workflows },
+ page,
+ );
+type Ctx = ReturnType<typeof ctx>;
+type Inputs = Parameters<typeof S.collectToolRequest>[2];
+const DOTS = ctx("dots_tts", "speak", "edit");
+const VEVO = ctx("vevo2", "clone", "edit");
+const FIRE = ctx("firered_audio", "clone", "edit");
+
+const core = (edited: string, mode: "words" | "delivery" = "words") => ({
+ text: edited,
+ edit: { transcript: S2, edited, mode, speed: 1, pitchSteps: 0 },
+});
+const applying = (c: Ctx, workflow = "edit") =>
+ L.EDIT_PANEL_LOGIC.filter((panel) =>
+ S.panelApplies(panel, workflow as "edit", c),
+ );
+
+test("each edit panel applies only to its family, and only when the model can edit", () => {
+ const rows: [Ctx, string, string[]][] = [
+ [DOTS, "edit", ["edit-dots_tts"]],
+ [VEVO, "edit", ["edit-vevo2"]],
+ [FIRE, "edit", ["edit-firered_audio"]],
+ // DotTTS-MF is dots_tts but lists no edit.
+ [ctx("dots_tts", "speak"), "edit", []],
+ [ctx("qwen3_tts", "clone", "edit"), "edit", []],
+ [DOTS, "speak", []],
+ [VEVO, "clone", []],
+ ];
+ for (const [c, workflow, ids] of rows) {
+ const got = applying(c, workflow).map((panel) => panel.id);
+ assert.deepEqual(got, ids, `${c.audioFamily} on ${workflow}`);
+ }
+});
+
+test("the panels claim the options the adapters set, so Advanced hides them", () => {
+ const claims = (c: Ctx) => [...S.claimedOptionNames(applying(c))];
+ const fire = ["template_name", "instruction"];
+ assert.deepEqual(claims(FIRE), fire);
+ for (const name of [...fire, "source_text", "target_text"]) {
+ assert.ok(claims(DOTS).includes(name), name);
+ }
+});
+
+test("collectToolRequest returns the adapter's reason and adds nothing to the request", () => {
+ // Six separate changes: every other word.
+ const six = "OKAY, I'm CEMO and WHAT you JUST heard WASN'T a HUMAN voice.";
+ const rows: [Ctx, Inputs, string | null][] = [
+ [FIRE, core(`${S2} Really.`), A.FIRERED_INSERT_AT_END],
+ [FIRE, core(six), A.FIRERED_TOO_MANY_CHANGES],
+ [FIRE, core(EDITED), null],
+ [DOTS, core(S2.replace("human", "<robot>")), A.DOTS_BAD_CHARACTERS],
+ [DOTS, core(S2, "delivery"), A.DELIVERY_NEEDS_FIRERED],
+ [FIRE, core(S2, "delivery"), null],
+ // Without the page's edit inputs a panel never blocks.
+ [FIRE, { text: "" }, null],
+ ];
+ for (const [c, inputs, error] of rows) {
+ const got = S.collectToolRequest(applying(c) as never, {}, inputs, c);
+ assert.equal(got.error, error);
+ assert.deepEqual(got.patch, {});
+ }
+});
+
+test("the registry lists the edit panels and the editor keeps its label", () => {
+ const registry = readSrc("features/audio/tools/registry.tsx");
+ assert.match(registry, /\.\.\.EDIT_TOOL_PANELS,/);
+ const editor = readSrc(
+ "features/audio/components/transcript-diff-editor.tsx",
+ );
+ assert.match(editor, /aria-label="Changes from the transcript"/);
+});
diff --git a/studio/frontend/tests/audio-edit-store.test.ts b/studio/frontend/tests/audio-edit-store.test.ts
new file mode 100644
index 00000000000..c9868524f80
--- /dev/null
+++ b/studio/frontend/tests/audio-edit-store.test.ts
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ installLocalStorageFake,
+ registerBundlerResolver,
+} from "./helpers/kit.ts";
+
+const { store } = installLocalStorageFake();
+registerBundlerResolver();
+
+const KEY = "unsloth_audio_edit_v1";
+const SAVED = {
+ source: { kind: "input", id: "in1", name: "take.wav", durationS: 4.7 },
+ transcript: "wasn't a human voice.",
+ transcriptFor: "in1",
+ edited: "wasn't a robot voice.",
+ editedTouched: true,
+ mode: "delivery",
+ delivery: { speed: 1.5, pitchSteps: 2 },
+};
+store.set(KEY, JSON.stringify({ state: SAVED, version: 1 }));
+store.set(
+ "unsloth_audio_clone_v1",
+ JSON.stringify({ state: { text: "Clone draft." }, version: 1 }),
+);
+
+const { AUDIO_EDIT_STORAGE_KEY, useAudioEditStore } = await import(
+ "../src/features/audio/stores/audio-edit-store.ts"
+);
+const { useAudioCloneStore } = await import(
+ "../src/features/audio/stores/audio-clone-store.ts"
+);
+const state = () => useAudioEditStore.getState();
+
+test("the edit draft comes back after a reload", () => {
+ assert.equal(AUDIO_EDIT_STORAGE_KEY, KEY);
+ for (const [key, value] of Object.entries(SAVED)) {
+ assert.deepEqual(state()[key as keyof typeof SAVED], value, key);
+ }
+});
+
+test("② follows ① until the user types in it, and Reset puts it back", () => {
+ const draft = () => [state().edited, state().transcriptFor];
+ state().resetEdited();
+ assert.deepEqual(draft(), ["wasn't a human voice.", "in1"]);
+ // A new recording's transcript resets ② while it is untouched.
+ state().setTranscript("It was a fine day.", "in2");
+ assert.deepEqual(draft(), ["It was a fine day.", "in2"]);
+ // Once typed in, ② stays; leaving transcriptFor out keeps its source.
+ state().setEdited("It was a rainy day.");
+ state().setTranscript("It was a fine day!");
+ assert.deepEqual(draft(), ["It was a rainy day.", "in2"]);
+ state().setDelivery({ pitchSteps: 4 });
+ assert.deepEqual(state().delivery, { speed: 1.5, pitchSteps: 4 });
+ const { state: saved, version } = JSON.parse(store.get(KEY) ?? "{}");
+ assert.deepEqual(
+ [version, saved.edited, saved.editedTouched],
+ [1, "It was a rainy day.", true],
+ );
+});
+
+test("editing never touches Clone's draft", () => {
+ state().setEdited("Something else.");
+ state().setSource(null);
+ assert.equal(useAudioCloneStore.getState().text, "Clone draft.");
+});
diff --git a/studio/frontend/tests/audio-handoff-intent.test.ts b/studio/frontend/tests/audio-handoff-intent.test.ts
index a11a01cf4b9..8eb7b12e3b6 100644
--- a/studio/frontend/tests/audio-handoff-intent.test.ts
+++ b/studio/frontend/tests/audio-handoff-intent.test.ts
@@ -56,7 +56,7 @@ test("the params the route accepted before pass through unchanged", () => {
});
test("a workflow param is kept when it names a workflow", () => {
- for (const workflow of ["speak", "clone", "music", "transcribe"]) {
+ for (const workflow of ["speak", "clone", "edit", "music", "transcribe"]) {
assert.deepEqual(validateAudioSearch({ workflow }), { workflow });
}
});
diff --git a/studio/frontend/tests/audio-page-policy.test.ts b/studio/frontend/tests/audio-page-policy.test.ts
index abbc3303781..27e08751444 100644
--- a/studio/frontend/tests/audio-page-policy.test.ts
+++ b/studio/frontend/tests/audio-page-policy.test.ts
@@ -842,3 +842,19 @@ test("a run that loads a model first says so before it starts", () => {
assert.equal(modelLoadNote({ model: "Kokoro", page: "Speak", seconds: 0.2 }), "Loads Kokoro for Speak, about 1 s");
assert.equal(modelLoadNote({ model: null, page: "Speak", seconds: 5 }), null);
});
+
+test("paging for a visible row skips hidden edit originals, like the list does", () => {
+ const gallery = readSrc("features/audio/hooks/use-audio-gallery.tsx");
+ assert.match(
+ gallery,
+ /const countVisible = \(\) =>\s*galleryCache\.clips\.filter\(\s*\(clip\) => clipWorkflow\(clip\) === workflow && clip\.role !== "source",\s*\)\.length;/,
+ );
+});
+
+test("trained speech checkpoints are offered only on Speak and Music", () => {
+ const host = readSrc("features/audio/audio-page.tsx");
+ assert.match(
+ host,
+ /ttsWorkflow !== "speak" && ttsWorkflow !== "music"\s*\?\s*\[\]\s*:\s*trainedTtsModels\.filter\(/,
+ );
+});
diff --git a/studio/frontend/tests/audio-picker-filter.test.ts b/studio/frontend/tests/audio-picker-filter.test.ts
index 52737cbeb3b..b70889f45f7 100644
--- a/studio/frontend/tests/audio-picker-filter.test.ts
+++ b/studio/frontend/tests/audio-picker-filter.test.ts
@@ -18,10 +18,12 @@ const { audioCppModelSpeaks } = await import(
"../src/features/audio/audio-cpp-catalog.ts"
);
-type Workflow = "speak" | "clone" | "music" | "transcribe";
-const WORKFLOWS: Workflow[] = ["speak", "clone", "music", "transcribe"];
+type Workflow = "speak" | "clone" | "edit" | "music" | "transcribe";
+const WORKFLOWS: Workflow[] = ["speak", "clone", "edit", "music", "transcribe"];
-function workflowsFor(row: Parameters<typeof audioRowMatchesWorkflow>[0]) {
+type Row = Parameters<typeof audioRowMatchesWorkflow>[0];
+
+function workflowsFor(row: Row) {
return WORKFLOWS.filter((workflow) => audioRowMatchesWorkflow(row, workflow));
}
@@ -170,6 +172,25 @@ test("an undefined rowFilter leaves the picker rows untouched", () => {
assert.match(selector, /rowFilter=\{rowFilter\}/);
});
+test("a speech editing model lists on Edit beside its other pages", () => {
+ const rows: [string, Partial<Row>, Workflow[]][] = [
+ [
+ "DotTTS-Edit-GGUF",
+ { audioWorkflows: ["speak", "edit"] },
+ ["speak", "edit"],
+ ],
+ // DotTTS's other packages do not edit.
+ ["DotTTS-MF-GGUF", { audioWorkflows: ["speak"] }, ["speak"]],
+ // Without the backend's list, the catalog seed decides.
+ ["Vevo2-GGUF", { audioType: "audiocpp_tts" }, ["clone", "edit"]],
+ ];
+ for (const [folder, row, workflows] of rows) {
+ const id = `audio-cpp/audio.cpp-gguf/${folder}`;
+ const got = workflowsFor({ id, task: "text-to-speech", ...row });
+ assert.deepEqual(got, workflows, folder);
+ }
+});
+
test("typed Hub search results are scoped to the page too", () => {
const pickers = readSrc("features/model-picker/components/model-selector/pickers.tsx");
const body = pickers.slice(pickers.indexOf("const searchIdsFrom = useCallback("));
diff --git a/studio/frontend/tests/audio-run-api.test.ts b/studio/frontend/tests/audio-run-api.test.ts
index 07bd3e620eb..b27501f11ef 100644
--- a/studio/frontend/tests/audio-run-api.test.ts
+++ b/studio/frontend/tests/audio-run-api.test.ts
@@ -140,3 +140,43 @@ test("api.ts sends run and voice bodies only through the builders", () => {
assert.match(api, /body: blob,/);
assert.doesNotMatch(api, /voice_ref/);
});
+
+test("an edit run sends the source by id and only the edit part's allowed keys", () => {
+ const markup = 'A <sub targ="robot">human</sub> voice.';
+ const body = buildAudioRunBody({
+ workflow: "edit",
+ text: "A robot voice.",
+ inputs: {
+ source: {
+ input_id: "in_1",
+ trim: { start_s: 0, end_s: 4 },
+ path: "/tmp/x.wav",
+ } as never,
+ reference_text: " A human voice. ",
+ },
+ edit: {
+ mode: "words",
+ markup,
+ instructions: ["", 3 as never],
+ speed: Number.NaN,
+ pitch_steps: 1.5,
+ path: "/tmp/y.wav",
+ } as never,
+ });
+ assert.deepEqual(body, {
+ workflow: "edit",
+ text: "A robot voice.",
+ inputs: {
+ source: { input_id: "in_1" },
+ reference_text: "A human voice.",
+ },
+ edit: { mode: "words", markup },
+ });
+ assert.doesNotMatch(JSON.stringify(body), PATHLIKE);
+ const editOf = (edit: object) =>
+ buildAudioRunBody({ workflow: "edit", text: "x", edit: edit as never })
+ .edit;
+ assert.equal(editOf({ mode: "bogus" }), undefined);
+ const delivery = { mode: "delivery", speed: 1.5, pitch_steps: 3 };
+ assert.deepEqual(editOf(delivery), delivery);
+});
diff --git a/studio/frontend/tests/audio-separate.test.ts b/studio/frontend/tests/audio-separate.test.ts
index 83ce643e24a..4f735a20b34 100644
--- a/studio/frontend/tests/audio-separate.test.ts
+++ b/studio/frontend/tests/audio-separate.test.ts
@@ -288,10 +288,10 @@ test("host and page wiring for Separate", () => {
);
assert.match(host, /ttsWorkflow === "separate"\s*\? separate\.blocker/);
assert.match(host, /reason: "The loaded model separates audio\."/);
- // Trained speech checkpoints are not separation models.
+ // Trained speech checkpoints are not separation models: only Speak and Music offer them.
assert.match(
host,
- /ttsWorkflow === "clone" \|\| ttsWorkflow === "separate"\s*\? \[\]/,
+ /ttsWorkflow !== "speak" && ttsWorkflow !== "music"\s*\? \[\]/,
);
// A stem sent to Transcribe mid-run would be dropped when the run stops.
assert.match(
diff --git a/studio/frontend/tests/audio-workflows.test.ts b/studio/frontend/tests/audio-workflows.test.ts
index 5d2b14b8bab..a36d6e40422 100644
--- a/studio/frontend/tests/audio-workflows.test.ts
+++ b/studio/frontend/tests/audio-workflows.test.ts
@@ -20,14 +20,16 @@ const {
workflowForLoadedModel,
} = await import("../src/features/audio/workflows.ts");
-test("the Audio page offers Speak, Clone, Music, Separate and Transcribe in that order", () => {
+test("the Audio page offers Speak, Clone, Edit, Music, Separate and Transcribe in that order", () => {
assert.deepEqual(
AUDIO_WORKFLOWS.map((tab) => tab.id),
- ["speak", "clone", "music", "separate", "transcribe"],
+ ["speak", "clone", "edit", "music", "separate", "transcribe"],
);
assert.equal(audioWorkflowTab("separate").heading, "Separate audio");
assert.equal(audioWorkflowTab("speak").heading, "Text to speech");
assert.equal(audioWorkflowTab("clone").heading, "Clone a voice");
+ assert.equal(audioWorkflowTab("edit").heading, "Edit speech");
+ assert.equal(audioWorkflowTab("edit").label, "Edit");
assert.equal(audioWorkflowTab("music").heading, "Create music");
assert.equal(audioWorkflowTab("transcribe").heading, "Transcribe");
});
@@ -35,6 +37,7 @@ test("the Audio page offers Speak, Clone, Music, Separate and Transcribe in that
test("Speak, Clone and Music share the main slot; only Transcribe uses the sidecar", () => {
assert.equal(slotForWorkflow("speak"), "speak");
assert.equal(slotForWorkflow("clone"), "speak");
+ assert.equal(slotForWorkflow("edit"), "speak");
assert.equal(slotForWorkflow("music"), "speak");
assert.equal(slotForWorkflow("separate"), "speak");
assert.equal(slotForWorkflow("transcribe"), "transcribe");
@@ -43,6 +46,7 @@ test("Speak, Clone and Music share the main slot; only Transcribe uses the sidec
test("Create|Train is offered on Speak, Clone and Transcribe, not Music", () => {
assert.equal(audioWorkflowTab("speak").createTrain, true);
assert.equal(audioWorkflowTab("clone").createTrain, true);
+ assert.equal(audioWorkflowTab("edit").createTrain, false);
assert.equal(audioWorkflowTab("music").createTrain, false);
assert.equal(audioWorkflowTab("separate").createTrain, false);
assert.equal(audioWorkflowTab("transcribe").createTrain, true);
@@ -66,8 +70,9 @@ test("a clip without a workflow falls back to its audio type", () => {
assert.equal(clipWorkflow({ audio_type: "snac" }), "speak");
assert.equal(clipWorkflow({ workflow: "music", audio_type: "snac" }), "music");
assert.equal(clipWorkflow({ workflow: "clone", audio_type: "audiocpp_tts" }), "clone");
+ assert.equal(clipWorkflow({ workflow: "edit", audio_type: "audiocpp_tts" }), "edit");
assert.equal(clipWorkflow({ workflow: "separate", audio_type: "audiocpp_sep" }), "separate");
- assert.equal(clipWorkflow({ workflow: "edit", audio_type: "audiocpp_music" }), "music");
+ assert.equal(clipWorkflow({ workflow: "remix", audio_type: "audiocpp_music" }), "music");
assert.equal(audioWorkflowForAudioType(null), "speak");
});
diff --git a/studio/frontend/tests/audio-workspace-split.test.ts b/studio/frontend/tests/audio-workspace-split.test.ts
index a914d3eb696..435d83ce714 100644
--- a/studio/frontend/tests/audio-workspace-split.test.ts
+++ b/studio/frontend/tests/audio-workspace-split.test.ts
@@ -73,7 +73,7 @@ test("Speak and Music keep separate drafts that survive a reload", () => {
test("history is per page, and Clear all clears only that page", () => {
assert.match(
gallery,
- /clips\.filter\(\(clip\) => clipWorkflow\(clip\) === workflow\)/,
+ /clips\.filter\(\(clip\) => clipWorkflow\(clip\) === workflow/,
);
assert.match(gallery, /await clearAudioGallery\(workflow\);/);
assert.match(host, /\(\) => handleClearGallery\(ttsWorkflow\)/);
@@ -94,7 +94,7 @@ test("Generate says why it is off and runs from Mod+Enter anywhere on the page",
assert.match(host, /!pageRootRef\.current\?\.contains\(event\.target\)/);
assert.match(
host,
- /const handlePageGenerate =\s*ttsWorkflow === "separate" \? separate\.handleGenerate :\s*ttsWorkflow === "clone" \? clone\.handleGenerate : handleGenerate;/,
+ /const handlePageGenerate =\s*ttsWorkflow === "separate" \? separate\.handleGenerate :\s*ttsWorkflow === "clone" \? clone\.handleGenerate :\s*ttsWorkflow === "edit" \? edit\.handleGenerate : handleGenerate;/,
);
assert.match(host, /if \(canGenerate\) void handlePageGenerate\(\);/);
});
diff --git a/studio/frontend/tests/fixtures/audio-edit-requests.json b/studio/frontend/tests/fixtures/audio-edit-requests.json
new file mode 100644
index 00000000000..e83b2fd9226
--- /dev/null
+++ b/studio/frontend/tests/fixtures/audio-edit-requests.json
@@ -0,0 +1,294 @@
+{
+ "_comment": "Golden Edit requests shared by tests/audio-edit-adapters.test.ts and backend tests/test_audio_edit_requests.py. run_body is POST /api/inference/audio/run; runtime is what the server posts to audio.cpp, with <model>, <recording> and <result of call N> standing in for server values.",
+ "cases": [
+ {
+ "name": "dots two subs",
+ "family": "dots_tts",
+ "input": {
+ "original": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "edited": "Okay, I'm Sam and what you just heard wasn't a robot voice.",
+ "mode": "words",
+ "advanced": {}
+ },
+ "run_body": {
+ "workflow": "edit",
+ "text": "Okay, I'm Sam and what you just heard wasn't a robot voice.",
+ "inputs": {
+ "source": {
+ "input_id": "in_0123456789abcdef"
+ },
+ "reference_text": "Okay, I'm Cemo and what you just heard wasn't a human voice."
+ },
+ "edit": {
+ "mode": "words",
+ "markup": "Okay, I'm <sub targ=\"Sam\">Cemo</sub> and what you just heard wasn't a <sub targ=\"robot\">human</sub> voice."
+ }
+ },
+ "runtime": [
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "text": "Okay, I'm <sub targ=\"Sam\">Cemo</sub> and what you just heard wasn't a <sub targ=\"robot\">human</sub> voice.",
+ "source_audio": "<recording>",
+ "options": {
+ "template_name": "edit"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "name": "dots delete and insert",
+ "family": "dots_tts",
+ "input": {
+ "original": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "edited": "Okay, I'm Cemo and what you heard really wasn't a human voice.",
+ "mode": "words",
+ "advanced": {}
+ },
+ "run_body": {
+ "workflow": "edit",
+ "text": "Okay, I'm Cemo and what you heard really wasn't a human voice.",
+ "inputs": {
+ "source": {
+ "input_id": "in_0123456789abcdef"
+ },
+ "reference_text": "Okay, I'm Cemo and what you just heard wasn't a human voice."
+ },
+ "edit": {
+ "mode": "words",
+ "markup": "Okay, I'm Cemo and what you <del>just</del> heard <ins>really</ins> wasn't a human voice."
+ }
+ },
+ "runtime": [
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "text": "Okay, I'm Cemo and what you <del>just</del> heard <ins>really</ins> wasn't a human voice.",
+ "source_audio": "<recording>",
+ "options": {
+ "template_name": "edit"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "name": "dots multi-word sub",
+ "family": "dots_tts",
+ "input": {
+ "original": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "edited": "Okay, I'm Cemo and what you just heard wasn't a robot sound.",
+ "mode": "words",
+ "advanced": {}
+ },
+ "run_body": {
+ "workflow": "edit",
+ "text": "Okay, I'm Cemo and what you just heard wasn't a robot sound.",
+ "inputs": {
+ "source": {
+ "input_id": "in_0123456789abcdef"
+ },
+ "reference_text": "Okay, I'm Cemo and what you just heard wasn't a human voice."
+ },
+ "edit": {
+ "mode": "words",
+ "markup": "Okay, I'm Cemo and what you just heard wasn't a <sub targ=\"robot sound.\">human voice.</sub>"
+ }
+ },
+ "runtime": [
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "text": "Okay, I'm Cemo and what you just heard wasn't a <sub targ=\"robot sound.\">human voice.</sub>",
+ "source_audio": "<recording>",
+ "options": {
+ "template_name": "edit"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "name": "vevo2 words",
+ "family": "vevo2",
+ "input": {
+ "original": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "edited": "Okay, I'm Cemo and what you just heard wasn't a robot voice.",
+ "mode": "words",
+ "advanced": {}
+ },
+ "run_body": {
+ "workflow": "edit",
+ "text": "Okay, I'm Cemo and what you just heard wasn't a robot voice.",
+ "inputs": {
+ "source": {
+ "input_id": "in_0123456789abcdef"
+ },
+ "reference_text": "Okay, I'm Cemo and what you just heard wasn't a human voice."
+ },
+ "edit": {
+ "mode": "words"
+ }
+ },
+ "runtime": [
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "route": "editing",
+ "source_audio": "<recording>",
+ "target_text": "Okay, I'm Cemo and what you just heard wasn't a robot voice.",
+ "reference_text": "Okay, I'm Cemo and what you just heard wasn't a human voice."
+ }
+ }
+ ]
+ },
+ {
+ "name": "firered two replaces chained",
+ "family": "firered_audio",
+ "input": {
+ "original": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "edited": "Okay, I'm Sam and what you just heard wasn't a robot voice.",
+ "mode": "words",
+ "advanced": {}
+ },
+ "run_body": {
+ "workflow": "edit",
+ "text": "Okay, I'm Sam and what you just heard wasn't a robot voice.",
+ "inputs": {
+ "source": {
+ "input_id": "in_0123456789abcdef"
+ },
+ "reference_text": "Okay, I'm Cemo and what you just heard wasn't a human voice."
+ },
+ "edit": {
+ "mode": "words",
+ "instructions": [
+ "Replace 'Cemo' with 'Sam'.",
+ "Replace 'human' with 'robot'."
+ ]
+ }
+ },
+ "runtime": [
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "audio": "<recording>",
+ "options": {
+ "template_name": "semantic_edit",
+ "instruction": "Replace 'Cemo' with 'Sam'."
+ }
+ }
+ },
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "audio": "<result of call 1>",
+ "options": {
+ "template_name": "semantic_edit",
+ "instruction": "Replace 'human' with 'robot'."
+ }
+ }
+ }
+ ]
+ },
+ {
+ "name": "firered insert before",
+ "family": "firered_audio",
+ "input": {
+ "original": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "edited": "Okay, I'm Cemo and what you just heard really wasn't a human voice.",
+ "mode": "words",
+ "advanced": {}
+ },
+ "run_body": {
+ "workflow": "edit",
+ "text": "Okay, I'm Cemo and what you just heard really wasn't a human voice.",
+ "inputs": {
+ "source": {
+ "input_id": "in_0123456789abcdef"
+ },
+ "reference_text": "Okay, I'm Cemo and what you just heard wasn't a human voice."
+ },
+ "edit": {
+ "mode": "words",
+ "instructions": [
+ "Insert 'really' before 'wasn't'."
+ ]
+ }
+ },
+ "runtime": [
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "audio": "<recording>",
+ "options": {
+ "template_name": "semantic_edit",
+ "instruction": "Insert 'really' before 'wasn't'."
+ }
+ }
+ }
+ ]
+ },
+ {
+ "name": "firered delivery speed and pitch",
+ "family": "firered_audio",
+ "input": {
+ "original": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "edited": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "mode": "delivery",
+ "delivery": {
+ "speed": 1.5,
+ "pitchSteps": 3
+ },
+ "advanced": {}
+ },
+ "run_body": {
+ "workflow": "edit",
+ "text": "Okay, I'm Cemo and what you just heard wasn't a human voice.",
+ "inputs": {
+ "source": {
+ "input_id": "in_0123456789abcdef"
+ },
+ "reference_text": "Okay, I'm Cemo and what you just heard wasn't a human voice."
+ },
+ "edit": {
+ "mode": "delivery",
+ "speed": 1.5,
+ "pitch_steps": 3
+ }
+ },
+ "runtime": [
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "audio": "<recording>",
+ "options": {
+ "template_name": "acoustic_edit",
+ "instruction": "adjust the speed to 1.5x"
+ }
+ }
+ },
+ {
+ "path": "/v1/tasks/run",
+ "body": {
+ "model": "<model>",
+ "audio": "<result of call 1>",
+ "options": {
+ "template_name": "acoustic_edit",
+ "instruction": "shift the pitch by 3 steps"
+ }
+ }
+ }
+ ]
+ }
+ ]
+}